From 4dde39e3401de8e54758a54a89f0b874a226aa14 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 11:43:25 -0700 Subject: [PATCH 01/23] feat(sync): compute a checkpoint's replay pin (T12343 S4-1b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit migrationJournalHash hashes the whole ordered __drizzle_migrations journal ((created_at, name) order); triggerSetHash hashes every non-capture trigger's DDL by name, normalized literal-safely (normalizeTriggerDdl: comments dropped, layout and word case folded outside literals, literals byte for byte, identifier quoting unified). replayPinOf assembles the wire ReplayPin a checkpoint/v3 manifest carries (journal spec §2.11 §7). Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-replay-pin.md | 20 +++ .../store/sync/__tests__/replay-pin.test.ts | 111 ++++++++++++ packages/core/src/store/sync/replay-pin.ts | 168 ++++++++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 .changeset/t12343-replay-pin.md create mode 100644 packages/core/src/store/sync/__tests__/replay-pin.test.ts create mode 100644 packages/core/src/store/sync/replay-pin.ts diff --git a/.changeset/t12343-replay-pin.md b/.changeset/t12343-replay-pin.md new file mode 100644 index 0000000000..50dd43cf2c --- /dev/null +++ b/.changeset/t12343-replay-pin.md @@ -0,0 +1,20 @@ +--- +id: t12343-replay-pin +tasks: [T12343] +kind: feat +summary: Sync checkpoints compute their replay pin - the ordered migration journal hash and a literal-safe trigger-set hash +--- + +A checkpoint/v3 manifest carries a `replayPin` (journal spec §2.11 §7). It records what an endorsing device must replay under to +recompute the checkpoint's tallies (T12343 S4-1b). + +- **`migrationJournalHash(db)`** is sha256 over the whole ordered `__drizzle_migrations` journal, as `[created_at, name, hash]` in + `(created_at, name)` order. A drifted row below the head changes it. +- **`triggerSetHash(db)`** is sha256 over every non-capture trigger's DDL, sorted by name. Capture triggers are excluded, because they + are generated per build and never run in a replay. +- **`normalizeTriggerDdl(sql)`** normalizes that DDL literal-safely: + - comments are dropped; + - outside literals, whitespace collapses and words are lower-cased; + - string literals are kept byte for byte; + - every quoted identifier is written `"x"`. +- **`replayPinOf(db, transitions)`** assembles the wire `ReplayPin`. A genesis checkpoint passes no transitions. diff --git a/packages/core/src/store/sync/__tests__/replay-pin.test.ts b/packages/core/src/store/sync/__tests__/replay-pin.test.ts new file mode 100644 index 0000000000..6b99d8b75c --- /dev/null +++ b/packages/core/src/store/sync/__tests__/replay-pin.test.ts @@ -0,0 +1,111 @@ +/** + * The replay pin (journal spec §2.11 §7; T12343 S4-1b): the ordered + * migration journal hash, and the literal-safe trigger-set hash. + * + * @task T12343 + */ + +import { DatabaseSync } from 'node:sqlite'; +import { ReplayPin } from '@cleocode/contracts/cloud'; +import { describe, expect, it } from 'vitest'; +import { + migrationJournalHash, + normalizeTriggerDdl, + replayPinOf, + triggerSetHash, +} from '../replay-pin.js'; + +const GUARD = `CREATE TRIGGER g BEFORE UPDATE ON t + WHEN NEW.status = 'Done' AND OLD.status IS NOT 'Done' + BEGIN SELECT RAISE(ABORT, 'no way'); END`; + +function store(): DatabaseSync { + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, status TEXT)'); + db.exec( + 'CREATE TABLE "__drizzle_migrations" (id INTEGER PRIMARY KEY, hash TEXT NOT NULL, created_at NUMERIC, name TEXT, applied_at TEXT)', + ); + return db; +} + +describe('normalizeTriggerDdl (literal-safe)', () => { + it('ignores layout, keyword case and comments outside literals', () => { + const relaid = `create trigger G + before update on T -- a comment + when new.STATUS='Done' and old.status is not 'Done' + begin select raise ( abort , 'no way' ) ; end;`; + expect(normalizeTriggerDdl(relaid)).toBe(normalizeTriggerDdl(GUARD)); + }); + + it('keeps string literals byte for byte', () => { + expect(normalizeTriggerDdl(GUARD.replaceAll("'Done'", "'done'"))).not.toBe( + normalizeTriggerDdl(GUARD), + ); + // Whitespace inside a literal is content. + expect(normalizeTriggerDdl(GUARD.replace("'no way'", "'no way'"))).not.toBe( + normalizeTriggerDdl(GUARD), + ); + expect(normalizeTriggerDdl("SELECT 'it''s'")).toBe("select 'it''s'"); + }); + + it('unifies identifier quoting', () => { + const forms = ['"t"', '`t`', '[t]'].map((q) => + normalizeTriggerDdl(`CREATE TRIGGER g AFTER INSERT ON ${q} BEGIN SELECT 1; END`), + ); + expect(new Set(forms).size).toBe(1); + }); +}); + +describe('triggerSetHash', () => { + it('covers every non-capture trigger, in name order, and ignores capture triggers', () => { + const a = store(); + const b = store(); + a.exec(GUARD); + a.exec("CREATE TRIGGER h AFTER INSERT ON t BEGIN SELECT 'h'; END"); + b.exec("CREATE TRIGGER h AFTER INSERT ON t BEGIN SELECT 'h'; END"); // other creation order + b.exec(GUARD); + expect(triggerSetHash(a)).toBe(triggerSetHash(b)); + const before = triggerSetHash(a); + a.exec("CREATE TRIGGER _sync_cap_t_i AFTER INSERT ON t BEGIN SELECT 'cap'; END"); + expect(triggerSetHash(a), 'a capture trigger changed the pin').toBe(before); + a.exec('DROP TRIGGER g'); + a.exec(GUARD.replaceAll("'Done'", "'done'")); + expect(triggerSetHash(a), 'a guard literal change kept the pin').not.toBe(before); + }); +}); + +describe('migrationJournalHash', () => { + it('hashes the whole journal in (created_at, name) order, whatever the row order', () => { + const a = store(); + const b = store(); + const ins = (db: DatabaseSync, hash: string, at: number, name: string) => + db + .prepare('INSERT INTO "__drizzle_migrations" (hash, created_at, name) VALUES (?, ?, ?)') + .run(hash, at, name); + ins(a, 'h1', 1, 'm1'); + ins(a, 'h2', 2, 'm2'); + ins(b, 'h2', 2, 'm2'); // inserted out of order + ins(b, 'h1', 1, 'm1'); + expect(migrationJournalHash(a)).toBe(migrationJournalHash(b)); + // A drifted row below the head changes it. + a.prepare("UPDATE \"__drizzle_migrations\" SET hash = 'hx' WHERE name = 'm1'").run(); + expect(migrationJournalHash(a)).not.toBe(migrationJournalHash(b)); + }); + + it('a store without the journal hashes an empty list', () => { + const db = new DatabaseSync(':memory:'); + expect(migrationJournalHash(db)).toMatch(/^[0-9a-f]{64}$/); + }); +}); + +describe('replayPinOf', () => { + it('is a valid wire replay pin and carries the transitions it is given', () => { + const db = store(); + db.exec(GUARD); + const rise = { seq: 4, schemaVersion: 2, journal: 'a'.repeat(64) }; + const pin = replayPinOf(db, [rise]); + expect(ReplayPin.safeParse(pin).error?.issues ?? []).toEqual([]); + expect(pin.transitions).toEqual([rise]); + expect(replayPinOf(db).transitions).toEqual([]); + }); +}); diff --git a/packages/core/src/store/sync/replay-pin.ts b/packages/core/src/store/sync/replay-pin.ts new file mode 100644 index 0000000000..8495e03a2d --- /dev/null +++ b/packages/core/src/store/sync/replay-pin.ts @@ -0,0 +1,168 @@ +/** + * The replay pin of a checkpoint (journal spec §2.11 §7; NEW-9, R5-8, + * T12797): what an endorsing device must replay under to recompute the + * checkpoint's tallies. + * + * - `journal`: sha256 over the store's WHOLE ordered migration journal, + * `(created_at, name, hash)` per applied migration, in drizzle's order + * `(created_at, name)`. A single head is not enough: stamped and drifted + * stores differ below it. + * - `triggerSetHash`: sha256 over the normalized DDL of every non-capture + * trigger (guards, claim release, the AC graveyard, side effects), sorted + * by name. Normalization is literal-safe ({@link normalizeTriggerDdl}). + * - `transitions`: the schema rise points inside the checkpoint's window; + * the caller computes them from the segments it replayed (empty for a + * genesis checkpoint, which has no window to replay). + * + * The server cannot check the two hashes; another device recomputes them + * from its own store and requires a match before it endorses. + * + * @task T12343 + * @module store/sync/replay-pin + */ + +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; +import type { ReplayPin } from '@cleocode/contracts/cloud'; +import { hasTable } from './schema.js'; +import { canonicalJson } from './sealer-values.js'; +import { CAPTURE_TRIGGER_PREFIX } from './trigger-classes.js'; + +const sha256Hex = (s: string): string => createHash('sha256').update(s, 'utf8').digest('hex'); + +/** + * Normalize trigger DDL for hashing, literal-safely (§2.11 §7): comments are + * dropped; outside string literals and quoted identifiers, whitespace runs + * collapse to one space (and vanish next to punctuation) and words are + * lower-cased; a string literal is kept byte for byte (`'Done'` stays + * `'Done'`); every quoted identifier (`"x"`, `` `x` ``, `[x]`) is written + * `"x"`, and a bare identifier stays bare (SQLite treats both alike for the + * names a trigger uses, so the two spellings of one trigger hash alike only + * when quoting is unified this way). + * + * @param sql - The trigger's `CREATE TRIGGER` text. + * @returns The normalized text. + */ +export function normalizeTriggerDdl(sql: string): string { + const out: string[] = []; + let i = 0; + let space = false; + const push = (tok: string, word: boolean): void => { + const prev = out[out.length - 1]; + // One space between two word-like tokens; none around punctuation. + if (space && prev !== undefined && word && /[\w"'\]`]$/.test(prev)) out.push(' '); + out.push(tok); + space = false; + }; + while (i < sql.length) { + const c = sql[i] as string; + if (/\s/.test(c)) { + space = true; + i += 1; + } else if (sql.startsWith('--', i)) { + const end = sql.indexOf('\n', i); + i = end === -1 ? sql.length : end; + space = true; + } else if (sql.startsWith('/*', i)) { + const end = sql.indexOf('*/', i + 2); + i = end === -1 ? sql.length : end + 2; + space = true; + } else if (c === "'") { + let j = i + 1; + for (;;) { + if (j >= sql.length) break; + if (sql[j] === "'") { + if (sql[j + 1] === "'") j += 2; + else break; + } else j += 1; + } + push(sql.slice(i, j + 1), true); + i = j + 1; + } else if (c === '"' || c === '`' || c === '[') { + const close = c === '[' ? ']' : c; + let j = i + 1; + let name = ''; + for (;;) { + if (j >= sql.length) break; + if (sql[j] === close) { + if (close !== ']' && sql[j + 1] === close) { + name += close; + j += 2; + } else break; + } else { + name += sql[j]; + j += 1; + } + } + push(`"${name.replaceAll('"', '""')}"`, true); + i = j + 1; + } else if (/[\w$]/.test(c)) { + let j = i; + while (j < sql.length && /[\w$]/.test(sql[j] as string)) j += 1; + push(sql.slice(i, j).toLowerCase(), true); + i = j; + } else { + push(c, false); + i += 1; + } + } + return out.join('').replace(/;$/, ''); +} + +/** + * sha256 of the store's whole ordered migration journal (§2.11 §7): every + * `__drizzle_migrations` row as `[created_at, name, hash]`, ordered by + * `(created_at, name)`. A store without the journal hashes an empty list. + * + * @param db - The store. + * @returns The hex digest. + */ +export function migrationJournalHash(db: DatabaseSync): string { + const rows = hasTable(db, '__drizzle_migrations') + ? (db + .prepare( + 'SELECT created_at, name, hash FROM main."__drizzle_migrations" ORDER BY created_at, name', + ) + .all() as Array<{ created_at: number | bigint | null; name: string | null; hash: string }>) + : []; + return sha256Hex( + canonicalJson( + rows.map((r) => [r.created_at === null ? null : Number(r.created_at), r.name, r.hash]), + ), + ); +} + +/** + * sha256 over every non-capture trigger's normalized DDL, sorted by name + * (§2.11 §7). Capture triggers are excluded: they are generated per build + * from the sync set and never run in a replay. + * + * @param db - The store. + * @returns The hex digest. + */ +export function triggerSetHash(db: DatabaseSync): string { + const rows = ( + db + .prepare("SELECT name, sql FROM main.sqlite_master WHERE type = 'trigger' ORDER BY name") + .all() as Array<{ name: string; sql: string | null }> + ).filter((r) => !r.name.startsWith(CAPTURE_TRIGGER_PREFIX) && r.sql !== null); + return sha256Hex(canonicalJson(rows.map((r) => [r.name, normalizeTriggerDdl(r.sql as string)]))); +} + +/** + * A checkpoint's replay pin, from the store it snapshots (§2.11 §7). + * + * @param db - The store (or the snapshot of it the checkpoint carries). + * @param transitions - The window's schema rise points (empty for genesis). + * @returns The pin. + */ +export function replayPinOf( + db: DatabaseSync, + transitions: ReplayPin['transitions'] = [], +): ReplayPin { + return { + journal: migrationJournalHash(db), + triggerSetHash: triggerSetHash(db), + transitions: [...transitions], + }; +} From aa68b99230b2cb9e4f161a14fe659be5d53a1728 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 12:46:59 -0700 Subject: [PATCH 02/23] fix(sync): the replay pin hashes owned triggers only (T13298) triggerSetHash covered every non-capture trigger, including device-local twin-collapse triggers: the docs freeze embeds the build that first froze that store, and the track triggers exist only on stores with legacy history. Devices with identical replay semantics computed different pins, so endorsement could never match. It now hashes only OWNED_TRIGGERS (the guard and side-effect triggers a replay runs under), before any checkpoint/v3 manifest carries the hash. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-replay-pin.md | 8 +++-- .../store/sync/__tests__/replay-pin.test.ts | 36 +++++++++++++------ packages/core/src/store/sync/replay-pin.ts | 21 ++++++----- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/.changeset/t12343-replay-pin.md b/.changeset/t12343-replay-pin.md index 50dd43cf2c..5634bd546a 100644 --- a/.changeset/t12343-replay-pin.md +++ b/.changeset/t12343-replay-pin.md @@ -1,6 +1,6 @@ --- id: t12343-replay-pin -tasks: [T12343] +tasks: [T12343, T13298] kind: feat summary: Sync checkpoints compute their replay pin - the ordered migration journal hash and a literal-safe trigger-set hash --- @@ -10,8 +10,10 @@ recompute the checkpoint's tallies (T12343 S4-1b). - **`migrationJournalHash(db)`** is sha256 over the whole ordered `__drizzle_migrations` journal, as `[created_at, name, hash]` in `(created_at, name)` order. A drifted row below the head changes it. -- **`triggerSetHash(db)`** is sha256 over every non-capture trigger's DDL, sorted by name. Capture triggers are excluded, because they - are generated per build and never run in a replay. +- **`triggerSetHash(db)`** is sha256 over the DDL of the store's owned triggers (`OWNED_TRIGGERS`: the guard and side-effect + triggers a replay runs under), sorted by name. Capture triggers (generated per build) and device-local maintenance triggers (the + twin-collapse docs freeze, whose message names the build that froze that store, and the legacy track triggers) are excluded. + Otherwise two devices with the same replay semantics would compute different pins and never endorse each other (T13298). - **`normalizeTriggerDdl(sql)`** normalizes that DDL literal-safely: - comments are dropped; - outside literals, whitespace collapses and words are lower-cased; diff --git a/packages/core/src/store/sync/__tests__/replay-pin.test.ts b/packages/core/src/store/sync/__tests__/replay-pin.test.ts index 6b99d8b75c..c3e2340918 100644 --- a/packages/core/src/store/sync/__tests__/replay-pin.test.ts +++ b/packages/core/src/store/sync/__tests__/replay-pin.test.ts @@ -57,20 +57,36 @@ describe('normalizeTriggerDdl (literal-safe)', () => { }); describe('triggerSetHash', () => { - it('covers every non-capture trigger, in name order, and ignores capture triggers', () => { + /** An owned guard trigger (its name is in OWNED_TRIGGERS), with GUARD's body. */ + const OWNED = GUARD.replace( + 'CREATE TRIGGER g ', + 'CREATE TRIGGER tasks_tasks_parent_cycle_guard_insert ', + ); + const freeze = (version: string) => + `CREATE TRIGGER t12535_freeze_docs_insert BEFORE INSERT ON t BEGIN SELECT RAISE(ABORT, 'docs frozen by ${version}'); END`; + + it('hashes the owned triggers only: device-local freeze and track triggers never split two devices (T13298)', () => { const a = store(); const b = store(); - a.exec(GUARD); - a.exec("CREATE TRIGGER h AFTER INSERT ON t BEGIN SELECT 'h'; END"); - b.exec("CREATE TRIGGER h AFTER INSERT ON t BEGIN SELECT 'h'; END"); // other creation order - b.exec(GUARD); + a.exec(OWNED); + b.exec(OWNED.replaceAll('\n', ' ').replace('BEFORE UPDATE', 'before update')); // same trigger, other layout + // Device-local maintenance: frozen by different builds, track triggers on one store only. + a.exec(freeze('2026.10.3')); + b.exec(freeze('2026.10.5')); + a.exec("CREATE TRIGGER t12535_track_docs_insert AFTER INSERT ON t BEGIN SELECT 'track'; END"); + a.exec("CREATE TRIGGER _sync_cap_t_i AFTER INSERT ON t BEGIN SELECT 'cap'; END"); expect(triggerSetHash(a)).toBe(triggerSetHash(b)); + }); + + it('changes when an owned trigger changes', () => { + const a = store(); + a.exec(OWNED); const before = triggerSetHash(a); - a.exec("CREATE TRIGGER _sync_cap_t_i AFTER INSERT ON t BEGIN SELECT 'cap'; END"); - expect(triggerSetHash(a), 'a capture trigger changed the pin').toBe(before); - a.exec('DROP TRIGGER g'); - a.exec(GUARD.replaceAll("'Done'", "'done'")); - expect(triggerSetHash(a), 'a guard literal change kept the pin').not.toBe(before); + a.exec('DROP TRIGGER tasks_tasks_parent_cycle_guard_insert'); + a.exec(OWNED.replaceAll("'Done'", "'done'")); + expect(triggerSetHash(a), 'an owned guard literal change kept the pin').not.toBe(before); + a.exec('DROP TRIGGER tasks_tasks_parent_cycle_guard_insert'); + expect(triggerSetHash(a), 'a dropped owned guard kept the pin').not.toBe(before); }); }); diff --git a/packages/core/src/store/sync/replay-pin.ts b/packages/core/src/store/sync/replay-pin.ts index 8495e03a2d..ec98fdf791 100644 --- a/packages/core/src/store/sync/replay-pin.ts +++ b/packages/core/src/store/sync/replay-pin.ts @@ -7,9 +7,13 @@ * `(created_at, name, hash)` per applied migration, in drizzle's order * `(created_at, name)`. A single head is not enough: stamped and drifted * stores differ below it. - * - `triggerSetHash`: sha256 over the normalized DDL of every non-capture - * trigger (guards, claim release, the AC graveyard, side effects), sorted - * by name. Normalization is literal-safe ({@link normalizeTriggerDdl}). + * - `triggerSetHash`: sha256 over the normalized DDL of the store's OWNED + * triggers ({@link OWNED_TRIGGERS}: the guard and side-effect triggers a + * replay runs under), sorted by name. Device-local maintenance triggers + * (the twin-collapse docs freeze, whose message names the build that froze + * that store, and the legacy track triggers) never touch a replayed table, + * so they stay out: two devices with the same replay semantics agree + * (T13298). Normalization is literal-safe ({@link normalizeTriggerDdl}). * - `transitions`: the schema rise points inside the checkpoint's window; * the caller computes them from the segments it replayed (empty for a * genesis checkpoint, which has no window to replay). @@ -26,7 +30,7 @@ import type { DatabaseSync } from 'node:sqlite'; import type { ReplayPin } from '@cleocode/contracts/cloud'; import { hasTable } from './schema.js'; import { canonicalJson } from './sealer-values.js'; -import { CAPTURE_TRIGGER_PREFIX } from './trigger-classes.js'; +import { OWNED_TRIGGERS } from './trigger-classes.js'; const sha256Hex = (s: string): string => createHash('sha256').update(s, 'utf8').digest('hex'); @@ -133,9 +137,10 @@ export function migrationJournalHash(db: DatabaseSync): string { } /** - * sha256 over every non-capture trigger's normalized DDL, sorted by name - * (§2.11 §7). Capture triggers are excluded: they are generated per build - * from the sync set and never run in a replay. + * sha256 over the normalized DDL of every owned trigger present in the store + * ({@link OWNED_TRIGGERS}), sorted by name (§2.11 §7; T13298). Capture + * triggers (generated per build) and device-local maintenance triggers are + * excluded: they never run in a replay. * * @param db - The store. * @returns The hex digest. @@ -145,7 +150,7 @@ export function triggerSetHash(db: DatabaseSync): string { db .prepare("SELECT name, sql FROM main.sqlite_master WHERE type = 'trigger' ORDER BY name") .all() as Array<{ name: string; sql: string | null }> - ).filter((r) => !r.name.startsWith(CAPTURE_TRIGGER_PREFIX) && r.sql !== null); + ).filter((r) => Object.hasOwn(OWNED_TRIGGERS, r.name) && r.sql !== null); return sha256Hex(canonicalJson(rows.map((r) => [r.name, normalizeTriggerDdl(r.sql as string)]))); } From 1ffe3b42b83c01093b77522d2d52f9af25ffc905 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 12:57:29 -0700 Subject: [PATCH 03/23] fix(sync): genesis cut commits before its snapshot; same-process writers wait (T13296, T13297) T13296: the bundle was snapshotted before closeCut wrote, so a device restored from the genesis checkpoint lacked the baseline row meta, the genesis keys, undo_enabled, sync.push and the folds. cutGenesisWithSnapshot now commits the cut first and snapshots after, under the genesis marker. A failed snapshot undoes the cut (keys dropped, push off, folded txns back to sealed, undo off and dropped when no stream is cut); so does a write that reached the store during the snapshot (GenesisRacedError), so no bundle holding a post-cut write is pushed. T13297: the marker's own pid was exempt, so in-process writers walked into the cut. A genesis marker now blocks every writer at the write chokepoint, the holder's own process included; the cut writes nothing during its snapshot. The changeset now names the chokepoint's actual callers instead of claiming every write path. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-genesis-snapshot.md | 31 ++-- packages/core/src/store/restore-marker.ts | 35 +++-- .../src/store/sync/__tests__/genesis.test.ts | 133 +++++++++++++----- packages/core/src/store/sync/genesis.ts | 101 ++++++++++--- scripts/lint-no-raw-table-writes.mjs | 17 ++- 5 files changed, 243 insertions(+), 74 deletions(-) diff --git a/.changeset/t12343-genesis-snapshot.md b/.changeset/t12343-genesis-snapshot.md index 49c94111d4..a98059ba44 100644 --- a/.changeset/t12343-genesis-snapshot.md +++ b/.changeset/t12343-genesis-snapshot.md @@ -1,22 +1,27 @@ --- id: t12343-genesis-snapshot -tasks: [T12343] +tasks: [T12343, T13296, T13297] kind: feat -summary: Sync genesis cut snapshots its checkpoint bundle at the cut; other cleo processes wait or refuse cleanly +summary: Sync genesis cut commits, then snapshots its checkpoint bundle under a genesis marker; writers wait or refuse cleanly --- -The genesis checkpoint's bundle must be the store exactly at the cut (journal spec §2.11 §10). Otherwise a write made after the -cut would travel both in the bundle and in a segment, and restore would apply it twice (T12343 S4-1b). +The genesis checkpoint's bundle must be the store exactly as the cut left it (journal spec §2.11 §10; T12343 S4-1b). Otherwise a +write made after the cut would travel both in the bundle and in a segment, and restore would apply it twice. -- **`cutGenesisWithSnapshot(db, {…, dbPath}, snapshot)`** runs `snapshot(cut)` while the cut's `BEGIN IMMEDIATE` is held, before the - cut commits. A snapshot taken on another connection sees exactly the cut state, because nothing else can commit. A failing snapshot - rolls the cut back. `cutGenesis` shares the same two phases. -- **A `genesis` store marker.** The export can outlast another writer's 30 s busy timeout, so for its duration the store carries a - `genesis` marker (the restore-in-progress marker, T13258). Another cleo process: - - waits at its store open, then refuses with `E_STORE_GENESIS` and a remedy; - - waits at the write chokepoint too (`assertExodusWriteSafe`, which now calls `awaitStoreWritable`, without blocking the event - loop), then refuses the same way. A write that waits it out lands after the snapshot. +- **`cutGenesisWithSnapshot(db, {…, dbPath}, snapshot)`** commits the cut FIRST, then runs `snapshot(cut)` (T13296). The bundle + therefore carries everything the cut wrote: genesis row meta, the genesis keys, `undo_enabled`, `sync.push` and the folded + transactions. A device restored from it has full row meta and never pushes a folded transaction again. +- **A failed snapshot undoes the cut.** It drops the stream's genesis keys, turns push off, returns the folded transactions to + `sealed`, and turns undo off (dropping undo written since) when no other stream is cut. The same happens when a write reached the + store during the snapshot (the capture position moved past the cut): `GenesisRacedError` (`E_SYNC_GENESIS_RACED`) is thrown, so no + bundle holding a post-cut write is ever pushed. +- **A `genesis` store marker** (the restore-in-progress marker, T13258) covers the whole run: + - another process waits at its store open, then refuses with `E_STORE_GENESIS` and a remedy; + - every writer, in this process too (T13297), waits at the write chokepoint without blocking the event loop, then refuses the same + way. A write that waits it out lands after the snapshot. - The holder itself is exempt, and the marker is always released. + The write chokepoint is `assertExodusWriteSafe`, which now calls `awaitStoreWritable`. Its callers are the task accessor's write + transactions, session creation, `insertIdempotent` and `upsertIdempotent`. A writer that bypasses those is not paused; the race + check above catches its write instead. The marker is always released. - **The restore-in-progress marker** (`store/restore-marker.ts`, `store/pid-alive.ts` and the open-path checks) comes from #1902 (T13258), cherry-picked unchanged. diff --git a/packages/core/src/store/restore-marker.ts b/packages/core/src/store/restore-marker.ts index 592af09496..addf8fc27e 100644 --- a/packages/core/src/store/restore-marker.ts +++ b/packages/core/src/store/restore-marker.ts @@ -64,11 +64,19 @@ function readMarker(dbPath: string): RestoreMarker | 'unreadable' | null { } } -/** Whether a marker blocks this process: held by another live process (or unreadable). */ -function blocks(marker: RestoreMarker | 'unreadable' | null): boolean { +/** + * Whether a marker blocks this process: held by another live process (or + * unreadable). A writer is blocked by its OWN process's `genesis` marker too + * (T13297): the genesis cut writes nothing while its snapshot runs, so any + * write then, from another handle or code path in the same process, would + * land after the cut and before the snapshot. + */ +function blocks(marker: RestoreMarker | 'unreadable' | null, forWrite = false): boolean { if (marker === null) return false; if (marker === 'unreadable') return true; - if (marker.host === hostname() && marker.pid === process.pid) return false; + if (marker.host === hostname() && marker.pid === process.pid) { + return forWrite && marker.kind === 'genesis'; + } // Stale: the restorer is gone. A marker from another host is honoured. return !(marker.host === hostname() && !isPidAlive(marker.pid)); } @@ -129,11 +137,16 @@ function markerRefusal(dbPath: string, marker: RestoreMarker | 'unreadable' | nu /** * The write-chokepoint form of {@link assertStoreNotRestoring}: a handle - * opened before another process took the marker must not write while the - * store is replaced or snapshotted (a genesis cut holds the write lock for - * its whole export, longer than a writer's busy timeout). Waits without - * blocking the event loop, up to `waitMs`, then refuses. Costs one - * `existsSync` when no marker is present. + * must not write while its store is replaced or snapshotted, including a + * handle opened before the marker was taken, and, for a genesis snapshot, any + * writer in the marker's own process (T13297). Waits without blocking the + * event loop, up to `waitMs`, then refuses. Costs one `existsSync` when no + * marker is present. + * + * Called from `assertExodusWriteSafe`, which the task accessor's write + * transactions, session creation, `insertIdempotent` and `upsertIdempotent` + * call first. A writer that bypasses those meets no wait; the genesis cut + * detects such a write after its snapshot and undoes itself. * * @param dbPath - The store file about to be written (null for an in-memory database: never blocks). * @param waitMs - How long to wait (default {@link RESTORE_MARKER_WAIT_MS}, or `CLEO_RESTORE_WAIT_MS`). @@ -146,13 +159,13 @@ export async function awaitStoreWritable( ): Promise { if (dbPath === null) return; let marker = readMarker(dbPath); - if (!blocks(marker)) return; + if (!blocks(marker, true)) return; const deadline = Date.now() + waitMs; - while (blocks(marker) && Date.now() < deadline) { + while (blocks(marker, true) && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 100)); marker = readMarker(dbPath); } - if (!blocks(marker)) return; + if (!blocks(marker, true)) return; // @sync-invariant none:local-only refused while another process replaces or snapshots the local store; nothing is written throw markerRefusal(dbPath, marker); } diff --git a/packages/core/src/store/sync/__tests__/genesis.test.ts b/packages/core/src/store/sync/__tests__/genesis.test.ts index de4a0e55d7..c909f07607 100644 --- a/packages/core/src/store/sync/__tests__/genesis.test.ts +++ b/packages/core/src/store/sync/__tests__/genesis.test.ts @@ -14,6 +14,7 @@ import { DatabaseSync } from 'node:sqlite'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { _resetDualScopeDbCache, + assertExodusWriteSafe, getDualScopeNativeDb, openDualScopeDbAtPath, } from '../../dual-scope-db.js'; @@ -31,6 +32,7 @@ import { GENESIS_CUT_KEY_PREFIX, GENESIS_PENDING_KEY_PREFIX, GENESIS_SOURCE_SEQ_KEY_PREFIX, + GenesisRacedError, genesisCutOf, genesisPending, UNDO_ENABLED_KEY, @@ -303,7 +305,7 @@ describe('genesis preconditions (T13032 AC2 step 0): a refusal cuts nothing', () }); }); -describe('genesis cut with its checkpoint snapshot (S4-1b)', () => { +describe('genesis cut with its checkpoint snapshot (S4-1b; T13296, T13297)', () => { const opts = () => ({ scope: 'project' as const, stream: STREAM, @@ -312,49 +314,112 @@ describe('genesis cut with its checkpoint snapshot (S4-1b)', () => { env: {}, allowUnreleased: true, }); + const sealedPacker = (db: DatabaseSync, replica: string) => + buildSegment(db, { + stream: STREAM, + replica, + scope: 'project', + project: null, + sealer: (_seq, plaintext) => Buffer.from(plaintext), + signTxn: (_stream, txn) => txn, + nowIso: new Date(++clock).toISOString(), + }); - it('the snapshot runs at the cut, under the write lock and the genesis marker, before the cut commits', async () => { - const { db } = await store(); + it('the snapshot is taken after the cut commits: a device restored from it has full row meta and never pushes a folded transaction (T13296)', async () => { + const { db, replica } = await store(); write(db, addTask('T1')); - let seen: { cut: number; marker: string; cutKeys: number; tasks: number; busy: string } | null = - null; - const r = await cutGenesisWithSnapshot(db, opts(), async (cut) => { - // Another connection, as the bundle export uses: the store exactly at the cut. - const other = new DatabaseSync(dbPath); + seal(db); + const copy = join(dir, 'bundle.db'); + let marker = ''; + const r = await cutGenesisWithSnapshot(db, opts(), async () => { + marker = JSON.parse(readFileSync(dbPath + RESTORE_MARKER_SUFFIX, 'utf8')).kind; + // As the bundle export does: a read-only connection, VACUUM INTO. + const src = new DatabaseSync(dbPath, { readOnly: true }); try { - other.exec('PRAGMA busy_timeout = 0'); - let busy = 'none'; - try { - other.exec("INSERT INTO _sync_meta (key, value, updated_at) VALUES ('x', 'x', 'x')"); - } catch (err) { - busy = (err as Error).message; - } - seen = { - cut, - marker: JSON.parse(readFileSync(dbPath + RESTORE_MARKER_SUFFIX, 'utf8')).kind, - cutKeys: n(other, "SELECT count(*) AS n FROM _sync_meta WHERE key LIKE 'genesis_cut:%'"), - tasks: n(other, 'SELECT count(*) AS n FROM tasks_tasks'), - busy, - }; + src.exec(`VACUUM INTO '${copy}'`); } finally { - other.close(); + src.close(); } }); - expect(r.refused).toBeNull(); - expect(seen).toEqual({ - cut: r.cut, - marker: 'genesis', - cutKeys: 0, // the cut is not committed while the snapshot runs - tasks: 2, // T0 and T1: every pre-cut effect - busy: expect.stringMatching(/locked|busy/i), // nothing else can commit + expect(r).toMatchObject({ refused: null, folded: 1 }); + expect(marker).toBe('genesis'); + expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); + const restored = new DatabaseSync(copy); + try { + // Every row has its meta (T0 was written before capture: baselined by the cut). + expect(n(restored, 'SELECT count(*) AS n FROM tasks_tasks')).toBe(2); + expect( + n(restored, "SELECT count(*) AS n FROM _sync_row_meta WHERE tbl = 'tasks_tasks'"), + ).toBe(2); + // The cut itself travels: keys, undo, push, and the folded transactions. + expect(genesisCutOf(restored, STREAM)).toBe(r.cut); + expect(meta(restored, UNDO_ENABLED_KEY)).toBe('1'); + expect(isSyncFlagOn(restored, 'sync.push', {})).toBe(true); + expect(n(restored, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed'")).toBe(0); + expect(sealedPacker(restored, replica), 'a folded transaction was packed again').toBeNull(); + } finally { + restored.close(); + } + }); + + it('a write in this same process during the snapshot waits it out and lands after, never inside the bundle (T13297)', async () => { + const { db } = await store(); + const copy = join(dir, 'bundle.db'); + let writer: Promise | null = null; + let refused = ''; + await cutGenesisWithSnapshot(db, opts(), async () => { + // Same process, same handle: a short wait is refused cleanly ... + vi.stubEnv('CLEO_RESTORE_WAIT_MS', '50'); + await assertExodusWriteSafe(db).catch((e: Error) => { + refused = e.message; + }); + // ... and a patient writer is deferred until the snapshot is done. + vi.stubEnv('CLEO_RESTORE_WAIT_MS', '5000'); + writer = assertExodusWriteSafe(db).then(() => write(db, addTask('LATE'))); + await new Promise((resolve) => setTimeout(resolve, 150)); + const src = new DatabaseSync(dbPath, { readOnly: true }); + try { + src.exec(`VACUUM INTO '${copy}'`); + } finally { + src.close(); + } }); - expect(genesisCutOf(db, STREAM)).toBe(r.cut); + expect(refused).toMatch(/E_STORE_GENESIS/); + await writer; + expect(n(db, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'LATE'")).toBe(1); + const bundle = new DatabaseSync(copy, { readOnly: true }); + try { + expect(n(bundle, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'LATE'")).toBe(0); + } finally { + bundle.close(); + } + // The late write is a journal write: captured after the cut, left live for the push. + expect(n(db, "SELECT count(*) AS n FROM _sync_capture WHERE state = 'live'")).toBeGreaterThan( + 0, + ); + }); + + it('a write that bypasses the chokepoint during the snapshot undoes the cut: no raced bundle is used', async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + await expect( + cutGenesisWithSnapshot(db, opts(), async () => { + write(db, addTask('RACED')); // a raw framed write: no chokepoint + }), + ).rejects.toBeInstanceOf(GenesisRacedError); + expectUncut(db); + // The pre-cut transaction is back to sealed, and the raced write is still there to journal. + expect(n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed'")).toBe(1); + expect(n(db, 'SELECT count(*) AS n FROM _sync_undo')).toBe(0); + expect(n(db, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'RACED'")).toBe(1); expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); }); - it('a failing snapshot rolls the cut back and releases the marker', async () => { + it('a failing snapshot undoes the committed cut and releases the marker', async () => { const { db } = await store(); write(db, addTask('T1')); + seal(db); await expect( cutGenesisWithSnapshot(db, opts(), async () => { throw new Error('upload failed'); @@ -362,7 +427,11 @@ describe('genesis cut with its checkpoint snapshot (S4-1b)', () => { ).rejects.toThrow('upload failed'); expect(db.isTransaction).toBe(false); expectUncut(db); + expect(n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed'")).toBe(1); expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); + // A second attempt cuts cleanly. + const again = await cutGenesisWithSnapshot(db, opts(), async () => {}); + expect(again).toMatchObject({ refused: null, folded: 1 }); }); it('a refused cut never runs the snapshot and releases the marker', async () => { diff --git a/packages/core/src/store/sync/genesis.ts b/packages/core/src/store/sync/genesis.ts index b86bd42d1b..162a8bc169 100644 --- a/packages/core/src/store/sync/genesis.ts +++ b/packages/core/src/store/sync/genesis.ts @@ -41,6 +41,7 @@ import { baselineRowMeta } from './repair.js'; import { activeReplica } from './replica.js'; import { hasTable } from './schema.js'; import { sealPending, sealPreconditions } from './sealer.js'; +import { capturePosition } from './sequencing.js'; import { hasTriggerSuspendTable, verifyOwnedTriggers } from './trigger-classes.js'; import { raiseMinWriterVersion } from './writer-version.js'; @@ -53,6 +54,9 @@ export const GENESIS_SOURCE_SEQ_KEY_PREFIX = 'genesis_source_seq:'; /** `_sync_meta` key prefix set at the cut and cleared once the genesis checkpoint is stored. */ export const GENESIS_PENDING_KEY_PREFIX = 'genesis_pending:'; +/** `_sync_meta` key prefix: the highest `local_seq` a stream's cut folded (so a raced cut can be undone). */ +export const GENESIS_FOLDED_UPTO_KEY_PREFIX = 'genesis_folded_upto:'; + /** `_sync_meta` key the capture triggers' undo `WHEN` checks (§3.5 Rule 2). */ export const UNDO_ENABLED_KEY = 'undo_enabled'; @@ -331,9 +335,17 @@ function closeCut(db: DatabaseSync, opts: GenesisCutOptions, o: OpenCut): Genesi const n = baselineRowMeta(db, opts.scope, table, o.replica, o.at); if (n) baselined[table] = n; } + const upto = Number( + ( + db.prepare("SELECT max(local_seq) AS m FROM _sync_txn WHERE state = 'sealed'").get() as { + m: number | null; + } + ).m ?? 0, + ); const folded = db .prepare("UPDATE _sync_txn SET state = 'folded' WHERE state = 'sealed'") .run().changes; + setMeta(db, `${GENESIS_FOLDED_UPTO_KEY_PREFIX}${opts.stream}`, String(upto), atIso); setMeta(db, `${GENESIS_CUT_KEY_PREFIX}${opts.stream}`, String(o.cut), atIso); setMeta(db, `${GENESIS_SOURCE_SEQ_KEY_PREFIX}${opts.stream}`, String(o.cut), atIso); setMeta(db, `${GENESIS_PENDING_KEY_PREFIX}${opts.stream}`, String(o.cut), atIso); @@ -367,24 +379,73 @@ export function cutGenesis(db: DatabaseSync, opts: GenesisCutOptions): GenesisCu } } +/** A write reached the store between the cut and the end of its snapshot; the cut was undone. */ +export class GenesisRacedError extends Error { + readonly code = 'E_SYNC_GENESIS_RACED'; +} + /** - * {@link cutGenesis} with the genesis checkpoint's bundle snapshotted at the - * cut (§2.11 §10: "snapshotted inside one read transaction that records - * genesis_cut"). `snapshot` runs while the cut's `BEGIN IMMEDIATE` is held, - * so a snapshot taken on another connection sees exactly the store at the - * cut: nothing else can commit, and the cut itself is not committed yet. + * Undo a committed cut whose genesis checkpoint was never pushed (its + * snapshot failed, or a write raced it), in one `BEGIN IMMEDIATE`: drop the + * stream's genesis keys, turn push off, return the transactions it folded to + * `sealed`, and, when no other stream is cut, turn undo off and drop the undo + * written since. Row meta it baselined stays: before the stream starts that + * is what the repair diff would write anyway (T13217). + */ +function uncutGenesis(db: DatabaseSync, opts: GenesisCutOptions): void { + db.exec('BEGIN IMMEDIATE'); + try { + const upto = Number(metaValue(db, `${GENESIS_FOLDED_UPTO_KEY_PREFIX}${opts.stream}`) ?? 0); + db.prepare( + "UPDATE _sync_txn SET state = 'sealed' WHERE state = 'folded' AND local_seq <= ?", + ).run(upto); + const del = db.prepare('DELETE FROM _sync_meta WHERE key = ?'); + for (const prefix of [ + GENESIS_CUT_KEY_PREFIX, + GENESIS_SOURCE_SEQ_KEY_PREFIX, + GENESIS_PENDING_KEY_PREFIX, + GENESIS_FOLDED_UPTO_KEY_PREFIX, + ]) { + del.run(`${prefix}${opts.stream}`); + } + if (!db.prepare(`SELECT 1 FROM _sync_meta WHERE key LIKE '${GENESIS_CUT_KEY_PREFIX}%'`).get()) { + del.run(UNDO_ENABLED_KEY); + db.exec('DELETE FROM _sync_undo'); + if (hasTable(db, '_sync_row_undo')) db.exec('DELETE FROM _sync_row_undo'); + setSyncFlag(db, 'sync.push', false); + } + db.exec('COMMIT'); + } catch (err) { + if (db.isTransaction) db.exec('ROLLBACK'); + throw err; + } +} + +/** + * {@link cutGenesis}, then the genesis checkpoint's bundle snapshotted at + * the committed cut (§2.11 §10; T13296). The cut commits FIRST, so the + * bundle carries everything it wrote: genesis row meta, the genesis keys, + * `undo_enabled`, `sync.push` and the folded transactions. A device restored + * from it therefore has full row meta and never pushes a folded transaction. * - * The export can outlast another writer's busy timeout, so for its whole - * duration the store carries a `genesis` marker ({@link writeRestoreMarker}): - * other cleo processes wait at their store open and write chokepoint, then - * refuse with `E_STORE_GENESIS`, instead of failing with SQLITE_BUSY - * mid-write. A failing snapshot rolls the cut back; the marker is always + * Nothing may write between the cut and the end of the snapshot. For the + * whole run the store carries a `genesis` marker ({@link writeRestoreMarker}): + * other processes wait at their store open, and every writer, in this + * process too, waits at the write chokepoint (`assertExodusWriteSafe`: the + * task accessor's write transactions, session creation, `insertIdempotent`, + * `upsertIdempotent`) and refuses with `E_STORE_GENESIS` if the snapshot + * outlasts its wait (T13297). The snapshot itself writes nothing. A writer + * that bypasses the chokepoint is caught after the snapshot (the capture + * position moved past the cut): the cut is undone and + * {@link GenesisRacedError} is thrown, so no bundle holding a post-cut write + * is ever pushed. A failing snapshot undoes the cut too. The marker is always * released. * * @param db - The store. * @param opts - {@link GenesisCutOptions}, plus the store file the marker guards. - * @param snapshot - Export the store as the checkpoint bundle; receives the cut. + * @param snapshot - Export the store as the checkpoint bundle; receives the cut. Must not write. * @returns What was cut, or why not. + * @throws {GenesisRacedError} When a write reached the store during the snapshot. */ export async function cutGenesisWithSnapshot( db: DatabaseSync, @@ -393,15 +454,23 @@ export async function cutGenesisWithSnapshot( ): Promise { const release = writeRestoreMarker(opts.dbPath, 'genesis'); try { - const open = openCut(db, opts); - if (!('replica' in open)) return open; + const report = cutGenesis(db, opts); + if (report.refused !== null || report.already || report.cut === null) return report; + const cut = report.cut; try { - await snapshot(open.cut); - return closeCut(db, opts, open); + await snapshot(cut); } catch (err) { - if (db.isTransaction) db.exec('ROLLBACK'); + uncutGenesis(db, opts); throw err; } + if (capturePosition(db) > cut) { + uncutGenesis(db, opts); + // @sync-invariant none:local-only a local write raced the genesis snapshot; the cut is undone and nothing is pushed + throw new GenesisRacedError( + 'E_SYNC_GENESIS_RACED: a write reached the store during the genesis snapshot; the cut was undone, run it again', + ); + } + return report; } finally { release(); } diff --git a/scripts/lint-no-raw-table-writes.mjs b/scripts/lint-no-raw-table-writes.mjs index 7652d708ee..f0cd559c4c 100644 --- a/scripts/lint-no-raw-table-writes.mjs +++ b/scripts/lint-no-raw-table-writes.mjs @@ -376,11 +376,24 @@ export const EXEMPT = { _sync_txn: { count: 1, reason: SEALER }, }, 'packages/core/src/store/sync/genesis.ts': { - _sync_meta: { count: 1, reason: SYNC_BOOKKEEPING }, + _sync_meta: { + count: 2, + reason: `${SYNC_BOOKKEEPING}; plus dropping a stream's genesis keys when a failed or raced cut is undone (T13296)`, + }, _sync_txn: { + count: 2, + reason: + 'sealed transactions (local-only): folds every pre-cut transaction into the genesis checkpoint, and returns them to sealed when a failed or raced cut is undone (§2.11 §10, T12343, T13296)', + }, + _sync_row_undo: { + count: 1, + reason: + 'row undo (local-only): dropped when a failed or raced genesis cut is undone and no stream is cut, so undo is off again (T13296)', + }, + _sync_undo: { count: 1, reason: - 'sealed transactions (local-only): folds every pre-cut transaction into the genesis checkpoint (§2.11 §10, T12343)', + 'capture undo (local-only): dropped when a failed or raced genesis cut is undone and no stream is cut, so undo is off again (T13296)', }, }, 'packages/core/src/store/sync/segments.ts': { From bb7989d72bd7ad533b1d35db94c1512340d1bf2d Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 13:14:30 -0700 Subject: [PATCH 04/23] fix(sync): resume a genesis cut that crashed before its snapshot (T13301) With the cut committed before its snapshot (T13296), a crash mid-export left the stream cut and genesis_pending set, and the next run returned `already` without a snapshot, so no genesis checkpoint could ever be made. cutGenesisWithSnapshot now snapshots such a cut when nothing was captured since (resumed), or undoes it and cuts again. A stored cut stays `already`. LOW-1 (review on #1953): undoing a cut now reverts only the transactions that cut folded (genesis_folded_from..genesis_folded_upto), never another stream's earlier fold. Co-Authored-By: Claude Opus 5.5 --- .changeset/t13301-genesis-crash-resume.md | 16 ++++ .../src/store/sync/__tests__/genesis.test.ts | 83 +++++++++++++++++++ packages/core/src/store/sync/genesis.ts | 61 +++++++++++--- 3 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 .changeset/t13301-genesis-crash-resume.md diff --git a/.changeset/t13301-genesis-crash-resume.md b/.changeset/t13301-genesis-crash-resume.md new file mode 100644 index 0000000000..e6d054b618 --- /dev/null +++ b/.changeset/t13301-genesis-crash-resume.md @@ -0,0 +1,16 @@ +--- +id: t13301-genesis-crash-resume +tasks: [T13301] +kind: fix +summary: A genesis cut that crashed before its snapshot resumes on the next run; undoing a cut un-folds only its own transactions +--- + +The genesis cut commits before its snapshot (T13296), so a crash, kill or sleep mid-export used to leave the stream cut and +`genesis_pending` set with no snapshot, and the next run returned `already` without ever producing the checkpoint. + +- **`cutGenesisWithSnapshot` now resumes such a cut.** + - With no capture since the crash, the store is still exactly the cut, and the snapshot runs at it (`resumed: true`). + - Otherwise the stale cut is undone and the store is cut again. + - A stored cut (`genesis_pending` cleared) is still `already`, and no snapshot runs. +- **Undoing a cut reverts only the transactions that cut folded** (`genesis_folded_from` to `genesis_folded_upto`). A store feeding + two streams no longer un-folds the other stream's earlier fold. diff --git a/packages/core/src/store/sync/__tests__/genesis.test.ts b/packages/core/src/store/sync/__tests__/genesis.test.ts index c909f07607..7daac48f44 100644 --- a/packages/core/src/store/sync/__tests__/genesis.test.ts +++ b/packages/core/src/store/sync/__tests__/genesis.test.ts @@ -446,3 +446,86 @@ describe('genesis cut with its checkpoint snapshot (S4-1b; T13296, T13297)', () expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); }); }); + +describe('a cut that crashed before its snapshot (T13301), and per-cut folds (LOW-1 on #1953)', () => { + const opts = (stream = STREAM) => ({ + scope: 'project' as const, + stream, + dbPath, + now: () => ++clock, + env: {}, + allowUnreleased: true, + }); + const STREAM_B = 'home:0192eeee-7f00-7000-8000-00000000000e'; + + it('a committed cut with no snapshot (a crash) is snapshotted at that cut on the next run', async () => { + const { db } = await store(); + write(db, addTask('T1')); + const crashed = cutGenesis(db, opts()); // the run died after COMMIT, before its snapshot + expect(genesisPending(db, STREAM)).toBe(true); + let at: number | null = null; + const r = await cutGenesisWithSnapshot(db, opts(), async (cut) => { + at = cut; + }); + expect(r).toMatchObject({ refused: null, already: false, resumed: true, cut: crashed.cut }); + expect(at).toBe(crashed.cut); + expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); + }); + + it('written to since the crash: the stale cut is undone and the store cut again', async () => { + const { db } = await store(); + write(db, addTask('T1')); + const crashed = cutGenesis(db, opts()); + write(db, addTask('T2')); // a journal write after the crashed cut + let at: number | null = null; + const r = await cutGenesisWithSnapshot(db, opts(), async (cut) => { + at = cut; + }); + expect(r.resumed).toBe(false); + expect(r.cut).toBeGreaterThan(crashed.cut ?? 0); + expect(at).toBe(r.cut); + expect(genesisCutOf(db, STREAM)).toBe(r.cut); + // The post-crash write was drained into the new cut: nothing live, nothing sealed. + expect(n(db, "SELECT count(*) AS n FROM _sync_capture WHERE state = 'live'")).toBe(0); + expect(n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed'")).toBe(0); + }); + + it('a stored cut (pending cleared) is already: no snapshot runs', async () => { + const { db } = await store(); + const first = cutGenesis(db, opts()); + db.prepare('DELETE FROM _sync_meta WHERE key = ?').run( + `${GENESIS_PENDING_KEY_PREFIX}${STREAM}`, + ); + let ran = false; + const r = await cutGenesisWithSnapshot(db, opts(), async () => { + ran = true; + }); + expect(r).toMatchObject({ already: true, cut: first.cut }); + expect(ran).toBe(false); + }); + + it("undoing one stream's failed cut never un-folds another stream's", async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + await cutGenesisWithSnapshot(db, opts(), async () => {}); // stream A, stored below + db.prepare('DELETE FROM _sync_meta WHERE key = ?').run( + `${GENESIS_PENDING_KEY_PREFIX}${STREAM}`, + ); + const aFolded = n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'folded'"); + expect(aFolded).toBe(1); + write(db, addTask('T2')); + seal(db); + await expect( + cutGenesisWithSnapshot(db, opts(STREAM_B), async () => { + throw new Error('export failed'); + }), + ).rejects.toThrow('export failed'); + // A's fold stands; only B's (T2's txn) is sealed again, and A keeps undo on. + expect(n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'folded'")).toBe(aFolded); + expect(n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed'")).toBe(1); + expect(genesisCutOf(db, STREAM_B)).toBeUndefined(); + expect(meta(db, UNDO_ENABLED_KEY)).toBe('1'); + expect(isSyncFlagOn(db, 'sync.push', {})).toBe(true); + }); +}); diff --git a/packages/core/src/store/sync/genesis.ts b/packages/core/src/store/sync/genesis.ts index 162a8bc169..3a149efbec 100644 --- a/packages/core/src/store/sync/genesis.ts +++ b/packages/core/src/store/sync/genesis.ts @@ -57,6 +57,9 @@ export const GENESIS_PENDING_KEY_PREFIX = 'genesis_pending:'; /** `_sync_meta` key prefix: the highest `local_seq` a stream's cut folded (so a raced cut can be undone). */ export const GENESIS_FOLDED_UPTO_KEY_PREFIX = 'genesis_folded_upto:'; +/** `_sync_meta` key prefix: the lowest `local_seq` a stream's cut folded (its range, never another cut's). */ +export const GENESIS_FOLDED_FROM_KEY_PREFIX = 'genesis_folded_from:'; + /** `_sync_meta` key the capture triggers' undo `WHEN` checks (§3.5 Rule 2). */ export const UNDO_ENABLED_KEY = 'undo_enabled'; @@ -91,12 +94,18 @@ export interface GenesisCutReport { readonly folded: number; /** Rows given genesis row meta, per table (tables with none are absent). */ readonly baselined: Readonly>; + /** + * A cut committed by an earlier run that never finished its snapshot (it + * crashed) was snapshotted now, at that same cut (T13301). + */ + readonly resumed: boolean; } const report = (stream: string, fields: Partial): GenesisCutReport => ({ stream, refused: null, already: false, + resumed: false, cut: null, sealed: 0, folded: 0, @@ -335,16 +344,18 @@ function closeCut(db: DatabaseSync, opts: GenesisCutOptions, o: OpenCut): Genesi const n = baselineRowMeta(db, opts.scope, table, o.replica, o.at); if (n) baselined[table] = n; } - const upto = Number( - ( - db.prepare("SELECT max(local_seq) AS m FROM _sync_txn WHERE state = 'sealed'").get() as { - m: number | null; - } - ).m ?? 0, - ); + const range = db + .prepare( + "SELECT min(local_seq) AS lo, max(local_seq) AS hi FROM _sync_txn WHERE state = 'sealed'", + ) + .get() as { lo: number | null; hi: number | null }; + // An empty fold records an empty range (from > upto). + const upto = Number(range.hi ?? 0); + const from = Number(range.lo ?? upto + 1); const folded = db .prepare("UPDATE _sync_txn SET state = 'folded' WHERE state = 'sealed'") .run().changes; + setMeta(db, `${GENESIS_FOLDED_FROM_KEY_PREFIX}${opts.stream}`, String(from), atIso); setMeta(db, `${GENESIS_FOLDED_UPTO_KEY_PREFIX}${opts.stream}`, String(upto), atIso); setMeta(db, `${GENESIS_CUT_KEY_PREFIX}${opts.stream}`, String(o.cut), atIso); setMeta(db, `${GENESIS_SOURCE_SEQ_KEY_PREFIX}${opts.stream}`, String(o.cut), atIso); @@ -395,16 +406,21 @@ export class GenesisRacedError extends Error { function uncutGenesis(db: DatabaseSync, opts: GenesisCutOptions): void { db.exec('BEGIN IMMEDIATE'); try { + // Only this cut's fold: another stream's earlier cut keeps its own (LOW-1 on #1953). const upto = Number(metaValue(db, `${GENESIS_FOLDED_UPTO_KEY_PREFIX}${opts.stream}`) ?? 0); + const from = Number( + metaValue(db, `${GENESIS_FOLDED_FROM_KEY_PREFIX}${opts.stream}`) ?? upto + 1, + ); db.prepare( - "UPDATE _sync_txn SET state = 'sealed' WHERE state = 'folded' AND local_seq <= ?", - ).run(upto); + "UPDATE _sync_txn SET state = 'sealed' WHERE state = 'folded' AND local_seq BETWEEN ? AND ?", + ).run(from, upto); const del = db.prepare('DELETE FROM _sync_meta WHERE key = ?'); for (const prefix of [ GENESIS_CUT_KEY_PREFIX, GENESIS_SOURCE_SEQ_KEY_PREFIX, GENESIS_PENDING_KEY_PREFIX, GENESIS_FOLDED_UPTO_KEY_PREFIX, + GENESIS_FOLDED_FROM_KEY_PREFIX, ]) { del.run(`${prefix}${opts.stream}`); } @@ -441,6 +457,12 @@ function uncutGenesis(db: DatabaseSync, opts: GenesisCutOptions): void { * is ever pushed. A failing snapshot undoes the cut too. The marker is always * released. * + * A cut that an earlier run committed but never snapshotted (a crash, kill + * or sleep mid-export; `genesis_pending` still set) is resumed (T13301): with + * no capture since, the snapshot runs at that cut (`resumed`); otherwise the + * stale cut is undone and the store is cut again. A stored cut + * (`genesis_pending` cleared) is `already`, and no snapshot runs. + * * @param db - The store. * @param opts - {@link GenesisCutOptions}, plus the store file the marker guards. * @param snapshot - Export the store as the checkpoint bundle; receives the cut. Must not write. @@ -454,9 +476,22 @@ export async function cutGenesisWithSnapshot( ): Promise { const release = writeRestoreMarker(opts.dbPath, 'genesis'); try { - const report = cutGenesis(db, opts); - if (report.refused !== null || report.already || report.cut === null) return report; - const cut = report.cut; + let r: GenesisCutReport; + const crashed = genesisPending(db, opts.stream) ? genesisCutOf(db, opts.stream) : undefined; + if (crashed !== undefined && capturePosition(db) === crashed) { + // T13301: an earlier run committed this cut and died before its + // snapshot finished; nothing was captured since, so the store is still + // exactly the cut. Snapshot it now. + r = report(opts.stream, { cut: crashed, resumed: true }); + } else { + // Written to since that crash: the cut no longer describes the store. + // Undo it and cut again. + if (crashed !== undefined) uncutGenesis(db, opts); + r = cutGenesis(db, opts); + if (r.refused !== null || r.already || r.cut === null) return r; + } + const cut = r.cut; + if (cut === null) return r; try { await snapshot(cut); } catch (err) { @@ -470,7 +505,7 @@ export async function cutGenesisWithSnapshot( 'E_SYNC_GENESIS_RACED: a write reached the store during the genesis snapshot; the cut was undone, run it again', ); } - return report; + return r; } finally { release(); } From 87f93021f1332396c747f104f1ff8d004cc80e1f Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 12:45:16 -0700 Subject: [PATCH 05/23] feat(sync): cleo sync enable push - genesis cut and checkpoint/v3 genesis push (T13300) enableSyncPush (core) / `cleo sync enable push [--scope]` (CLI, refused while sync.push is unreleased) checks that the store's replica is the link's, that this machine synced the head, and that no other device started the stream's journal; then, under the write lease, commits the genesis cut and exports the store through the vault's bundle export (cutGenesisWithSnapshot), saves the bundle beside the store, and pushes a checkpoint/v3 genesis (empty accounting, the store's replay pin), parented on the current head with a count-delta segment when needed. completeGenesis then raises the replicaSeq high-water mark to the replica's last segment on the stream and clears genesis_pending. Resume: a failed checkpoint push keeps the bundle and pushes it next run; a cut that crashed before its bundle was saved is resumed or re-cut (T13301); a genesis the server stored before the process died is adopted by the checkpoint id recorded before each upload (Journal.pushCheckpoint takes a caller-minted id), never cut twice (T13302). cleo cloud push refuses a store past its cut. The vault's replay/delta/checkpoint loop is shared as appendCheckpoint. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-genesis-checkpoint.md | 38 ++ packages/cleo/src/cli/commands/sync.ts | 29 + .../src/cli/generated/command-manifest.ts | 2 +- .../cli/lib/__tests__/nexus-vault-cli.test.ts | 20 + packages/cleo/src/cli/lib/nexus-vault-cli.ts | 19 + packages/contracts/src/index.ts | 1 + packages/contracts/src/nexus-account.ts | 5 + packages/contracts/src/nexus-vault.ts | 25 + .../src/cloud/__tests__/nexus-vault.test.ts | 256 ++++++- packages/core/src/cloud/journal.ts | 8 +- packages/core/src/cloud/nexus-vault.ts | 637 ++++++++++++++---- packages/core/src/store/sync/genesis.ts | 56 +- scripts/lint-no-raw-table-writes.mjs | 4 +- 13 files changed, 977 insertions(+), 123 deletions(-) create mode 100644 .changeset/t12343-genesis-checkpoint.md diff --git a/.changeset/t12343-genesis-checkpoint.md b/.changeset/t12343-genesis-checkpoint.md new file mode 100644 index 0000000000..452c80c923 --- /dev/null +++ b/.changeset/t12343-genesis-checkpoint.md @@ -0,0 +1,38 @@ +--- +id: t12343-genesis-checkpoint +tasks: [T12343, T13300, T13302] +kind: feat +summary: "`cleo sync enable push` - genesis cut plus genesis checkpoint (checkpoint/v3), behind the unreleased sync.push flag" +--- + +`cleo sync enable push [--scope project|global]` starts a store's change-journal push (journal spec §2.11 §10; T12343 S4-1b). It is +refused while `sync.push` is unreleased; the core call `enableSyncPush` takes an `allowUnreleased` opt-in for tests and staging +only. + +- **What it checks first:** + - the store's bound replica must be the one the stream knows (the link); + - this machine must have synced the stream's head (pull first); + - a stream whose head is already a journal checkpoint is refused with `E_NEXUS_SYNC_STREAM_JOURNALED`, so this store joins it by + pulling instead of writing a second genesis. +- **Under the write lease:** + - It commits the genesis cut, then exports the store as the cut left it through the vault's bundle export + (`cutGenesisWithSnapshot`; the genesis marker holds every other writer off meanwhile). A failed export, or a write that raced + it, undoes the cut, and nothing is saved or pushed. + - It saves the bundle beside the store, then pushes a **checkpoint/v3** genesis: the vault manifest with empty + `pending`/`voided`/`revived`/`pruned` and the store's replay pin. The parent is the current head (null only on an empty + stream), and a count-delta segment is appended when the rows differ from the parent plus the window. + - A window that crosses a sync-schema rise is refused (its pin would need that rise's journal hash). +- **After the checkpoint is stored:** + - `completeGenesis` raises the store's replicaSeq high-water mark to the replica's last segment on the stream (the vault's delta + segments spend this replica's seqs), and clears `genesis_pending`. + - The saved bundle is removed. + - A push that fails after the cut keeps the bundle and resumes on the next run (`status: 'resumed'`), without a second cut. A + cut that crashed before its bundle was saved is resumed at that cut, or re-cut if the store moved on (T13301). +- **A genesis the server already stored is adopted, never cut twice (T13302).** `genesis_pending` means "not confirmed stored". If + a run died after the upload was accepted, the next run finds that checkpoint on the stream by the id recorded beside the bundle + before each upload (`Journal.pushCheckpoint` now takes a caller-minted `checkpointId`). It verifies the checkpoint, records it + (high-water mark, pending cleared), and pushes no second genesis, even after a local write. +- **`cleo cloud push` refuses a store past its genesis cut.** Its changes travel as journal segments; a vault delta would count them + twice. +- **New error codes:** `E_NEXUS_SYNC_REFUSED`, `E_NEXUS_SYNC_STREAM_JOURNALED`. +- **Refactor:** the vault's replay, delta and checkpoint loop is now `appendCheckpoint`, shared by `cloud push` and the genesis push. diff --git a/packages/cleo/src/cli/commands/sync.ts b/packages/cleo/src/cli/commands/sync.ts index 6c634c2b5c..cbc5489b03 100644 --- a/packages/cleo/src/cli/commands/sync.ts +++ b/packages/cleo/src/cli/commands/sync.ts @@ -11,6 +11,7 @@ * cleo sync links remove — remove all links for a provider * cleo sync reconcile --provider [--conflict-policy ] * — reconcile external tasks from a JSON file + * cleo sync enable push [--scope] — genesis cut + genesis checkpoint (T12343) * * @task T473 * @task T483 @@ -134,6 +135,33 @@ const reconcileCommand = defineCommand({ }, }); +/** cleo sync enable push — start the change journal's push for this store (T12343 S4-1b) */ +const enableCommand = defineCommand({ + meta: { + name: 'enable', + description: + "Turn on a change-journal flag. 'push': record this store's genesis cut and push its genesis checkpoint to Cleo Nexus; from then on its changes travel as journal segments, not vault snapshots. The store is snapshotted under its write lock: while the bundle exports (seconds; longer on a large store), other cleo processes on this store pause their writes, wait, and refuse with E_STORE_GENESIS if it outlasts their wait (CLEO_RESTORE_WAIT_MS, default 15 s); run them again afterwards. A push that fails after the cut resumes on the next run. Refused while sync.push is unreleased.", + }, + args: { + flag: { type: 'positional', description: "The flag to enable: 'push'", required: true }, + scope: { type: 'string', description: "Which store: 'project' (default) or 'global'" }, + 'api-url': { type: 'string', description: 'Cleo Nexus API URL (default: the configured one)' }, + json: { type: 'boolean', description: 'Output as JSON envelope' }, + }, + async run({ args }) { + if (args.flag !== 'push') { + cliError( + `unknown sync flag '${String(args.flag)}': only 'push' can be enabled here`, + ExitCode.INVALID_INPUT, + { name: 'E_VALIDATION' }, + ); + process.exit(ExitCode.INVALID_INPUT); + } + const { runSyncEnablePush } = await import('../lib/nexus-vault-cli.js'); + await runSyncEnablePush(args as Record); + }, +}); + /** * Root sync command group — registers all sync subcommands. * @@ -144,6 +172,7 @@ export const syncCommand = defineCommand({ subCommands: { links: linksCommand, reconcile: reconcileCommand, + enable: enableCommand, }, async run({ cmd, rawArgs }) { const firstArg = rawArgs?.find((a) => !a.startsWith('-')); diff --git a/packages/cleo/src/cli/generated/command-manifest.ts b/packages/cleo/src/cli/generated/command-manifest.ts index 8181747181..3ff6ba8c98 100644 --- a/packages/cleo/src/cli/generated/command-manifest.ts +++ b/packages/cleo/src/cli/generated/command-manifest.ts @@ -219,7 +219,7 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [ exportName: 'cloudCommand', name: 'cloud', description: - 'Cleo Nexus with the device credential: status (one-call verification), whoami, devices, projects [show], activity; the encrypted vault: push, pull, restore, verify, vault, lease release.', + 'Cleo Nexus with the device credential: status (one-call verification), whoami, devices, projects [show], activity; the encrypted vault: push, pull, restore, verify, vault, lease release; the local sync conflict log: conflicts [resolve ].', load: async () => (await import('../commands/cloud.js')).cloudCommand as CommandDef, }, { diff --git a/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts b/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts index 6f3e0d92f6..a196417dc1 100644 --- a/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts +++ b/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts @@ -12,6 +12,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const pushNexusVault = vi.fn(); +const enableSyncPush = vi.fn(); const restoreNexusVault = vi.fn(); const verifyNexusVault = vi.fn(); const nexusVaultStatus = vi.fn(); @@ -22,6 +23,7 @@ const resolveNexusProjectRef = vi.fn(); const assertNexusRestoreTarget = vi.fn(); vi.mock('@cleocode/core/cloud/nexus-vault.js', () => ({ + enableSyncPush, pushNexusVault, restoreNexusVault, verifyNexusVault, @@ -46,6 +48,7 @@ const { runCloudRestore, runCloudVault, runCloudVerify, + runSyncEnablePush, } = await import('../nexus-vault-cli.js'); const API = 'https://api.nexus.test'; @@ -157,6 +160,23 @@ describe('flags reach the core calls', () => { expect(written()).toContain('cp-1'); }); + it('sync enable push passes scope and the API URL, never the unreleased opt-in (T12343)', async () => { + enableSyncPush.mockReset().mockResolvedValue({ + ...base, + status: 'enabled', + cut: 7, + sealed: 1, + folded: 2, + baselined: {}, + snapshot, + deltaSegmentSeq: null, + replicaSeqFloor: null, + }); + await runSyncEnablePush({ 'api-url': API, scope: 'global' }); + expect(opts(enableSyncPush)).toEqual({ apiUrl: API, scope: 'global' }); + expect(written()).toContain('cp-1'); + }); + it('restore passes checkpoint, project, into and force', async () => { await runCloudRestore({ scope: 'project', diff --git a/packages/cleo/src/cli/lib/nexus-vault-cli.ts b/packages/cleo/src/cli/lib/nexus-vault-cli.ts index 78dbd49025..b2c25cb40f 100644 --- a/packages/cleo/src/cli/lib/nexus-vault-cli.ts +++ b/packages/cleo/src/cli/lib/nexus-vault-cli.ts @@ -17,6 +17,7 @@ import type { CloudLeaseReleaseResult, CloudPushResult, CloudRestoreResult, + CloudSyncPushEnableResult, CloudVaultScope, CloudVaultStatusResult, CloudVerifyResult, @@ -87,6 +88,24 @@ export async function runCloudPush(args: Args): Promise { ); } +/** + * `cleo sync enable push [--scope]`: record the store's genesis cut and push + * its genesis checkpoint (T12343 S4-1b). Refused while `sync.push` is + * unreleased: this CLI never opts in. + * + * @param args - Parsed args. + */ +export async function runSyncEnablePush(args: Args): Promise { + await runCloudRead( + 'sync.enable.push', + async () => (await vaultModule()).enableSyncPush(common(args, 'sync.enable.push')), + (r) => + r.status === 'already' + ? `Push is already on for ${r.streamId} (genesis cut at capture ${r.cut}).` + : `Push is on for ${r.streamId}: genesis cut at capture ${r.cut}, checkpoint ${r.snapshot?.checkpointId}${r.status === 'resumed' ? ' (resumed)' : ''}.`, + ); +} + /** * `cleo cloud pull [--scope] [--force]`. * diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 727ffdc7c6..2518c8c505 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -3136,6 +3136,7 @@ export { type CloudLeaseReleaseResult, type CloudPushResult, type CloudRestoreResult, + type CloudSyncPushEnableResult, type CloudVaultLease, type CloudVaultScope, type CloudVaultSnapshot, diff --git a/packages/contracts/src/nexus-account.ts b/packages/contracts/src/nexus-account.ts index a567948fe9..f5ef38f2e8 100644 --- a/packages/contracts/src/nexus-account.ts +++ b/packages/contracts/src/nexus-account.ts @@ -101,6 +101,11 @@ export const NEXUS_ACCOUNT_ERROR_CODES = [ 'E_NEXUS_VAULT_STORE_BUSY', /** A project restore target already holds a different project. */ 'E_NEXUS_VAULT_TARGET_OCCUPIED', + // Change-journal push (`cleo sync enable push`, T12343 S4-1b). + /** The store is not fit to reach genesis (a precondition failed); nothing was cut. */ + 'E_NEXUS_SYNC_REFUSED', + /** Another device already started this stream's change journal: this store joins it by pulling. */ + 'E_NEXUS_SYNC_STREAM_JOURNALED', // Projects by name (`cleo cloud restore `, T13102). /** No project of the account has that name, label or id. */ 'E_NEXUS_PROJECT_NOT_FOUND', diff --git a/packages/contracts/src/nexus-vault.ts b/packages/contracts/src/nexus-vault.ts index 10322ba43c..cf2ab80b25 100644 --- a/packages/contracts/src/nexus-vault.ts +++ b/packages/contracts/src/nexus-vault.ts @@ -179,6 +179,31 @@ export interface CloudPushResult { warnings: CloudWarning[]; } +/** `cleo sync enable push`: the stream's genesis cut and its genesis checkpoint (T12343 S4-1b). */ +export interface CloudSyncPushEnableResult { + apiUrl: string; + scope: CloudVaultScope; + streamId: string; + /** + * `enabled`: the cut was recorded and its genesis checkpoint stored; `resumed`: a cut recorded + * by an earlier run whose checkpoint push failed was pushed now; `already`: push was already on. + */ + status: 'enabled' | 'resumed' | 'already'; + /** The genesis cut: the highest capture seq the checkpoint carries. */ + cut: number; + /** Transactions sealed before the cut, folded into the checkpoint, and rows given genesis row meta. */ + sealed: number; + folded: number; + baselined: Record; + /** The genesis checkpoint (null when `already`). */ + snapshot: CloudVaultSnapshot | null; + /** The delta segment appended so the counts reconcile with the parent, if one was needed. */ + deltaSegmentSeq: number | null; + /** The replicaSeq this replica's first journal segment follows (null: it starts at 0). */ + replicaSeqFloor: number | null; + warnings: CloudWarning[]; +} + /** `cleo cloud pull` / `cleo cloud restore`. */ export interface CloudRestoreResult { apiUrl: string; diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 7502b9777b..be9a5163c5 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -50,10 +50,24 @@ import { import { drizzle } from 'drizzle-orm/node-sqlite'; import { create as tarCreate, extract as tarExtract } from 'tar'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { _resetDualScopeDbCache, openDualScopeDb } from '../../store/dual-scope-db.js'; +import { + _resetDualScopeDbCache, + getDualScopeNativeDb, + openDualScopeDb, + openDualScopeDbAtPath, +} from '../../store/dual-scope-db.js'; import { runBracketedMigrations } from '../../store/migration-runner.js'; import { computeManifestHash, exportPortableBundle } from '../../store/portable-bundle.js'; import { resolveCorePackageMigrationsFolder } from '../../store/resolve-migrations-folder.js'; +import { + ROW_IDENTITY_META_TABLE, + ROW_IDENTITY_RECIPE, + ROW_IDENTITY_RECIPE_KEY, +} from '../../store/row-identity.js'; +import { setCaptureEnabled } from '../../store/sync/capture.js'; +import { isSyncFlagOn, setSyncFlag } from '../../store/sync/flags.js'; +import { cutGenesis, genesisCutOf, genesisPending } from '../../store/sync/genesis.js'; +import { ensureProjectReplica, storeHwm } from '../../store/sync/replica.js'; import { ensureSyncSchema } from '../../store/sync/schema.js'; import { emptyVaultTableHash, @@ -104,6 +118,7 @@ import { hasUnsyncedNexusBackup, runNexusFirstRun } from '../nexus-first-run.js' import { linkProjectToNexus } from '../nexus-link.js'; import { listNexusNamedProjects, resolveNexusProjectRef } from '../nexus-project-names.js'; import { + enableSyncPush, nexusVaultStatus, pushNexusVault, releaseNexusVaultLease, @@ -333,6 +348,8 @@ class FakeNexus { refuseCheckpoint: string[] = []; /** Runs once before the next checkpoint create by `deviceId` (simulates a concurrent author). */ beforeCheckpoint: { deviceId: string; run: () => Promise } | null = null; + /** Runs once after the server stored a checkpoint, before the response reaches the client. */ + afterCheckpoint: { deviceId: string; run: () => Promise } | null = null; /** The server's MAX_SCHEMA_VERSION. */ maxSchemaVersion = 10_000; /** Whether the stream head carries `maxSchemaVersion` (a server from before it does not). */ @@ -429,7 +446,19 @@ class FakeNexus { this.beforeCheckpoint = null; await cpHook.run(); } - return this.route(device, method, url, body); + const response = await this.route(device, method, url, body); + const afterHook = this.afterCheckpoint; + if ( + afterHook !== null && + afterHook.deviceId === device.deviceId && + method === 'POST' && + url.pathname.endsWith('/checkpoints') && + response.ok + ) { + this.afterCheckpoint = null; + await afterHook.run(); + } + return response; } catch (err) { if (err instanceof ApiFail) { return json(err.status, { @@ -4818,3 +4847,226 @@ describe('cloud project link stores a new project key with its registration (onb expect(fake.calls.filter((c) => c === 'POST /v1/projects')).toEqual([]); }); }); + +describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { + const SYNC_JOURNAL = path.resolve(import.meta.dirname, '../../../migrations/sync-journal'); + + /** A machine whose project store is a real cleo.db with capture and seal on, bound and linked. */ + async function journalMachine(): Promise<{ m: Machine; dbPath: string }> { + const m = await machine('a', DEVICE_A, REPLICA_A); + const cleo = path.join(m.root, '.cleo'); + fs.mkdirSync(cleo, { recursive: true }); + fs.writeFileSync(path.join(cleo, 'project-id'), `${LOCAL_PROJECT}\n`); + fs.writeFileSync( + path.join(cleo, 'project-info.json'), + JSON.stringify({ projectId: LOCAL_PROJECT, name: 'demo' }), + ); + const dbPath = path.join(cleo, 'cleo.db'); + const replicaId = await on(m, async () => { + const db = getDualScopeNativeDb(await openDualScopeDbAtPath('project', dbPath)); + for (const id of ['T1', 'T2']) { + db.exec( + `INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('${id}', 'title ${id}', 'task', 'pending', 'medium', 'uid-${id}', 'fp-${id}')`, + ); + } + db.prepare( + `INSERT INTO ${ROW_IDENTITY_META_TABLE} (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, + ).run(ROW_IDENTITY_RECIPE_KEY, ROW_IDENTITY_RECIPE); + setCaptureEnabled(db, 'project', true, { schemaRoot: SYNC_JOURNAL }); + setSyncFlag(db, 'sync.seal', true, { schemaRoot: SYNC_JOURNAL, allowUnreleased: true }); + return ensureProjectReplica(db, { dbPath, mode: 'live' }).replicaId; + }); + m.replicaId = replicaId; + fake.addProject(REMOTE_PROJECT, { [replicaId]: DEVICE_A }); + link(m); + return { m, dbPath }; + } + + const storeOf = async (dbPath: string) => + getDualScopeNativeDb(await openDualScopeDbAtPath('project', dbPath)); + + it('cuts the store and pushes a checkpoint/v3 genesis the server accepts; push turns on and the vault steps aside', async () => { + const { m, dbPath } = await journalMachine(); + const r = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + expect(r.status).toBe('enabled'); + expect(r.baselined).toEqual({ tasks_tasks: 2 }); + const s = fake.stream(STREAM); + expect(s.checkpoints).toHaveLength(1); + const cp = s.checkpoints[0]; + expect(cp?.parentCheckpointId).toBeNull(); + expect(manifestVersion(cp?.manifest ?? { schemaVersion: 1, tables: {} })).toBe(3); + expect(cp?.manifest).toMatchObject({ pending: [], voided: [], revived: [], pruned: {} }); + expect(cp?.manifest.replayPin?.transitions).toEqual([]); + expect(cp?.manifest.tables['tasks_tasks']?.rows).toBe(2); + expect(r.snapshot?.checkpointId).toBe(cp?.checkpointId); + expect(fake.leases.size).toBe(0); + await on(m, async () => { + const db = await storeOf(dbPath); + expect(genesisCutOf(db, STREAM)).toBe(r.cut); + expect(genesisPending(db, STREAM)).toBe(false); + expect(isSyncFlagOn(db, 'sync.push', {})).toBe(true); + }); + // Nothing waits beside the store. + expect(fs.readdirSync(path.join(m.root, '.cleo', 'sync-genesis'))).toEqual([]); + // A second run changes nothing; the vault no longer pushes this store. + expect((await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true })))).status).toBe( + 'already', + ); + const vault = await failure(on(m, () => pushNexusVault(vopts(m)))); + expect(vault.code).toBe('E_NEXUS_VAULT_STREAM_UPGRADED'); + expect(s.checkpoints).toHaveLength(1); + // A store that never cut this stream (its cut forgotten) meets a journal head: it joins by + // pulling, never with a second genesis. + await on(m, async () => { + (await storeOf(dbPath)).exec("DELETE FROM _sync_meta WHERE key LIKE 'genesis_%'"); + }); + const second = await failure(on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true })))); + expect(second.code).toBe('E_NEXUS_SYNC_STREAM_JOURNALED'); + expect(s.checkpoints).toHaveLength(1); + }); + + it('over a vault head, genesis parents it, and the first journal segment follows the vault delta', async () => { + const { m, dbPath } = await journalMachine(); + await on(m, () => pushNexusVault(vopts(m))); + // A captured write the vault pushes as a delta segment (this replica's replicaSeq 0). + await on(m, async () => { + const db = await storeOf(dbPath); + db.exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T3', 'title T3', 'task', 'pending', 'medium', 'uid-T3', 'fp-T3')", + ); + }); + const second = await on(m, () => pushNexusVault(vopts(m))); + expect(second.deltaSegmentSeq).toBe(1); + const head = fake.stream(STREAM).checkpoints.at(-1); + const r = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + expect(r.status).toBe('enabled'); + const cp = fake.stream(STREAM).checkpoints.at(-1); + expect(cp?.parentCheckpointId).toBe(head?.checkpointId); + expect(manifestVersion(cp?.manifest ?? { schemaVersion: 1, tables: {} })).toBe(3); + // The replica's last segment on the stream: the vault's delta (0), or the genesis push's own (1). + const floor = r.deltaSegmentSeq === null ? 0 : 1; + expect(r.replicaSeqFloor).toBe(floor); + await on(m, async () => { + expect(storeHwm(await storeOf(dbPath), m.replicaId)).toEqual({ [STREAM]: floor }); + }); + }); + + it('a checkpoint push that fails after the cut keeps the bundle and resumes without a second cut', async () => { + const { m, dbPath } = await journalMachine(); + fake.beforeCheckpoint = { + deviceId: DEVICE_A, + run: async () => { + throw new ApiFail(403, 'E_FORBIDDEN'); + }, + }; + await expect( + on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))), + ).rejects.toThrow(); + expect(fake.leases.size).toBe(0); + const cut = await on(m, async () => { + const db = await storeOf(dbPath); + expect(genesisPending(db, STREAM)).toBe(true); + return genesisCutOf(db, STREAM); + }); + expect(cut).toBeDefined(); + expect(fs.readdirSync(path.join(m.root, '.cleo', 'sync-genesis')).sort()).toHaveLength(2); + // Cut but not yet checkpointed: the vault must not push this store's changes as a delta either. + const vault = await failure(on(m, () => pushNexusVault(vopts(m)))); + expect(vault.code).toBe('E_NEXUS_VAULT_STREAM_UPGRADED'); + expect(fake.stream(STREAM).checkpoints).toHaveLength(0); + // A write after the cut: it is not in the saved bundle, and stays for the journal. + await on(m, async () => { + const db = await storeOf(dbPath); + db.exec("UPDATE tasks_tasks SET title = 'after the cut' WHERE id = 'T1'"); + }); + const r = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + expect(r.status).toBe('resumed'); + expect(r.cut).toBe(cut); + await on(m, async () => { + const db = await storeOf(dbPath); + expect(genesisCutOf(db, STREAM)).toBe(cut); + expect(genesisPending(db, STREAM)).toBe(false); + expect( + ( + db.prepare("SELECT count(*) AS n FROM _sync_capture WHERE state = 'live'").get() as { + n: number; + } + ).n, + ).toBeGreaterThan(0); + }); + }); + + it('a cut that crashed before its bundle was saved is snapshotted at that cut and pushed (T13301)', async () => { + const { m, dbPath } = await journalMachine(); + const crashed = await on(m, async () => + cutGenesis(await storeOf(dbPath), { + scope: 'project', + stream: STREAM, + env: {}, + allowUnreleased: true, + }), + ); + expect(crashed.cut).not.toBeNull(); + const r = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + expect(r).toMatchObject({ status: 'resumed', cut: crashed.cut }); + expect(fake.stream(STREAM).checkpoints).toHaveLength(1); + await on(m, async () => { + expect(genesisPending(await storeOf(dbPath), STREAM)).toBe(false); + }); + }); + + it('a genesis the server stored before the process died is adopted on the next run, never cut twice (T13302)', async () => { + const { m, dbPath } = await journalMachine(); + // The process dies after the server stored the checkpoint, before it cleared genesis_pending: + // the bundle and its record (with the uploaded checkpoint id) stay beside the store. + const dir = path.join(m.root, '.cleo', 'sync-genesis'); + let keptFiles: Array<{ name: string; bytes: Buffer }> = []; + fake.afterCheckpoint = { + deviceId: DEVICE_A, + run: async () => { + keptFiles = fs + .readdirSync(dir) + .map((name) => ({ name, bytes: fs.readFileSync(path.join(dir, name)) })); + }, + }; + const first = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + for (const f of keptFiles) fs.writeFileSync(path.join(dir, f.name), f.bytes); + expect(keptFiles).toHaveLength(2); + // Then a local write, and genesis_pending still set. + await on(m, async () => { + const db = await storeOf(dbPath); + db.prepare( + "INSERT INTO _sync_meta (key, value, updated_at) VALUES (?, ?, 'x') ON CONFLICT (key) DO UPDATE SET value = excluded.value", + ).run(`genesis_pending:${STREAM}`, String(first.cut)); + db.exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T9', 'title T9', 'task', 'pending', 'medium', 'uid-T9', 'fp-T9')", + ); + }); + const r = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + expect(r).toMatchObject({ status: 'resumed', cut: first.cut }); + expect(r.snapshot?.checkpointId).toBe(first.snapshot?.checkpointId); + expect(fake.stream(STREAM).checkpoints).toHaveLength(1); + await on(m, async () => { + const db = await storeOf(dbPath); + expect(genesisCutOf(db, STREAM)).toBe(first.cut); + expect(genesisPending(db, STREAM)).toBe(false); + }); + }); + + it('refuses without the unreleased opt-in, on a journaled stream, and when the store and link name different replicas', async () => { + const { m } = await journalMachine(); + expect((await failure(on(m, () => enableSyncPush(vopts(m))))).code).toBe( + 'E_NEXUS_SYNC_REFUSED', + ); + const realReplica = m.replicaId; + m.replicaId = REPLICA_A; + fake.addProject(REMOTE_PROJECT, { [REPLICA_A]: DEVICE_A }); + link(m); + const mismatch = await failure( + on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))), + ); + expect(mismatch.code).toBe('E_NEXUS_SYNC_REFUSED'); + expect(mismatch.message).toContain(realReplica); + expect(fake.stream(STREAM).checkpoints).toHaveLength(0); + }); +}); diff --git a/packages/core/src/cloud/journal.ts b/packages/core/src/cloud/journal.ts index bb0c658562..002baeeafa 100644 --- a/packages/core/src/cloud/journal.ts +++ b/packages/core/src/cloud/journal.ts @@ -504,6 +504,12 @@ export class Journal { manifest: Manifest; cursor: PullCursor; parentCheckpointId: string | null; + /** + * The id to register it under, minted by the caller so it can be recorded + * before the upload (a genesis push adopts its own stored checkpoint after + * a crash by this id, T13302). A fresh uuidv7 when absent. + */ + checkpointId?: string; }): Promise { const { streamId, replicaId, deviceId } = this.o; if (!args.cursor.knowsAllReplicas) @@ -517,7 +523,7 @@ export class Journal { replicas[id] = { deviceId: r.deviceId, lastReplicaSeq: r.replicaSeq }; } const coversSeq = args.cursor.after; - const checkpointId = uuidv7(); + const checkpointId = args.checkpointId ?? uuidv7(); const ciphertext = seal( this.o.key, args.bundle, diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index fa49a5ce25..9e1234f5f1 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -53,6 +53,7 @@ */ import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -60,6 +61,7 @@ import type { CloudLeaseReleaseResult, CloudPushResult, CloudRestoreResult, + CloudSyncPushEnableResult, CloudVaultLease, CloudVaultScope, CloudVaultSnapshot, @@ -73,6 +75,8 @@ import { type Checkpoint, ListCheckpointsResult, ListLeasesResult, + type Manifest, + ReplayPin, } from '@cleocode/contracts/cloud'; import { nexusCloudDevicePageSchema } from '@cleocode/contracts/nexus-cloud.js'; import { @@ -96,7 +100,17 @@ import { sha256File, } from '../store/portable-bundle-scan.js'; import { FIRST_OPEN_LOCK_SUFFIX } from '../store/sqlite.js'; -import { readActiveReplicaId } from '../store/sync/replica.js'; +import { UNRELEASED_FLAGS } from '../store/sync/flags.js'; +import { + completeGenesis, + cutGenesisWithSnapshot, + GenesisRacedError, + genesisCutOf, + genesisPending, + readGenesisCut, +} from '../store/sync/genesis.js'; +import { replayPinOf } from '../store/sync/replay-pin.js'; +import { activeReplica, readActiveReplicaId } from '../store/sync/replica.js'; import { buildVaultManifest, type CarriedMachineState, @@ -122,7 +136,7 @@ import { vaultStripColumns, } from '../store/vault-manifest.js'; import { foreignWriterLeases, storeOpenElsewhere } from '../store/writer-lease.js'; -import { deriveKey } from './crypto.js'; +import { deriveKey, uuidv7 } from './crypto.js'; import { NexusError } from './http.js'; import { cursorFromCheckpoint, initialPullCursor, Journal, type PullCursor } from './journal.js'; import type { TrustedSigners } from './keys.js'; @@ -1038,6 +1052,154 @@ function tempDir(prefix: string): string { } /** `cleo cloud push`. */ +/** + * Register a checkpoint of `manifest` over `parent` (T12975): replay the + * window since the parent, append this replica's count-delta segment when the + * store's rows differ from the parent plus the window, then push the bundle + * as the checkpoint. A segment another device appends in between makes the + * server's count or replica-map check fail: replay once more and retry, once. + * + * @returns The stored checkpoint, the delta segment's seq (or null), and the + * cursor the checkpoint was signed with. + */ +async function appendCheckpoint(a: { + readonly conn: NexusVaultConnection; + readonly t: VaultTarget; + readonly journal: Journal; + readonly signers: TrustedSigners; + readonly parent: Checkpoint | null; + readonly manifest: Manifest; + readonly bundlePath: string; + readonly replicaId: string; + /** Sees each replayed window before the delta and the checkpoint (a genesis refuses a schema rise). */ + readonly onReplay?: (replay: { readonly maxSchemaVersion: number }) => void; + /** Mints each attempt's checkpoint id before its upload (a genesis records them, T13302). */ + readonly mintCheckpointId?: () => string; +}): Promise<{ cp: Checkpoint; deltaSegmentSeq: number | null; cursor: PullCursor }> { + const { conn, t, journal, parent, manifest, bundlePath, replicaId } = a; + let deltaSegmentSeq: number | null = null; + let cp: Checkpoint | null = null; + let pushedWith: PullCursor | null = null; + // A segment another device appends between our replay and our checkpoint + // makes the server's count or replica-map check fail: replay once more + // (our own delta is then part of the history) and retry (T12975). The + // retry seals the bundle under a new checkpoint id (the AAD covers it), + // so the first attempt's uploaded blob is left unreferenced: one orphan + // per retry, for the server's blob garbage collection. + for (let attempt = 0; cp === null; attempt++) { + const replay = await replaySegments( + journal, + parent ? cursorFromCheckpoint(parent) : initialPullCursor(), + a.signers, + ); + let cursor = replay.cursor; + // A segment of a newer sync schema appended since the head was read: refused + // before our delta segment (the push hands the lease back). + const windowAhead = aheadOfThisCleo(t, replay.maxSchemaVersion, null); + if (windowAhead) throw windowAhead; + if (parent) { + const between: Record = {}; + for (const d of replay.deltas) { + for (const [table, v] of Object.entries(d)) { + const p = between[table] ?? { created: 0, deleted: 0 }; + between[table] = { created: p.created + v.created, deleted: p.deleted + v.deleted }; + } + } + const need: Record = {}; + const deltas: Record = {}; + for (const table of new Set([ + ...Object.keys(manifest.tables), + ...Object.keys(parent.manifest.tables), + ...Object.keys(between), + ])) { + const expected = + (parent.manifest.tables[table]?.rows ?? 0) + + (between[table]?.created ?? 0) - + (between[table]?.deleted ?? 0); + const diff = (manifest.tables[table]?.rows ?? 0) - expected; + if (diff !== 0) { + need[table] = diff; + deltas[table] = { created: Math.max(0, diff), deleted: Math.max(0, -diff) }; + } + } + if (Object.keys(deltas).length > 0) { + // A replica's first segment is replicaSeq 0 (the server and every puller enforce it). + const mine = cursor.replicas[replicaId]; + const replicaSeq = mine ? mine.replicaSeq + 1 : 0; + const hlc = `${String(Date.now()).padStart(13, '0')}-000000-${replicaId}`; + const appended = await journal.push( + replicaSeq, + Buffer.from( + JSON.stringify({ + kind: 'cleo-vault-delta/v1', + base: parent.checkpointId, + tables: need, + }), + 'utf8', + ), + { + opCount: 1, + hlcMin: hlc, + hlcMax: hlc, + deltas, + schemaVersion: SYNC_SCHEMA_VERSION, + }, + ); + deltaSegmentSeq = appended.seq; + cursor = { + after: appended.seq, + knowsAllReplicas: cursor.knowsAllReplicas, + replicas: { + ...cursor.replicas, + [replicaId]: { deviceId: conn.deviceId, replicaSeq }, + }, + }; + } + } + try { + cp = await journal.pushCheckpoint({ + bundle: fs.readFileSync(bundlePath), + manifest, + cursor, + parentCheckpointId: parent?.checkpointId ?? null, + ...(a.mintCheckpointId ? { checkpointId: a.mintCheckpointId() } : {}), + }); + pushedWith = cursor; + } catch (err) { + if (err instanceof NexusError && err.code === 'E_LINEAGE') { + throw vaultError( + 'E_NEXUS_VAULT_BEHIND', + 'another device pushed a snapshot while this one was uploading', + 'run `cleo cloud pull`, then push again', + ); + } + // The server's v3 ratchet (reason `stream-v3`). The check above refuses a v3 head before + // anything is written; this maps a server whose head this client did not see as v3. + if (err instanceof NexusError && err.code === 'E_STREAM_VERSION') { + throw streamUpgradedError( + `the server refused the snapshot: ${t.streamId} takes only checkpoint/v3 snapshots (${err.message})`, + JOURNAL_STREAM_REMEDY, + ); + } + const countRefusal = + err instanceof NexusError && (err.code === 'E_REGRESSION' || err.code === 'E_VALIDATION'); + if (countRefusal && attempt === 0) continue; + if (countRefusal) { + throw vaultError( + 'E_NEXUS_VAULT_REFUSED', + `the server refused the snapshot: ${(err as Error).message}`, + ); + } + throw err; + } + } + if (cp === null || pushedWith === null) { + // @sync-invariant none:local-only unreachable: the loop exits only once a checkpoint is stored + throw new Error('appendCheckpoint: no checkpoint stored'); + } + return { cp, deltaSegmentSeq, cursor: pushedWith }; +} + async function pushNexusVaultImpl( opts: NexusVaultCommandOptions & { force?: boolean; hold?: boolean } = {}, ): Promise { @@ -1054,6 +1216,15 @@ async function pushNexusVaultImpl( 'run `cleo project link`', ); } + // Past its genesis cut, this store's changes travel through the change + // journal: a vault delta would count them twice (T12343 S4-1b). + const cut = await readGenesisCut(t.dbPath, t.streamId); + if (cut !== undefined) { + throw streamUpgradedError( + `this store started ${t.streamId}'s change journal (genesis cut at capture ${cut}); its changes are pushed as journal segments, not vault snapshots`, + `nothing was written; ${JOURNAL_STREAM_REMEDY}`, + ); + } const force = opts.force === true; const journal = journalFor(conn, t); const head = await streamHead(conn, t.streamId); @@ -1185,120 +1356,16 @@ async function pushNexusVaultImpl( parent?.checkpointId ?? null, { gitTracked: t.scope === 'project' ? gitTracked : [], files: syncFiles(files) }, ); - let deltaSegmentSeq: number | null = null; - let cp: Checkpoint | null = null; - // A segment another device appends between our replay and our checkpoint - // makes the server's count or replica-map check fail: replay once more - // (our own delta is then part of the history) and retry (T12975). The - // retry seals the bundle under a new checkpoint id (the AAD covers it), - // so the first attempt's uploaded blob is left unreferenced: one orphan - // per retry, for the server's blob garbage collection. - for (let attempt = 0; cp === null; attempt++) { - const replay = await replaySegments( - journal, - parent ? cursorFromCheckpoint(parent) : initialPullCursor(), - key.signers, - ); - let cursor = replay.cursor; - // A segment of a newer sync schema appended since the head was read: refused - // before our delta segment (the push hands the lease back). - const windowAhead = aheadOfThisCleo(t, replay.maxSchemaVersion, null); - if (windowAhead) throw windowAhead; - if (parent) { - const between: Record = {}; - for (const d of replay.deltas) { - for (const [table, v] of Object.entries(d)) { - const p = between[table] ?? { created: 0, deleted: 0 }; - between[table] = { created: p.created + v.created, deleted: p.deleted + v.deleted }; - } - } - const need: Record = {}; - const deltas: Record = {}; - for (const table of new Set([ - ...Object.keys(manifest.tables), - ...Object.keys(parent.manifest.tables), - ...Object.keys(between), - ])) { - const expected = - (parent.manifest.tables[table]?.rows ?? 0) + - (between[table]?.created ?? 0) - - (between[table]?.deleted ?? 0); - const diff = (manifest.tables[table]?.rows ?? 0) - expected; - if (diff !== 0) { - need[table] = diff; - deltas[table] = { created: Math.max(0, diff), deleted: Math.max(0, -diff) }; - } - } - if (Object.keys(deltas).length > 0) { - // A replica's first segment is replicaSeq 0 (the server and every puller enforce it). - const mine = cursor.replicas[replicaId]; - const replicaSeq = mine ? mine.replicaSeq + 1 : 0; - const hlc = `${String(Date.now()).padStart(13, '0')}-000000-${replicaId}`; - const appended = await journal.push( - replicaSeq, - Buffer.from( - JSON.stringify({ - kind: 'cleo-vault-delta/v1', - base: parent.checkpointId, - tables: need, - }), - 'utf8', - ), - { - opCount: 1, - hlcMin: hlc, - hlcMax: hlc, - deltas, - schemaVersion: SYNC_SCHEMA_VERSION, - }, - ); - deltaSegmentSeq = appended.seq; - cursor = { - after: appended.seq, - knowsAllReplicas: cursor.knowsAllReplicas, - replicas: { - ...cursor.replicas, - [replicaId]: { deviceId: conn.deviceId, replicaSeq }, - }, - }; - } - } - try { - cp = await journal.pushCheckpoint({ - bundle: fs.readFileSync(bundlePath), - manifest, - cursor, - parentCheckpointId: parent?.checkpointId ?? null, - }); - } catch (err) { - if (err instanceof NexusError && err.code === 'E_LINEAGE') { - throw vaultError( - 'E_NEXUS_VAULT_BEHIND', - 'another device pushed a snapshot while this one was uploading', - 'run `cleo cloud pull`, then push again', - ); - } - // The server's v3 ratchet (reason `stream-v3`). The check above refuses a v3 head before - // anything is written; this maps a server whose head this client did not see as v3. - if (err instanceof NexusError && err.code === 'E_STREAM_VERSION') { - throw streamUpgradedError( - `the server refused the snapshot: ${t.streamId} takes only checkpoint/v3 snapshots (${err.message})`, - JOURNAL_STREAM_REMEDY, - ); - } - const countRefusal = - err instanceof NexusError && - (err.code === 'E_REGRESSION' || err.code === 'E_VALIDATION'); - if (countRefusal && attempt === 0) continue; - if (countRefusal) { - throw vaultError( - 'E_NEXUS_VAULT_REFUSED', - `the server refused the snapshot: ${(err as Error).message}`, - ); - } - throw err; - } - } + const { cp, deltaSegmentSeq } = await appendCheckpoint({ + conn, + t, + journal, + signers: key.signers, + parent, + manifest, + bundlePath, + replicaId, + }); saveSynced(conn, t, cp, { gitTracked, files: syncFiles(files) }); if (opts.hold !== true) await releaseLeaseQuietly(conn, t); warnings.push(...conn.state.drainWarnings()); @@ -1323,6 +1390,323 @@ async function pushNexusVaultImpl( } } +/** Where a cut's genesis bundle waits until its checkpoint is stored: beside the store, per stream. */ +function genesisBundlePaths(t: VaultTarget): { dir: string; bundle: string; meta: string } { + const dir = path.join(path.dirname(t.dbPath), 'sync-genesis'); + const name = createHash('sha256').update(t.streamId).digest('hex').slice(0, 16); + return { + dir, + bundle: path.join(dir, `${name}.cleobundle.tar.gz`), + meta: path.join(dir, `${name}.json`), + }; +} + +/** What a cut saved for its checkpoint push: the vault manifest, replay pin and git marks. */ +const GenesisSaved = z.object({ + cut: z.number().int().nonnegative(), + vault: z.object({ + schemaVersion: z.number().int().positive(), + tables: z.record( + z.string(), + z.object({ rows: z.number().int().nonnegative(), hash: z.string() }), + ), + }), + replayPin: ReplayPin, + gitTracked: z.array(z.string()), + files: z.record(z.string(), z.string()), + /** Every checkpoint id an upload of this genesis was attempted under (T13302). */ + checkpointIds: z.array(z.string()).default([]), +}); + +async function enableSyncPushImpl( + opts: NexusVaultCommandOptions & { + /** Enable the unreleased `sync.push` (tests and staging only). Never set from user input. */ + allowUnreleased?: boolean; + } = {}, +): Promise { + if (UNRELEASED_FLAGS.has('sync.push') && opts.allowUnreleased !== true) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + 'sync.push is unreleased: its slices are still landing (T12343)', + 'nothing was cut; push stays off until the owner releases it', + ); + } + const conn = await connectNexusVault(opts); + const key = await unlockNexusAccountKey(conn); + const t = await resolveTarget(conn, key.masterKey, opts, 'push'); + const warnings: CloudWarning[] = [...conn.warnings]; + const replicaId = t.replicaId; + if (!replicaId) { + throw vaultError( + 'E_NEXUS_VAULT_NOT_LINKED', + 'this copy of the project is not attached from this device', + 'run `cleo project link`', + ); + } + const { openDualScopeDbAtPath, getDualScopeNativeDb } = await import('../store/dual-scope-db.js'); + const db = getDualScopeNativeDb( + t.scope === 'global' + ? await openDualScopeDbAtPath('global', t.dbPath) + : await openDualScopeDbAtPath('project', t.dbPath), + ); + // The cut seals as the store's bound replica; the stream knows it by the link's. + const bound = activeReplica(db, tableScopeOf(t))?.replicaId ?? null; + if (bound !== replicaId) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `this store is bound to replica ${bound ?? '(none)'}, but ${t.streamId} knows it as ${replicaId}`, + 'nothing was cut; relink the project (`cleo project link`) so both name one replica', + ); + } + const saved = genesisBundlePaths(t); + const cutBefore = genesisCutOf(db, t.streamId); + const pendingCut = cutBefore !== undefined && genesisPending(db, t.streamId); + // A cut whose checkpoint push failed kept its bundle: push that. A cut that + // crashed before its bundle was saved is resumed (or re-cut) by + // cutGenesisWithSnapshot (T13301). + const resuming = pendingCut && fs.existsSync(saved.bundle) && fs.existsSync(saved.meta); + const base = { apiUrl: conn.apiUrl, scope: t.scope, streamId: t.streamId }; + if (cutBefore !== undefined && !pendingCut) { + return { + ...base, + status: 'already', + cut: cutBefore, + sealed: 0, + folded: 0, + baselined: {}, + snapshot: null, + deltaSegmentSeq: null, + replicaSeqFloor: null, + warnings, + }; + } + const journal = journalFor(conn, t); + const head = await streamHead(conn, t.streamId); + const checkpoints = head.headCheckpointId ? await listCheckpoints(conn, t.streamId) : []; + const parent = checkpoints.find((c) => c.checkpointId === head.headCheckpointId) ?? null; + if (head.headCheckpointId && !parent) { + throw vaultError( + 'E_NEXUS_VAULT_REFUSED', + `the head snapshot ${head.headCheckpointId} is not in the stream's list`, + ); + } + if (parent) journal.verifyCheckpoint(parent, key.signers); + const ahead = aheadOfThisCleo( + t, + Math.max(head.maxSchemaVersion ?? 0, parent?.manifest.schemaVersion ?? 0), + parent, + ); + if (ahead) throw ahead; + // T13302: `genesis_pending` means "not confirmed stored", not "never + // pushed". A push the server accepted before this process died left the + // genesis checkpoint on the stream: adopt it, never cut a second genesis. + if (pendingCut) { + const kept = fs.existsSync(saved.meta) + ? GenesisSaved.parse(JSON.parse(fs.readFileSync(saved.meta, 'utf8'))) + : null; + // Only this cut's own upload, by the id recorded before it was sent. + const stored = checkpoints.find( + (c) => + isJournalSnapshot(c) && + c.replicaId === replicaId && + (kept?.checkpointIds.includes(c.checkpointId) ?? false), + ); + if (stored) { + journal.verifyCheckpoint(stored, key.signers); + const replicaSeqFloor = stored.replicas[replicaId]?.lastReplicaSeq ?? null; + completeGenesis(db, { + stream: t.streamId, + replica: replicaId, + replicaSeqFloor, + nowIso: new Date().toISOString(), + }); + saveSynced(conn, t, stored, { gitTracked: kept?.gitTracked ?? [], files: kept?.files ?? {} }); + fs.rmSync(saved.bundle, { force: true }); + fs.rmSync(saved.meta, { force: true }); + warnings.push(...conn.state.drainWarnings()); + return { + ...base, + status: 'resumed', + cut: cutBefore ?? 0, + sealed: 0, + folded: 0, + baselined: {}, + snapshot: snapshotOf(stored, await deviceNames(conn)), + deltaSegmentSeq: null, + replicaSeqFloor, + warnings, + }; + } + } + // Another device already started this stream's journal: this store joins it + // by pulling (S5), never with a second genesis. + if (isJournalSnapshot(parent)) { + throw vaultError( + 'E_NEXUS_SYNC_STREAM_JOURNALED', + `${t.streamId} already carries the change journal: its head snapshot ${parent?.checkpointId} is a journal checkpoint`, + 'nothing was cut; this store joins the journal by pulling it (`cleo cloud sync`, once pull ships)', + ); + } + const { state: synced } = syncedState(conn, t, parent ? [parent] : [], head.headCheckpointId); + if (head.headCheckpointId !== null && head.headCheckpointId !== synced?.lastCheckpointId) { + throw vaultError( + 'E_NEXUS_VAULT_BEHIND', + `another device pushed snapshot ${head.headCheckpointId} after this machine's last sync`, + 'run `cleo cloud pull` first, then enable push again', + ); + } + const git = gitTracking(t); + if (git.mode === 'unknown') warnings.push(gitUnknownWarning(t)); + + await acquireLease(conn, t, false, 'cleo sync enable push'); + try { + let cut = cutBefore ?? 0; + let sealed = 0; + let folded = 0; + let baselined: Record = {}; + let resumedCut = false; + if (!resuming) { + fs.mkdirSync(saved.dir, { recursive: true }); + const work = tempDir('cleo-sync-genesis-'); + try { + const report = await cutGenesisWithSnapshot( + db, + { + scope: tableScopeOf(t), + stream: t.streamId, + dbPath: t.dbPath, + ...(opts.allowUnreleased ? { allowUnreleased: true } : {}), + }, + async (at) => { + // The store exactly as the committed cut left it: the genesis marker + // holds every other writer off until this returns (T13296, T13297). + const exported = await exportAndRead( + t, + work, + `sync-genesis-${t.scope}`, + trackedForPush(t, git, synced), + ); + fs.copyFileSync(exported.bundlePath, saved.bundle); + const record: z.infer = { + cut: at, + vault: exported.vault, + replayPin: replayPinOf(db), + gitTracked: exported.gitTracked, + files: syncFiles(exported.files), + checkpointIds: [], + }; + fs.writeFileSync(saved.meta, `${JSON.stringify(record)}\n`); + }, + ); + if (report.refused !== null) { + fs.rmSync(saved.bundle, { force: true }); + fs.rmSync(saved.meta, { force: true }); + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `the genesis cut was refused: ${report.refused}`, + ); + } + cut = report.cut ?? 0; + resumedCut = report.resumed; + sealed = report.sealed; + folded = report.folded; + baselined = { ...report.baselined }; + } catch (err) { + // The cut was undone (a failed export, or a write raced it): nothing is saved or pushed. + fs.rmSync(saved.bundle, { force: true }); + fs.rmSync(saved.meta, { force: true }); + if (err instanceof GenesisRacedError) { + throw vaultError('E_NEXUS_SYNC_REFUSED', err.message, 'nothing was pushed; run it again'); + } + throw err; + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } + } + + const record = GenesisSaved.parse(JSON.parse(fs.readFileSync(saved.meta, 'utf8'))); + // checkpoint/v3 with empty accounting: genesis snapshots the store (§2.11 §10). + const manifest: Manifest = { + schemaVersion: SYNC_SCHEMA_VERSION, + tables: { + ...record.vault.tables, + [VAULT_FORMAT_KEY]: vaultFormatEntry(hashKeyOf(t.dataKey)), + }, + pending: [], + voided: [], + revived: [], + pruned: {}, + replayPin: record.replayPin, + }; + if (parent) { + for (const table of Object.keys(parent.manifest.tables)) { + if (isVaultAnnotationKey(table)) continue; + manifest.tables[table] ??= { rows: 0, hash: buildEmptyHash(t, table) }; + } + } + const parentVersion = parent?.manifest.schemaVersion ?? 0; + const { cp, deltaSegmentSeq, cursor } = await appendCheckpoint({ + conn, + t, + journal, + signers: key.signers, + parent, + manifest, + bundlePath: saved.bundle, + replicaId, + // Recorded before each upload, so a run that dies after the server + // stored it adopts this checkpoint instead of cutting again (T13302). + mintCheckpointId: () => { + const id = uuidv7(); + const now = GenesisSaved.parse(JSON.parse(fs.readFileSync(saved.meta, 'utf8'))); + fs.writeFileSync( + saved.meta, + `${JSON.stringify({ ...now, checkpointIds: [...now.checkpointIds, id] })}\n`, + ); + return id; + }, + onReplay: (replay) => { + // A schema rise inside the window would need its journal hash in the + // pin's transitions; genesis is cut on a window without one. + if (parent !== null && replay.maxSchemaVersion > parentVersion) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `${t.streamId} rose to sync schema ${replay.maxSchemaVersion} since its head snapshot (schema ${parentVersion})`, + 'pull first (`cleo cloud pull`), then enable push again', + ); + } + }, + }); + const replicaSeqFloor = cursor.replicas[replicaId]?.replicaSeq ?? null; + completeGenesis(db, { + stream: t.streamId, + replica: replicaId, + replicaSeqFloor, + nowIso: new Date().toISOString(), + }); + saveSynced(conn, t, cp, { gitTracked: record.gitTracked, files: record.files }); + fs.rmSync(saved.bundle, { force: true }); + fs.rmSync(saved.meta, { force: true }); + await releaseLeaseQuietly(conn, t); + warnings.push(...conn.state.drainWarnings()); + return { + ...base, + status: resuming || resumedCut ? 'resumed' : 'enabled', + cut: resuming ? record.cut : cut, + sealed, + folded, + baselined, + snapshot: snapshotOf(cp, await deviceNames(conn)), + deltaSegmentSeq, + replicaSeqFloor, + warnings, + }; + } catch (err) { + await releaseLeaseQuietly(conn, t); + throw err; + } +} + /** The keyed hash of an empty table (a table the parent lists that this store no longer has). */ function buildEmptyHash(t: VaultTarget, table: string): string { return emptyVaultTableHash(hashKeyOf(t.dataKey), table); @@ -2137,6 +2521,27 @@ export function pushNexusVault( return mapped(() => pushNexusVaultImpl(opts)); } +/** + * `cleo sync enable push`: record this store's genesis cut on its stream and + * push the genesis checkpoint (journal spec §2.11 §10; T12343 S4-1b). The + * store is snapshotted under its write lock, so other cleo processes pause + * their writes (they wait, then refuse with `E_STORE_GENESIS`) while the + * bundle is exported. A push that fails after the cut keeps the bundle beside + * the store and resumes on the next run; a cut that crashed before its bundle + * was saved is resumed or re-cut (T13301). + * + * @param opts - Scope and overrides; `allowUnreleased` for tests and staging only. + * @returns What was cut and pushed. + * @throws {NexusAccountError} `E_NEXUS_SYNC_REFUSED`, `E_NEXUS_SYNC_STREAM_JOURNALED`, + * `E_NEXUS_VAULT_BEHIND`, `E_NEXUS_VAULT_LEASE_HELD`, + * `E_NEXUS_VAULT_NOT_LINKED`, or a mapped API error. + */ +export function enableSyncPush( + opts: NexusVaultCommandOptions & { allowUnreleased?: boolean } = {}, +): Promise { + return mapped(() => enableSyncPushImpl(opts)); +} + /** * `cleo cloud pull` / `cleo cloud restore`: verify and activate a snapshot. * diff --git a/packages/core/src/store/sync/genesis.ts b/packages/core/src/store/sync/genesis.ts index 3a149efbec..546bed902d 100644 --- a/packages/core/src/store/sync/genesis.ts +++ b/packages/core/src/store/sync/genesis.ts @@ -26,6 +26,7 @@ * @module store/sync/genesis */ +import { existsSync } from 'node:fs'; import type { DatabaseSync } from 'node:sqlite'; import type { TableScope } from '@cleocode/contracts'; import { writeRestoreMarker } from '../restore-marker.js'; @@ -38,7 +39,7 @@ import { import { captureTriggerDrift, syncSetTables } from './capture.js'; import { isSyncFlagOn, setSyncFlag } from './flags.js'; import { baselineRowMeta } from './repair.js'; -import { activeReplica } from './replica.js'; +import { activeReplica, persistStoreSeq } from './replica.js'; import { hasTable } from './schema.js'; import { sealPending, sealPreconditions } from './sealer.js'; import { capturePosition } from './sequencing.js'; @@ -510,3 +511,56 @@ export async function cutGenesisWithSnapshot( release(); } } + +/** + * Finish a stream's genesis once its checkpoint is stored (S4-1b), in one + * `BEGIN IMMEDIATE`: raise the persisted replicaSeq high-water mark to the + * replica's last segment on the stream (the vault's delta segments spend this + * replica's seqs, so the first journal segment follows them), then clear + * `genesis_pending:`, which lets the push send segments. + * + * @param db - The store, outside a transaction. + * @param o - Stream, replica, the replica's last replicaSeq on the stream (null: none yet) and the time. + */ +export function completeGenesis( + db: DatabaseSync, + o: { + readonly stream: string; + readonly replica: string; + readonly replicaSeqFloor: number | null; + readonly nowIso: string; + }, +): void { + db.exec('BEGIN IMMEDIATE'); + try { + if (o.replicaSeqFloor !== null) { + persistStoreSeq(db, o.replica, o.stream, o.replicaSeqFloor, new Date(o.nowIso)); + } + db.prepare('DELETE FROM _sync_meta WHERE key = ?').run( + `${GENESIS_PENDING_KEY_PREFIX}${o.stream}`, + ); + db.exec('COMMIT'); + } catch (err) { + if (db.isTransaction) db.exec('ROLLBACK'); + throw err; + } +} + +/** + * A stream's genesis cut read from a store file with a read-only open, or + * undefined (no cut, no store, or no journal): the vault reads it before a + * push without binding or migrating anything. + * + * @param dbPath - The `cleo.db` file. + * @param stream - The stream. + */ +export async function readGenesisCut(dbPath: string, stream: string): Promise { + if (!existsSync(dbPath)) return undefined; + const { openNativeDatabase } = await import('../sqlite-native.js'); + const db = openNativeDatabase(dbPath, { readonly: true, enableWal: false }); + try { + return genesisCutOf(db, stream); + } finally { + db.close(); + } +} diff --git a/scripts/lint-no-raw-table-writes.mjs b/scripts/lint-no-raw-table-writes.mjs index f0cd559c4c..a0c53b7998 100644 --- a/scripts/lint-no-raw-table-writes.mjs +++ b/scripts/lint-no-raw-table-writes.mjs @@ -377,8 +377,8 @@ export const EXEMPT = { }, 'packages/core/src/store/sync/genesis.ts': { _sync_meta: { - count: 2, - reason: `${SYNC_BOOKKEEPING}; plus dropping a stream's genesis keys when a failed or raced cut is undone (T13296)`, + count: 3, + reason: `${SYNC_BOOKKEEPING}; plus dropping a stream's genesis keys when a failed or raced cut is undone (T13296), and clearing genesis_pending once the genesis checkpoint is stored (T13300)`, }, _sync_txn: { count: 2, From 3e85dfc0a10e8701606bfaa9fcfb6ec46dd5d670 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 14:14:19 -0700 Subject: [PATCH 06/23] fix(sync): genesis push saves its record atomically, after the race check (T13300 review) review-hotfix LOWs on #1955: - LOW-1: the bundle and its record are replaced through tmp+rename, so a crash never leaves truncated JSON that would wedge every later run or lose the recorded checkpoint ids. - LOW-2: saveSynced runs before completeGenesis, in both the success and the adoption paths; a crash in between leaves the cut pending and the next run adopts the stored checkpoint by its id. - LOW-3: the record is held in memory during the snapshot and written only after cutGenesisWithSnapshot returns, so a record on disk always means the race check passed; a crash before it is a pending cut with nothing saved, which the next run resumes (T13301). Co-Authored-By: Claude Opus 5.5 --- packages/core/src/cloud/nexus-vault.ts | 57 +++++++++++++++++++------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 9e1234f5f1..f96edd8733 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -1390,6 +1390,17 @@ async function pushNexusVaultImpl( } } +/** + * Replace `dest` atomically (tmp file, then rename): a crash leaves the old + * file or the new one, never a truncated one. + */ +function replaceFile(dest: string, from: { text: string } | { copyFrom: string }): void { + const tmp = `${dest}.${process.pid}.tmp`; + if ('text' in from) fs.writeFileSync(tmp, from.text); + else fs.copyFileSync(from.copyFrom, tmp); + fs.renameSync(tmp, dest); +} + /** Where a cut's genesis bundle waits until its checkpoint is stored: beside the store, per stream. */ function genesisBundlePaths(t: VaultTarget): { dir: string; bundle: string; meta: string } { const dir = path.join(path.dirname(t.dbPath), 'sync-genesis'); @@ -1514,13 +1525,13 @@ async function enableSyncPushImpl( if (stored) { journal.verifyCheckpoint(stored, key.signers); const replicaSeqFloor = stored.replicas[replicaId]?.lastReplicaSeq ?? null; + saveSynced(conn, t, stored, { gitTracked: kept?.gitTracked ?? [], files: kept?.files ?? {} }); completeGenesis(db, { stream: t.streamId, replica: replicaId, replicaSeqFloor, nowIso: new Date().toISOString(), }); - saveSynced(conn, t, stored, { gitTracked: kept?.gitTracked ?? [], files: kept?.files ?? {} }); fs.rmSync(saved.bundle, { force: true }); fs.rmSync(saved.meta, { force: true }); warnings.push(...conn.state.drainWarnings()); @@ -1568,6 +1579,10 @@ async function enableSyncPushImpl( if (!resuming) { fs.mkdirSync(saved.dir, { recursive: true }); const work = tempDir('cleo-sync-genesis-'); + // Filled by the snapshot callback (a holder, so the assignment survives the closure). + const snap: { taken: { bundlePath: string; record: z.infer } | null } = { + taken: null, + }; try { const report = await cutGenesisWithSnapshot( db, @@ -1586,16 +1601,17 @@ async function enableSyncPushImpl( `sync-genesis-${t.scope}`, trackedForPush(t, git, synced), ); - fs.copyFileSync(exported.bundlePath, saved.bundle); - const record: z.infer = { - cut: at, - vault: exported.vault, - replayPin: replayPinOf(db), - gitTracked: exported.gitTracked, - files: syncFiles(exported.files), - checkpointIds: [], + snap.taken = { + bundlePath: exported.bundlePath, + record: { + cut: at, + vault: exported.vault, + replayPin: replayPinOf(db), + gitTracked: exported.gitTracked, + files: syncFiles(exported.files), + checkpointIds: [], + }, }; - fs.writeFileSync(saved.meta, `${JSON.stringify(record)}\n`); }, ); if (report.refused !== null) { @@ -1606,6 +1622,16 @@ async function enableSyncPushImpl( `the genesis cut was refused: ${report.refused}`, ); } + // Saved only once the cut's race check passed: a bundle and record on + // disk always describe a cut that stands. A crash before this is a + // pending cut with nothing saved, which the next run resumes (T13301). + const snapshotted = snap.taken; + if (snapshotted === null) { + // @sync-invariant none:local-only programming error: a committed cut always ran its snapshot + throw new Error('the genesis cut returned without a snapshot'); + } + replaceFile(saved.bundle, { copyFrom: snapshotted.bundlePath }); + replaceFile(saved.meta, { text: `${JSON.stringify(snapshotted.record)}\n` }); cut = report.cut ?? 0; resumedCut = report.resumed; sealed = report.sealed; @@ -1659,10 +1685,9 @@ async function enableSyncPushImpl( mintCheckpointId: () => { const id = uuidv7(); const now = GenesisSaved.parse(JSON.parse(fs.readFileSync(saved.meta, 'utf8'))); - fs.writeFileSync( - saved.meta, - `${JSON.stringify({ ...now, checkpointIds: [...now.checkpointIds, id] })}\n`, - ); + replaceFile(saved.meta, { + text: `${JSON.stringify({ ...now, checkpointIds: [...now.checkpointIds, id] })}\n`, + }); return id; }, onReplay: (replay) => { @@ -1678,13 +1703,15 @@ async function enableSyncPushImpl( }, }); const replicaSeqFloor = cursor.replicas[replicaId]?.replicaSeq ?? null; + // Synced first: a crash before completeGenesis leaves the cut pending, + // and the next run adopts this checkpoint by its recorded id (T13302). + saveSynced(conn, t, cp, { gitTracked: record.gitTracked, files: record.files }); completeGenesis(db, { stream: t.streamId, replica: replicaId, replicaSeqFloor, nowIso: new Date().toISOString(), }); - saveSynced(conn, t, cp, { gitTracked: record.gitTracked, files: record.files }); fs.rmSync(saved.bundle, { force: true }); fs.rmSync(saved.meta, { force: true }); await releaseLeaseQuietly(conn, t); From 1b439ca857ae7013af6da0a0049caf85598495f3 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 14:56:31 -0700 Subject: [PATCH 07/23] feat(sync): push a stream's journal - seal, persist, upload (T12343 S4-2) pushStream (store/sync/push.ts) checks push is on and the stream's genesis checkpoint is stored, pauses while this device's clock runs ahead of the server's (sync.clock_ahead for the doctor; writes are never refused), refuses a store behind the server (it must rebind), then seals what is pending, packs and persists segments (advancing the device registry's high-water mark after each), and uploads the persisted bytes lowest replicaSeq first, one at a time; a duplicate counts as stored and a failed upload is resent with the same bytes. pushSyncStream (cloud/nexus-vault.ts) wires it to the journal client, the device's txn signatures, the server's Date (new Http.lastServerDate), E3's replica.lastReplicaSeq and the device registry; a segment conflict maps to E_NEXUS_SYNC_REFUSED with the rebind remedy. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-push.md | 28 ++ .../src/cloud/__tests__/nexus-vault.test.ts | 60 ++++- packages/core/src/cloud/http.ts | 8 + packages/core/src/cloud/nexus-vault.ts | 90 ++++++- .../src/store/sync/__tests__/push.test.ts | 240 ++++++++++++++++++ packages/core/src/store/sync/push.ts | 222 ++++++++++++++++ scripts/lint-no-raw-table-writes.mjs | 6 + 7 files changed, 652 insertions(+), 2 deletions(-) create mode 100644 .changeset/t12343-push.md create mode 100644 packages/core/src/store/sync/__tests__/push.test.ts create mode 100644 packages/core/src/store/sync/push.ts diff --git a/.changeset/t12343-push.md b/.changeset/t12343-push.md new file mode 100644 index 0000000000..46a5f96d91 --- /dev/null +++ b/.changeset/t12343-push.md @@ -0,0 +1,28 @@ +--- +id: t12343-push +tasks: [T12343] +kind: feat +summary: Sync push - seal, persist and upload a stream's journal segments, behind the unreleased sync.push flag (S4-2) +--- + +A store past its genesis can now push its change journal (journal spec §2.8, §2.9, §1.3; T12343 S4-2). + +- **`pushStream(db, …)`** (`store/sync/push.ts`) runs these steps in order: + 1. **Preconditions:** push is on, the stream is cut, and its genesis checkpoint is stored. No segment ever precedes the checkpoint + it extends. + 2. **Local clock check:** if this device's clock runs more than `MAX_DRIFT` (5 min) ahead of the server's `Date`, push pauses and + records `sync.clock_ahead` for `cleo doctor`. Local writes are never refused, and the key clears once the clock recovers. + 3. **Server-authoritative rollback check:** a later segment from this replica on the server than the store ever persisted means + the store was restored or copied behind it. Nothing is sent; it must rebind (T12753). + 4. **Seal and pack:** everything pending is sealed, then packed into persisted segments. The device registry's high-water mark + advances after each persist. + 5. **Upload:** the persisted bytes are uploaded, lowest `replicaSeq` first, one at a time. A duplicate counts as stored, and a + failed upload is resent with the same bytes next run. +- **`pushSyncStream(opts)`** (`cloud/nexus-vault.ts`) wires `pushStream` to the journal client: + - `sealSegment` and `push` with the persisted bytes; + - the device's transaction signatures; + - the server's `Date` (the new `Http.lastServerDate`); + - E3's `replica.lastReplicaSeq` (project streams); + - the device registry. + + A segment conflict maps to `E_NEXUS_SYNC_REFUSED` with the rebind remedy. `cleo cloud sync` (T12996) will call it. diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index be9a5163c5..456669e067 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -121,6 +121,7 @@ import { enableSyncPush, nexusVaultStatus, pushNexusVault, + pushSyncStream, releaseNexusVaultLease, restoreNexusVault, verifyNexusVault, @@ -276,10 +277,17 @@ class ApiFail extends Error { } } +/** A `Date` header the fake sends on every response, when a test sets it (the server's clock, §1.3). */ +let fakeDateHeader: string | null = null; + function json(status: number, body: object): Response { return new Response(JSON.stringify(body), { status, - headers: { 'content-type': 'application/json', 'x-request-id': 'req-test' }, + headers: { + 'content-type': 'application/json', + 'x-request-id': 'req-test', + ...(fakeDateHeader !== null ? { date: fakeDateHeader } : {}), + }, }); } @@ -5053,6 +5061,56 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { }); }); + it("S4-2: after genesis, a local write is sealed, packed and pushed as this replica's next segment; a rerun sends nothing", async () => { + const { m, dbPath } = await journalMachine(); + const enabled = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + await on(m, async () => { + (await storeOf(dbPath)).exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T5', 'title T5', 'task', 'pending', 'medium', 'uid-T5', 'fp-T5')", + ); + }); + const before = fake.stream(STREAM).segments.length; + const r = await on(m, () => pushSyncStream(vopts(m, { allowUnreleased: true }))); + expect(r).toMatchObject({ + refused: null, + clockAhead: false, + built: 1, + pushed: 1, + duplicates: 0, + }); + const s = fake.stream(STREAM); + expect(s.segments).toHaveLength(before + 1); + const seg = s.segments.at(-1); + expect(seg?.replicaId).toBe(m.replicaId); + expect(seg?.replicaSeq).toBe((enabled.replicaSeqFloor ?? -1) + 1); + expect(seg?.txnDeltas).toEqual([ + { txn: 0, deltas: { tasks_tasks: { created: 1, deleted: 0 } } }, + ]); + expect(r.lastServerSeq).toBe(seg?.seq); + const again = await on(m, () => pushSyncStream(vopts(m, { allowUnreleased: true }))); + expect(again).toMatchObject({ built: 0, pushed: 0 }); + expect(s.segments).toHaveLength(before + 1); + }); + + it('S4-2: a device clock ahead of the server pauses push', async () => { + const { m, dbPath } = await journalMachine(); + await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + await on(m, async () => { + (await storeOf(dbPath)).exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T6', 'title T6', 'task', 'pending', 'medium', 'uid-T6', 'fp-T6')", + ); + }); + const before = fake.stream(STREAM).segments.length; + fakeDateHeader = new Date(Date.now() - 10 * 60 * 1000).toUTCString(); + try { + const r = await on(m, () => pushSyncStream(vopts(m, { allowUnreleased: true }))); + expect(r).toMatchObject({ clockAhead: true, pushed: 0 }); + expect(fake.stream(STREAM).segments).toHaveLength(before); + } finally { + fakeDateHeader = null; + } + }); + it('refuses without the unreleased opt-in, on a journaled stream, and when the store and link name different replicas', async () => { const { m } = await journalMachine(); expect((await failure(on(m, () => enableSyncPush(vopts(m))))).code).toBe( diff --git a/packages/core/src/cloud/http.ts b/packages/core/src/cloud/http.ts index 938cf02dd7..73533a6b8d 100644 --- a/packages/core/src/cloud/http.ts +++ b/packages/core/src/cloud/http.ts @@ -67,6 +67,12 @@ export function isSecureUrl(url: string): boolean { export class Http { private readonly fetch: FetchLike; private readonly sleep: (ms: number) => Promise; + /** + * The server's clock, from the `Date` header of its latest response, or + * null before one arrived (or when it sent none). Push pauses when this + * device's clock runs ahead of it (journal spec §1.3). + */ + lastServerDate: Date | null = null; constructor(private readonly o: HttpOptions) { if (!isSecureUrl(o.baseUrl)) { @@ -133,6 +139,8 @@ export class Http { } | null; const requestId = res.headers.get('x-request-id'); + const served = Date.parse(res.headers.get('date') ?? ''); + if (Number.isFinite(served)) this.lastServerDate = new Date(served); if (json?.success === true) { const parsed = schema.safeParse(json.data); if (!parsed.success) { diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index f96edd8733..79e09b5b66 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -78,7 +78,10 @@ import { type Manifest, ReplayPin, } from '@cleocode/contracts/cloud'; -import { nexusCloudDevicePageSchema } from '@cleocode/contracts/nexus-cloud.js'; +import { + nexusCloudDevicePageSchema, + nexusCloudStatusSchema, +} from '@cleocode/contracts/nexus-cloud.js'; import { NEXUS_VAULT_LEASE_ROLE, nexusLeaseAcquireSchema, @@ -109,8 +112,11 @@ import { genesisPending, readGenesisCut, } from '../store/sync/genesis.js'; +import { type PushStreamReport, pushStream } from '../store/sync/push.js'; import { replayPinOf } from '../store/sync/replay-pin.js'; import { activeReplica, readActiveReplicaId } from '../store/sync/replica.js'; +import { ReplicaRegistry } from '../store/sync/replica-registry.js'; +import { signTxn } from '../store/sync/txn-signing.js'; import { buildVaultManifest, type CarriedMachineState, @@ -143,6 +149,7 @@ import type { TrustedSigners } from './keys.js'; import { manifestVersion } from './manifest-check.js'; import { canonicalGlobalReplicaBinder } from './nexus-attach.js'; import { NexusAccountError } from './nexus-auth.js'; +import { nexusQueryPath } from './nexus-cloud.js'; import { nexusApiErrorToAccountError } from './nexus-enrol.js'; import { readNexusProjectLink } from './nexus-link.js'; import { @@ -1734,6 +1741,69 @@ async function enableSyncPushImpl( } } +async function pushSyncStreamImpl( + opts: NexusVaultCommandOptions & { + /** Push although `sync.push` is unreleased (tests and staging only). Never set from user input. */ + allowUnreleased?: boolean; + } = {}, +): Promise { + const conn = await connectNexusVault(opts); + const key = await unlockNexusAccountKey(conn); + const t = await resolveTarget(conn, key.masterKey, opts, 'push'); + const replicaId = t.replicaId; + if (!replicaId) { + throw vaultError( + 'E_NEXUS_VAULT_NOT_LINKED', + 'this copy of the project is not attached from this device', + 'run `cleo project link`', + ); + } + const journal = journalFor(conn, t); + // A first request also gives the server's clock (Date header, §1.3). + await streamHead(conn, t.streamId); + // The server-authoritative rollback check (§1.5): E3 knows this replica's + // last stored segment on a project stream. + const status = + t.projectId !== null + ? await conn.find( + nexusQueryPath('/v1/status', { projectId: t.projectId, replicaId }), + nexusCloudStatusSchema, + ) + : null; + const { openDualScopeDbAtPath, getDualScopeNativeDb } = await import('../store/dual-scope-db.js'); + const db = getDualScopeNativeDb( + t.scope === 'global' + ? await openDualScopeDbAtPath('global', t.dbPath) + : await openDualScopeDbAtPath('project', t.dbPath), + ); + try { + return await pushStream(db, { + scope: tableScopeOf(t), + stream: t.streamId, + replica: replicaId, + project: t.projectId, + sealer: (replicaSeq, plaintext, meta) => journal.sealSegment(replicaSeq, plaintext, meta), + signTxn: (stream, txn) => signTxn(conn.keys.signing, stream, txn), + upload: async (seg) => journal.push(seg.replicaSeq, new Uint8Array(0), seg.meta, seg.sealed), + serverDate: conn.http.lastServerDate, + serverLastReplicaSeq: status?.replica?.lastReplicaSeq ?? null, + registry: ReplicaRegistry.forDevice(conn.deviceId), + ...(opts.allowUnreleased ? { allowUnreleased: true } : {}), + }); + } catch (err) { + // The server holds a different segment at this replicaSeq, or a later one + // than this store has: the store is behind (§1.5, §2.9). + if (err instanceof NexusError && err.code === 'E_CONFLICT') { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `the server refused a segment of replica ${replicaId} on ${t.streamId}: ${err.message}`, + 'this store is behind the stream (restored or copied); it must rebind to a new replica (T12753)', + ); + } + throw err; + } +} + /** The keyed hash of an empty table (a table the parent lists that this store no longer has). */ function buildEmptyHash(t: VaultTarget, table: string): string { return emptyVaultTableHash(hashKeyOf(t.dataKey), table); @@ -2569,6 +2639,24 @@ export function enableSyncPush( return mapped(() => enableSyncPushImpl(opts)); } +/** + * Push this store's change journal on its stream (journal spec §2.8, §2.9; + * T12343 S4-2): seal, persist segments, upload the lowest unpushed first. + * Pauses while this device's clock runs ahead of the server's, and refuses a + * store behind the server (it must rebind). `cleo cloud sync` (T12996) runs + * it with pull and apply. + * + * @param opts - Scope and overrides; `allowUnreleased` for tests and staging only. + * @returns What was sealed, persisted and stored, or why nothing was sent. + * @throws {NexusAccountError} `E_NEXUS_SYNC_REFUSED` (a segment conflict: the store is behind), + * `E_NEXUS_VAULT_NOT_LINKED`, or a mapped API error. + */ +export function pushSyncStream( + opts: NexusVaultCommandOptions & { allowUnreleased?: boolean } = {}, +): Promise { + return mapped(() => pushSyncStreamImpl(opts)); +} + /** * `cleo cloud pull` / `cleo cloud restore`: verify and activate a snapshot. * diff --git a/packages/core/src/store/sync/__tests__/push.test.ts b/packages/core/src/store/sync/__tests__/push.test.ts new file mode 100644 index 0000000000..ee59ae5c5e --- /dev/null +++ b/packages/core/src/store/sync/__tests__/push.test.ts @@ -0,0 +1,240 @@ +/** + * Pushing a stream's journal (T12343 S4-2; journal spec §2.8, §2.9, §1.3). + * + * Real stores (capture, seal, the real sealer and segment builder) and a + * fake uploader standing in for the journal client. + * + * @task T12343 + */ + +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import type { DatabaseSync } from 'node:sqlite'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + _resetDualScopeDbCache, + getDualScopeNativeDb, + openDualScopeDbAtPath, +} from '../../dual-scope-db.js'; +import { + ROW_IDENTITY_META_TABLE, + ROW_IDENTITY_RECIPE, + ROW_IDENTITY_RECIPE_KEY, +} from '../../row-identity.js'; +import { finishCaptureFrame, openCaptureFrame, setCaptureEnabled } from '../capture.js'; +import { setSyncFlag } from '../flags.js'; +import { completeGenesis, cutGenesis } from '../genesis.js'; +import { CLOCK_AHEAD_KEY, pushStream, type SegmentUploader } from '../push.js'; +import { ensureProjectReplica, storeHwm } from '../replica.js'; +import { ReplicaRegistry } from '../replica-registry.js'; +import type { PersistedSegment } from '../segments.js'; + +const SYNC_SCHEMA = resolve(import.meta.dirname, '../../../../migrations/sync-journal'); +const STREAM = 'project:0192ffff-7f00-7000-8000-00000000000f'; +let clock = Date.now(); +let dir: string; +let dbPath: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'cleo-push-')); + mkdirSync(join(dir, 'cleo'), { recursive: true }); + mkdirSync(join(dir, 'p', '.cleo'), { recursive: true }); + dbPath = join(dir, 'p', '.cleo', 'cleo.db'); + vi.stubEnv('CLEO_HOME', join(dir, 'cleo')); + vi.stubEnv('CLEO_ROOT', undefined); + vi.stubEnv('CLEO_DIR', undefined); +}); + +afterEach(() => { + _resetDualScopeDbCache(); + vi.unstubAllEnvs(); + rmSync(dir, { recursive: true, force: true }); +}); + +const addTask = (id: string) => + `INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('${id}', 'title ${id}', 'task', 'pending', 'medium', 'uid-${id}', 'fp-${id}')`; + +const n = (db: DatabaseSync, sql: string, ...args: Array): number => + Number((db.prepare(sql).get(...args) as { n: number }).n); + +function write(db: DatabaseSync, sql: string): void { + db.exec('BEGIN IMMEDIATE'); + const frame = openCaptureFrame(db, 'write', null); + db.exec(sql); + finishCaptureFrame(db, frame); + db.exec('COMMIT'); +} + +/** A store past its genesis (cut, checkpoint stored): push may send. */ +async function pushing( + floor: number | null = null, +): Promise<{ db: DatabaseSync; replica: string; registry: ReplicaRegistry }> { + const db = getDualScopeNativeDb(await openDualScopeDbAtPath('project', dbPath)); + db.prepare( + `INSERT INTO ${ROW_IDENTITY_META_TABLE} (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, + ).run(ROW_IDENTITY_RECIPE_KEY, ROW_IDENTITY_RECIPE); + setCaptureEnabled(db, 'project', true, { schemaRoot: SYNC_SCHEMA }); + setSyncFlag(db, 'sync.seal', true, { schemaRoot: SYNC_SCHEMA, allowUnreleased: true }); + const registry = new ReplicaRegistry(join(dir, 'registry.json'), 'host-1'); + const { replicaId } = ensureProjectReplica(db, { dbPath, mode: 'test', registry }); + const cut = cutGenesis(db, { + scope: 'project', + stream: STREAM, + now: () => ++clock, + env: {}, + allowUnreleased: true, + }); + expect(cut.refused).toBeNull(); + completeGenesis(db, { + stream: STREAM, + replica: replicaId, + replicaSeqFloor: floor, + nowIso: new Date(++clock).toISOString(), + }); + return { db, replica: replicaId, registry }; +} + +/** A fake server: stores segments by (replicaSeq, hash), assigns seqs, answers duplicates. */ +function fakeServer() { + const stored = new Map(); + const sent: PersistedSegment[] = []; + let head = 0; + let failNext = false; + const upload: SegmentUploader = async (seg) => { + sent.push(seg); + if (failNext) { + failNext = false; + throw new Error('network down'); + } + const prior = stored.get(seg.replicaSeq); + if (prior) { + expect(prior.hash, 'a resend carried different bytes').toBe(seg.segmentHash); + return { seq: prior.seq, duplicate: true }; + } + head += 1; + stored.set(seg.replicaSeq, { hash: seg.segmentHash, seq: head }); + return { seq: head, duplicate: false }; + }; + return { + upload, + sent, + stored, + failOnce: () => { + failNext = true; + }, + /** The server stored the next upload but the response was lost. */ + storeThenLose: (seg: PersistedSegment) => { + head += 1; + stored.set(seg.replicaSeq, { hash: seg.segmentHash, seq: head }); + }, + }; +} + +const opts = (replica: string, server: ReturnType, extra = {}) => ({ + scope: 'project' as const, + stream: STREAM, + replica, + project: null, + sealer: (_seq: number, plaintext: Uint8Array) => Buffer.from(plaintext), + signTxn: (_stream: string, txn: Parameters[1]['signTxn']>[1]) => + txn, + upload: server.upload, + serverDate: new Date(clock), + serverLastReplicaSeq: null, + now: () => ++clock, + env: {}, + allowUnreleased: true, + ...extra, +}); + +describe('pushStream (S4-2)', () => { + it('seals, persists and uploads the pending journal once; a second run sends nothing', async () => { + const { db, replica, registry } = await pushing(); + write(db, addTask('T1')); + write(db, addTask('T2')); + const server = fakeServer(); + const r = await pushStream(db, opts(replica, server, { registry })); + expect(r).toMatchObject({ refused: null, clockAhead: false, sealed: 2, built: 1, pushed: 1 }); + expect(server.sent).toHaveLength(1); + expect(server.sent[0]?.replicaSeq).toBe(0); + expect(r.lastServerSeq).toBe(1); + expect( + db.prepare('SELECT state, server_seq FROM _sync_segment WHERE replica_seq = 0').get(), + ).toEqual({ state: 'pushed', server_seq: 1 }); + // The device registry follows the store's persisted mark. + expect(registry.get(replica)?.hwm[STREAM]).toBe(0); + expect(storeHwm(db, replica)[STREAM]).toBe(0); + const again = await pushStream(db, opts(replica, server)); + expect(again).toMatchObject({ sealed: 0, built: 0, pushed: 0 }); + expect(server.sent).toHaveLength(1); + }); + + it('an upload that fails is resent next run with the same bytes; a stored-but-lost one counts as a duplicate', async () => { + const { db, replica } = await pushing(); + write(db, addTask('T1')); + const server = fakeServer(); + server.failOnce(); + await expect(pushStream(db, opts(replica, server))).rejects.toThrow('network down'); + expect(n(db, "SELECT count(*) AS n FROM _sync_segment WHERE state = 'sealed'")).toBe(1); + // The server stores it this time, but the answer is lost: the next run resends, a duplicate. + const [first] = server.sent; + if (!first) throw new Error('fixture'); + server.storeThenLose(first); + const r = await pushStream(db, opts(replica, server)); + expect(r).toMatchObject({ pushed: 1, duplicates: 1, built: 0 }); + expect(server.sent.map((s) => s.segmentHash)).toEqual([first.segmentHash, first.segmentHash]); + expect(n(db, "SELECT count(*) AS n FROM _sync_segment WHERE state = 'pushed'")).toBe(1); + }); + + it('the first journal segment follows the replica seq the genesis push recorded', async () => { + const { db, replica } = await pushing(4); // the vault's deltas spent replicaSeq 0..4 + write(db, addTask('T1')); + const server = fakeServer(); + await pushStream(db, opts(replica, server, { serverLastReplicaSeq: 4 })); + expect(server.sent[0]?.replicaSeq).toBe(5); + }); + + it('a clock ahead of the server pauses push (no seal, no segment) and is reported until it recovers', async () => { + const { db, replica } = await pushing(); + write(db, addTask('T1')); + const server = fakeServer(); + const paused = await pushStream( + db, + opts(replica, server, { serverDate: new Date(clock - 10 * 60 * 1000) }), + ); + expect(paused).toMatchObject({ clockAhead: true, sealed: 0, built: 0, pushed: 0 }); + expect(n(db, 'SELECT count(*) AS n FROM _sync_meta WHERE key = ?', CLOCK_AHEAD_KEY)).toBe(1); + expect(n(db, "SELECT count(*) AS n FROM _sync_capture WHERE state = 'live'")).toBeGreaterThan( + 0, + ); + const ok = await pushStream(db, opts(replica, server)); + expect(ok).toMatchObject({ clockAhead: false, pushed: 1 }); + expect(n(db, 'SELECT count(*) AS n FROM _sync_meta WHERE key = ?', CLOCK_AHEAD_KEY)).toBe(0); + }); + + it('a server ahead of the store (restored or copied) is refused before anything is sealed or sent', async () => { + const { db, replica } = await pushing(); + write(db, addTask('T1')); + const server = fakeServer(); + const r = await pushStream(db, opts(replica, server, { serverLastReplicaSeq: 2 })); + expect(r.refused).toMatch(/behind .*rebind/); + expect(r).toMatchObject({ sealed: 0, built: 0, pushed: 0 }); + expect(server.sent).toHaveLength(0); + }); + + it('refuses with push off, with no cut, and while the genesis checkpoint is not stored', async () => { + const { db, replica } = await pushing(); + const server = fakeServer(); + db.prepare("DELETE FROM _sync_meta WHERE key = 'genesis_pending:' || ?").run(STREAM); + db.prepare( + "INSERT INTO _sync_meta (key, value, updated_at) VALUES ('genesis_pending:' || ?, '0', 'x')", + ).run(STREAM); + expect((await pushStream(db, opts(replica, server))).refused).toMatch(/not stored yet/); + db.prepare("DELETE FROM _sync_meta WHERE key LIKE 'genesis_%'").run(); + expect((await pushStream(db, opts(replica, server))).refused).toMatch(/no genesis cut/); + setSyncFlag(db, 'sync.push', false, { schemaRoot: SYNC_SCHEMA }); + expect((await pushStream(db, opts(replica, server))).refused).toBe('sync.push is off'); + expect(server.sent).toHaveLength(0); + }); +}); diff --git a/packages/core/src/store/sync/push.ts b/packages/core/src/store/sync/push.ts new file mode 100644 index 0000000000..6b4aae2f28 --- /dev/null +++ b/packages/core/src/store/sync/push.ts @@ -0,0 +1,222 @@ +/** + * Push one stream's journal (journal spec §2.8 "Persist before push", + * "Upload", §2.9, §1.3 "Local clock ahead"; T12343 S4-2). + * + * {@link pushStream} runs, in order: + * 1. the preconditions: `sync.push` on, the stream cut and its genesis + * checkpoint stored (`genesis_pending` clear), so no segment ever + * precedes the checkpoint it extends; + * 2. the local-clock check: when this device's clock runs more than + * `MAX_DRIFT` ahead of the server's date, push pauses (persisted as + * `sync.clock_ahead` for `cleo doctor`), so future-stamped HLCs do not + * spread; local writes are never refused; + * 3. the rollback check: when the server already holds a later segment from + * this replica than the store has persisted, the store is behind (a + * restore or a copy) and must rebind (§1.5, §2.9); nothing is sent; + * 4. seal everything pending, then pack and persist segments + * ({@link buildSegment}), advancing the device registry's high-water mark + * after each persist (the store ahead of the registry is the safe + * direction); + * 5. upload the persisted segments, lowest `replicaSeq` first, one at a time, + * always the exact persisted bytes; a duplicate counts as stored. + * + * The upload goes through a {@link SegmentUploader} port: the journal client + * in production (`cloud/nexus-vault.ts`), a fake in tests. + * + * @task T12343 + * @module store/sync/push + */ + +import type { DatabaseSync } from 'node:sqlite'; +import type { TableScope } from '@cleocode/contracts'; +import { isSyncFlagOn } from './flags.js'; +import { genesisCutOf, genesisPending } from './genesis.js'; +import { storeHwm } from './replica.js'; +import type { ReplicaRegistry } from './replica-registry.js'; +import { hasTable } from './schema.js'; +import { sealPending } from './sealer.js'; +import { + type BuildSegmentOptions, + buildSegment, + markSegmentPushed, + type PersistedSegment, + type SegmentSealer, + unpushedSegments, +} from './segments.js'; + +/** `MAX_DRIFT` (§1.3, Q6): how far this device's clock may run ahead of the server's. */ +export const MAX_DRIFT_MS = 5 * 60 * 1000; + +/** `_sync_meta` key set while push is paused for a clock ahead of the server's (`cleo doctor`). */ +export const CLOCK_AHEAD_KEY = 'sync.clock_ahead'; + +/** Uploads one persisted segment; resolves with the server's seq, `duplicate` when it was stored already. */ +export type SegmentUploader = ( + segment: PersistedSegment, +) => Promise<{ readonly seq: number; readonly duplicate: boolean }>; + +/** Options for {@link pushStream}. */ +export interface PushStreamOptions { + readonly scope: TableScope; + readonly stream: string; + /** This store's bound replica (the stream knows it by the same id). */ + readonly replica: string; + /** The merge key's project, or null on a home stream. */ + readonly project: string | null; + readonly sealer: SegmentSealer; + /** Signs each packed transaction for the stream ({@link BuildSegmentOptions.signTxn}). */ + readonly signTxn: BuildSegmentOptions['signTxn']; + readonly upload: SegmentUploader; + /** The server's date from its latest response, or null when unknown. */ + readonly serverDate: Date | null; + /** The server's last stored `replicaSeq` for this replica on the stream, or null when unknown or none. */ + readonly serverLastReplicaSeq: number | null; + /** The device registry, whose high-water mark advances after each persist. */ + readonly registry?: ReplicaRegistry; + /** Wall clock. @defaultValue Date.now */ + readonly now?: () => number; + readonly env?: NodeJS.ProcessEnv; + /** Seal and push although the flags are unreleased (tests and staging only). Never set from user input. */ + readonly allowUnreleased?: boolean; + /** Override {@link MAX_DRIFT_MS} (tests). */ + readonly maxDriftMs?: number; +} + +/** What {@link pushStream} did. */ +export interface PushStreamReport { + readonly stream: string; + /** Why nothing was sent, or null. */ + readonly refused: string | null; + /** Push paused because this device's clock is ahead of the server's. */ + readonly clockAhead: boolean; + /** Transactions sealed, segments packed and persisted, segments the server stored (`duplicates` of them retries). */ + readonly sealed: number; + readonly built: number; + readonly pushed: number; + readonly duplicates: number; + /** The server seq of the last segment stored, or null. */ + readonly lastServerSeq: number | null; +} + +const empty = (stream: string, fields: Partial): PushStreamReport => ({ + stream, + refused: null, + clockAhead: false, + sealed: 0, + built: 0, + pushed: 0, + duplicates: 0, + lastServerSeq: null, + ...fields, +}); + +/** Set or clear {@link CLOCK_AHEAD_KEY}. */ +function markClockAhead(db: DatabaseSync, value: string | null, atIso: string): void { + if (value === null) { + db.prepare('DELETE FROM _sync_meta WHERE key = ?').run(CLOCK_AHEAD_KEY); + return; + } + db.prepare( + 'INSERT INTO _sync_meta (key, value, updated_at) VALUES (?, ?, ?) ' + + 'ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at', + ).run(CLOCK_AHEAD_KEY, value, atIso); +} + +/** + * Seal, persist and upload one stream's pending journal (module docs). + * Must run outside a transaction. An upload failure propagates after the + * segments before it were recorded; the next run resends from the lowest + * unpushed `replicaSeq`, with the same bytes. + * + * @param db - The store. + * @param o - {@link PushStreamOptions}. + * @returns What was sealed, persisted and stored, or why nothing was sent. + */ +export async function pushStream( + db: DatabaseSync, + o: PushStreamOptions, +): Promise { + const env = o.env ?? process.env; + const now = o.now ?? Date.now; + if (!hasTable(db, '_sync_segment')) + return empty(o.stream, { refused: 'sync schema not installed' }); + if (!isSyncFlagOn(db, 'sync.push', env)) return empty(o.stream, { refused: 'sync.push is off' }); + if (genesisCutOf(db, o.stream) === undefined) { + return empty(o.stream, { + refused: `${o.stream} has no genesis cut on this store: run \`cleo sync enable push\``, + }); + } + if (genesisPending(db, o.stream)) { + return empty(o.stream, { + refused: `${o.stream}'s genesis checkpoint is not stored yet: run \`cleo sync enable push\` again`, + }); + } + const atIso = new Date(now()).toISOString(); + // §1.3: a clock running ahead of the server's pauses push, never writes. + const drift = o.serverDate === null ? 0 : now() - o.serverDate.getTime(); + if (drift > (o.maxDriftMs ?? MAX_DRIFT_MS)) { + markClockAhead( + db, + JSON.stringify({ driftMs: drift, serverDate: o.serverDate?.toISOString() ?? null }), + atIso, + ); + return empty(o.stream, { clockAhead: true }); + } + markClockAhead(db, null, atIso); + // §1.5 / §2.9: the server is authoritative. A later segment there than this + // store ever persisted means the store was restored or copied behind it. + const local = storeHwm(db, o.replica)[o.stream] ?? null; + if (o.serverLastReplicaSeq !== null && (local === null || o.serverLastReplicaSeq > local)) { + return empty(o.stream, { + refused: `the server holds replicaSeq ${o.serverLastReplicaSeq} of replica ${o.replica} on ${o.stream}, but this store persisted only ${local ?? 'none'}: the store is behind (restored or copied) and must rebind to a new replica (T12753)`, + }); + } + + let sealed = 0; + for (;;) { + const r = sealPending(db, { + scope: o.scope, + replica: o.replica, + now, + env, + ...(o.allowUnreleased ? { allowUnreleased: true } : {}), + }); + sealed += r.txns; + if (r.refused !== null || r.captures === 0 || r.pending.length > 0) break; + } + let built = 0; + for (;;) { + const seg = buildSegment(db, { + stream: o.stream, + replica: o.replica, + scope: o.scope, + project: o.project, + sealer: o.sealer, + signTxn: o.signTxn, + nowIso: new Date(now()).toISOString(), + }); + if (seg === null) break; + built += 1; + // §2.8: the registry follows the store; a crash before this leaves the + // store ahead of it, the safe direction. + if (o.registry?.get(o.replica)) o.registry.advanceHwm(o.replica, o.stream, seg.replicaSeq); + } + + let pushed = 0; + let duplicates = 0; + let lastServerSeq: number | null = null; + for (const seg of unpushedSegments(db, o.stream, o.replica)) { + const stored = await o.upload(seg); + markSegmentPushed(db, { + stream: o.stream, + replica: o.replica, + replicaSeq: seg.replicaSeq, + serverSeq: stored.seq, + nowIso: new Date(now()).toISOString(), + }); + pushed += 1; + if (stored.duplicate) duplicates += 1; + lastServerSeq = stored.seq; + } + return empty(o.stream, { sealed, built, pushed, duplicates, lastServerSeq }); +} diff --git a/scripts/lint-no-raw-table-writes.mjs b/scripts/lint-no-raw-table-writes.mjs index a0c53b7998..2fb3fcbf73 100644 --- a/scripts/lint-no-raw-table-writes.mjs +++ b/scripts/lint-no-raw-table-writes.mjs @@ -396,6 +396,12 @@ export const EXEMPT = { 'capture undo (local-only): dropped when a failed or raced genesis cut is undone and no stream is cut, so undo is off again (T13296)', }, }, + 'packages/core/src/store/sync/push.ts': { + _sync_meta: { + count: 2, + reason: `${SYNC_BOOKKEEPING}; sets and clears sync.clock_ahead while push pauses for a clock ahead of the server's (§1.3, T12343)`, + }, + }, 'packages/core/src/store/sync/segments.ts': { _sync_row_meta: { count: 1, From 512046ce133b39f77c719ed0cf3ec0c153dcd615 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 15:17:20 -0700 Subject: [PATCH 08/23] fix(sync): push refuses a store bound to another replica than the link (T13304) review-hotfix on #1957: - MED T13304: pushSyncStream passed the link's replica into sealing, packing and the high-water mark without checking the store's bound replica. After a relink or rebind, push sealed as one replica while the other sealers sealed as the bound one, whose txns were then never selected. It now refuses the mismatch, nothing sealed or sent, as enableSyncPush does. - LOW-1: the clock drift is measured as the server's offset at receipt (Http.lastServerOffsetMs), so time spent after the response never counts as drift. - LOW-2: sealing is bounded to 8 passes per push; the next run seals the rest. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-push.md | 16 +++++---- .../src/cloud/__tests__/nexus-vault.test.ts | 35 +++++++++++++++++++ packages/core/src/cloud/http.ts | 12 ++++--- packages/core/src/cloud/nexus-vault.ts | 12 ++++++- .../src/store/sync/__tests__/push.test.ts | 7 ++-- packages/core/src/store/sync/push.ts | 22 +++++++----- 6 files changed, 78 insertions(+), 26 deletions(-) diff --git a/.changeset/t12343-push.md b/.changeset/t12343-push.md index 46a5f96d91..0027f0eb3d 100644 --- a/.changeset/t12343-push.md +++ b/.changeset/t12343-push.md @@ -1,6 +1,6 @@ --- id: t12343-push -tasks: [T12343] +tasks: [T12343, T13304] kind: feat summary: Sync push - seal, persist and upload a stream's journal segments, behind the unreleased sync.push flag (S4-2) --- @@ -10,19 +10,23 @@ A store past its genesis can now push its change journal (journal spec §2.8, § - **`pushStream(db, …)`** (`store/sync/push.ts`) runs these steps in order: 1. **Preconditions:** push is on, the stream is cut, and its genesis checkpoint is stored. No segment ever precedes the checkpoint it extends. - 2. **Local clock check:** if this device's clock runs more than `MAX_DRIFT` (5 min) ahead of the server's `Date`, push pauses and - records `sync.clock_ahead` for `cleo doctor`. Local writes are never refused, and the key clears once the clock recovers. + 2. **Local clock check:** if this device's clock runs more than `MAX_DRIFT` (5 min) ahead of the server's, push pauses and + records `sync.clock_ahead` for `cleo doctor`. The offset is measured when the server's `Date` arrives, so later time never + counts. Local writes are never refused, and the key clears once the clock recovers. 3. **Server-authoritative rollback check:** a later segment from this replica on the server than the store ever persisted means the store was restored or copied behind it. Nothing is sent; it must rebind (T12753). - 4. **Seal and pack:** everything pending is sealed, then packed into persisted segments. The device registry's high-water mark + 4. **Seal and pack:** pending captures are sealed (at most 8 passes, so steady writers never hold the push back), then packed + into persisted segments. The device registry's high-water mark advances after each persist. 5. **Upload:** the persisted bytes are uploaded, lowest `replicaSeq` first, one at a time. A duplicate counts as stored, and a failed upload is resent with the same bytes next run. - **`pushSyncStream(opts)`** (`cloud/nexus-vault.ts`) wires `pushStream` to the journal client: - `sealSegment` and `push` with the persisted bytes; - the device's transaction signatures; - - the server's `Date` (the new `Http.lastServerDate`); + - the server's clock offset (the new `Http.lastServerOffsetMs`, measured at receipt); - E3's `replica.lastReplicaSeq` (project streams); - the device registry. - A segment conflict maps to `E_NEXUS_SYNC_REFUSED` with the rebind remedy. `cleo cloud sync` (T12996) will call it. + It refuses, with nothing sealed or sent, when the store's bound replica is not the one the stream knows (T13304); push would + otherwise seal as one replica and never select the other's transactions. A segment conflict maps to `E_NEXUS_SYNC_REFUSED` + with the rebind remedy. `cleo cloud sync` (T12996) will call it. diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 456669e067..52b734f633 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5092,6 +5092,41 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { expect(s.segments).toHaveLength(before + 1); }); + it('S4-2: a store bound to another replica than the link names is refused, nothing sealed or sent (T13304)', async () => { + const { m, dbPath } = await journalMachine(); + await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + await on(m, async () => { + (await storeOf(dbPath)).exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T7', 'title T7', 'task', 'pending', 'medium', 'uid-T7', 'fp-T7')", + ); + }); + const segments = fake.stream(STREAM).segments.length; + const bound = m.replicaId; + m.replicaId = REPLICA_A; // relinked: the stream now knows this store as another replica + link(m); + const refused = await failure(on(m, () => pushSyncStream(vopts(m, { allowUnreleased: true })))); + expect(refused.code).toBe('E_NEXUS_SYNC_REFUSED'); + expect(refused.message).toContain(bound); + expect(fake.stream(STREAM).segments).toHaveLength(segments); + await on(m, async () => { + const db = await storeOf(dbPath); + expect( + ( + db.prepare("SELECT count(*) AS n FROM _sync_capture WHERE state = 'live'").get() as { + n: number; + } + ).n, + ).toBeGreaterThan(0); + expect( + ( + db.prepare("SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed'").get() as { + n: number; + } + ).n, + ).toBe(0); + }); + }); + it('S4-2: a device clock ahead of the server pauses push', async () => { const { m, dbPath } = await journalMachine(); await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); diff --git a/packages/core/src/cloud/http.ts b/packages/core/src/cloud/http.ts index 73533a6b8d..c3abc6f53b 100644 --- a/packages/core/src/cloud/http.ts +++ b/packages/core/src/cloud/http.ts @@ -68,11 +68,13 @@ export class Http { private readonly fetch: FetchLike; private readonly sleep: (ms: number) => Promise; /** - * The server's clock, from the `Date` header of its latest response, or - * null before one arrived (or when it sent none). Push pauses when this - * device's clock runs ahead of it (journal spec §1.3). + * The server's clock relative to this device's, from the `Date` header of + * its latest response: `serverDate − receivedAt` in ms (negative when this + * device runs ahead), or null before one arrived (or when it sent none). + * Measured at receipt, so time spent after the response never counts as + * drift. Push pauses when this device runs ahead (journal spec §1.3). */ - lastServerDate: Date | null = null; + lastServerOffsetMs: number | null = null; constructor(private readonly o: HttpOptions) { if (!isSecureUrl(o.baseUrl)) { @@ -140,7 +142,7 @@ export class Http { | null; const requestId = res.headers.get('x-request-id'); const served = Date.parse(res.headers.get('date') ?? ''); - if (Number.isFinite(served)) this.lastServerDate = new Date(served); + if (Number.isFinite(served)) this.lastServerOffsetMs = served - Date.now(); if (json?.success === true) { const parsed = schema.safeParse(json.data); if (!parsed.success) { diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 79e09b5b66..4024f02c1d 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -1776,6 +1776,16 @@ async function pushSyncStreamImpl( ? await openDualScopeDbAtPath('global', t.dbPath) : await openDualScopeDbAtPath('project', t.dbPath), ); + // T13304: push seals and packs as the replica the stream knows; the store + // must be bound to that same one, or its own txns would never be selected. + const bound = activeReplica(db, tableScopeOf(t))?.replicaId ?? null; + if (bound !== replicaId) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `this store is bound to replica ${bound ?? '(none)'}, but ${t.streamId} knows it as ${replicaId}`, + 'nothing was sealed or sent; relink the project (`cleo project link`) so both name one replica', + ); + } try { return await pushStream(db, { scope: tableScopeOf(t), @@ -1785,7 +1795,7 @@ async function pushSyncStreamImpl( sealer: (replicaSeq, plaintext, meta) => journal.sealSegment(replicaSeq, plaintext, meta), signTxn: (stream, txn) => signTxn(conn.keys.signing, stream, txn), upload: async (seg) => journal.push(seg.replicaSeq, new Uint8Array(0), seg.meta, seg.sealed), - serverDate: conn.http.lastServerDate, + serverOffsetMs: conn.http.lastServerOffsetMs, serverLastReplicaSeq: status?.replica?.lastReplicaSeq ?? null, registry: ReplicaRegistry.forDevice(conn.deviceId), ...(opts.allowUnreleased ? { allowUnreleased: true } : {}), diff --git a/packages/core/src/store/sync/__tests__/push.test.ts b/packages/core/src/store/sync/__tests__/push.test.ts index ee59ae5c5e..10fe2eff50 100644 --- a/packages/core/src/store/sync/__tests__/push.test.ts +++ b/packages/core/src/store/sync/__tests__/push.test.ts @@ -140,7 +140,7 @@ const opts = (replica: string, server: ReturnType, extra = {} signTxn: (_stream: string, txn: Parameters[1]['signTxn']>[1]) => txn, upload: server.upload, - serverDate: new Date(clock), + serverOffsetMs: 0, serverLastReplicaSeq: null, now: () => ++clock, env: {}, @@ -199,10 +199,7 @@ describe('pushStream (S4-2)', () => { const { db, replica } = await pushing(); write(db, addTask('T1')); const server = fakeServer(); - const paused = await pushStream( - db, - opts(replica, server, { serverDate: new Date(clock - 10 * 60 * 1000) }), - ); + const paused = await pushStream(db, opts(replica, server, { serverOffsetMs: -10 * 60 * 1000 })); expect(paused).toMatchObject({ clockAhead: true, sealed: 0, built: 0, pushed: 0 }); expect(n(db, 'SELECT count(*) AS n FROM _sync_meta WHERE key = ?', CLOCK_AHEAD_KEY)).toBe(1); expect(n(db, "SELECT count(*) AS n FROM _sync_capture WHERE state = 'live'")).toBeGreaterThan( diff --git a/packages/core/src/store/sync/push.ts b/packages/core/src/store/sync/push.ts index 6b4aae2f28..3db8df7a9a 100644 --- a/packages/core/src/store/sync/push.ts +++ b/packages/core/src/store/sync/push.ts @@ -47,6 +47,9 @@ import { /** `MAX_DRIFT` (§1.3, Q6): how far this device's clock may run ahead of the server's. */ export const MAX_DRIFT_MS = 5 * 60 * 1000; +/** Sealing passes per push before it sends what is persisted (the rest waits for the next run). */ +const SEAL_PASSES = 8; + /** `_sync_meta` key set while push is paused for a clock ahead of the server's (`cleo doctor`). */ export const CLOCK_AHEAD_KEY = 'sync.clock_ahead'; @@ -67,8 +70,11 @@ export interface PushStreamOptions { /** Signs each packed transaction for the stream ({@link BuildSegmentOptions.signTxn}). */ readonly signTxn: BuildSegmentOptions['signTxn']; readonly upload: SegmentUploader; - /** The server's date from its latest response, or null when unknown. */ - readonly serverDate: Date | null; + /** + * The server's clock minus this device's, measured when its latest response + * arrived (negative: this device runs ahead), or null when unknown. + */ + readonly serverOffsetMs: number | null; /** The server's last stored `replicaSeq` for this replica on the stream, or null when unknown or none. */ readonly serverLastReplicaSeq: number | null; /** The device registry, whose high-water mark advances after each persist. */ @@ -153,13 +159,9 @@ export async function pushStream( } const atIso = new Date(now()).toISOString(); // §1.3: a clock running ahead of the server's pauses push, never writes. - const drift = o.serverDate === null ? 0 : now() - o.serverDate.getTime(); + const drift = o.serverOffsetMs === null ? 0 : -o.serverOffsetMs; if (drift > (o.maxDriftMs ?? MAX_DRIFT_MS)) { - markClockAhead( - db, - JSON.stringify({ driftMs: drift, serverDate: o.serverDate?.toISOString() ?? null }), - atIso, - ); + markClockAhead(db, JSON.stringify({ driftMs: drift }), atIso); return empty(o.stream, { clockAhead: true }); } markClockAhead(db, null, atIso); @@ -173,7 +175,9 @@ export async function pushStream( } let sealed = 0; - for (;;) { + // Bounded: steady concurrent writers never keep the push from sending; the + // next run seals the rest. + for (let pass = 0; pass < SEAL_PASSES; pass++) { const r = sealPending(db, { scope: o.scope, replica: o.replica, From 9f1fd32e916ca58239db43ac95998a9d626f202c Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 18:15:48 -0700 Subject: [PATCH 09/23] feat(sync): pull a stream's journal - verify, stage once, apply (T12343 S5-1) pullStream (store/sync/pull.ts) pages through the stream after the store's persisted cursor and, per page in one transaction, decodes each segment (a cleo-vault-delta/v1 segment carries no ops and is passed over), verifies every transaction's device signature before writing anything, skips transaction ids already staged from the stream (_sync_seen_txn: a re-delivered transaction is never staged or applied twice), stages the rest and advances _sync_cursor; then the applier applies, own echoes included. pullSyncStream (cloud/nexus-vault.ts) wires it to the journal client and the trusted device keys, refuses a store bound to another replica than the link, and starts a store with no position after the checkpoint it last synced. New local-only migration t12343-pull-cursor; both tables are cleared from vault bundles. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-pull.md | 25 ++ .../migration.sql | 28 ++ .../src/cloud/__tests__/nexus-attach.test.ts | 2 + .../src/cloud/__tests__/nexus-vault.test.ts | 26 ++ packages/core/src/cloud/nexus-vault.ts | 115 +++++++- .../table-classification-gate.test.ts | 4 + .../src/store/sync/__tests__/pull.test.ts | 231 ++++++++++++++++ packages/core/src/store/sync/pull.ts | 250 ++++++++++++++++++ .../core/src/store/table-classification.ts | 24 ++ scripts/lint-no-raw-table-writes.mjs | 12 + 10 files changed, 715 insertions(+), 2 deletions(-) create mode 100644 .changeset/t12343-pull.md create mode 100644 packages/core/migrations/sync-journal/20261008120000_t12343-pull-cursor/migration.sql create mode 100644 packages/core/src/store/sync/__tests__/pull.test.ts create mode 100644 packages/core/src/store/sync/pull.ts diff --git a/.changeset/t12343-pull.md b/.changeset/t12343-pull.md new file mode 100644 index 0000000000..f5af7912e9 --- /dev/null +++ b/.changeset/t12343-pull.md @@ -0,0 +1,25 @@ +--- +id: t12343-pull +tasks: [T12343] +kind: feat +summary: Sync pull - stage and apply a stream's verified segments, never a transaction twice (S5-1) +--- + +A store can now pull its stream's journal (journal spec §3.1, §2.8; T12343 S5-1). + +- **`pullStream(db, …)`** (`store/sync/pull.ts`) pages through the stream after the store's persisted cursor. For each page, in one + local transaction: + - **Decode:** `deflate-raw` canonical JSON of `LedgerTxn[]`, parsed with the ledger schema. A vault count-delta segment + (`cleo-vault-delta/v1`) carries no ops and is passed over. + - **Verify:** every transaction's signature is checked against the segment's device key, before anything of the page is written. + A bad signature or a malformed body refuses the segment (`SegmentRefusedError`). + - **Deduplicate:** every transaction id already staged from the stream is skipped (`_sync_seen_txn`), so a re-delivered + transaction is never staged or applied twice and a counter delta never doubles. + - **Stage and advance:** the rest are staged, and the cursor advances (`_sync_cursor`). + + Then the applier applies, in stream order, own echoes included (they feed the fast path and the scoped rebase). +- **`pullSyncStream(opts)`** (`cloud/nexus-vault.ts`) wires `pullStream` to the journal client and the trusted device keys. It refuses + a store bound to another replica than the link. A store with no pull position starts after the checkpoint it last synced (its + genesis, or the snapshot it restored), seeded from that checkpoint's signed replica map. +- **New sync-journal migration** `t12343-pull-cursor`, local-only: `_sync_cursor` and `_sync_seen_txn`. Both are cleared from vault + bundles, so a restored store resumes from its own checkpoint. diff --git a/packages/core/migrations/sync-journal/20261008120000_t12343-pull-cursor/migration.sql b/packages/core/migrations/sync-journal/20261008120000_t12343-pull-cursor/migration.sql new file mode 100644 index 0000000000..f2a7e131ff --- /dev/null +++ b/packages/core/migrations/sync-journal/20261008120000_t12343-pull-cursor/migration.sql @@ -0,0 +1,28 @@ +-- T12343 S5-1 (journal spec §3.1 "Receive") — where a store's pull stands. +-- +-- _sync_cursor: per stream, the PullCursor the journal client verifies the +-- next page against (the last applied stream seq, and per replica its pinned +-- device and last replicaSeq). It is advanced in the SAME local transaction +-- that stages the segments it covers (§3.1 step 3), so a crash re-pulls +-- nothing twice and loses nothing. +-- +-- _sync_seen_txn: every transaction id this store has staged from a stream. +-- A transaction re-delivered under another segment (a replayed or re-sealed +-- segment, a retired replica's late segment carrying a txn already seen) is +-- skipped, never staged twice: a counter delta ($inc) applied twice would +-- double (T12343 S5-1). +-- +-- Local-only. +-- +-- Do not edit this file once applied: its hash is recorded per store. +CREATE TABLE IF NOT EXISTS _sync_cursor ( + stream TEXT PRIMARY KEY, + cursor_json TEXT NOT NULL, + updated_at TEXT NOT NULL +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS _sync_seen_txn ( + stream TEXT NOT NULL, + txn TEXT NOT NULL, + seq INTEGER NOT NULL, + PRIMARY KEY (stream, txn) +) WITHOUT ROWID; diff --git a/packages/core/src/cloud/__tests__/nexus-attach.test.ts b/packages/core/src/cloud/__tests__/nexus-attach.test.ts index e180eb624d..780a40a5e1 100644 --- a/packages/core/src/cloud/__tests__/nexus-attach.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-attach.test.ts @@ -307,6 +307,7 @@ describe('presence and binding hold their guarantees (review M3)', () => { '_sync_capture', '_sync_clock', '_sync_conflict', + '_sync_cursor', '_sync_field_leave', '_sync_foreign_touch', '_sync_frame', @@ -318,6 +319,7 @@ describe('presence and binding hold their guarantees (review M3)', () => { '_sync_replica', '_sync_row_meta', '_sync_row_undo', + '_sync_seen_txn', '_sync_segment', '_sync_segment_txn', '_sync_sequenced', diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 52b734f633..f95aa138dd 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -120,6 +120,7 @@ import { listNexusNamedProjects, resolveNexusProjectRef } from '../nexus-project import { enableSyncPush, nexusVaultStatus, + pullSyncStream, pushNexusVault, pushSyncStream, releaseNexusVaultLease, @@ -5127,6 +5128,31 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { }); }); + it('S5-1: pull starts after the genesis checkpoint, verifies and stages the own echo, and sequences it', async () => { + const { m, dbPath } = await journalMachine(); + await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + await on(m, async () => { + (await storeOf(dbPath)).exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T8', 'title T8', 'task', 'pending', 'medium', 'uid-T8', 'fp-T8')", + ); + }); + const pushed = await on(m, () => pushSyncStream(vopts(m, { allowUnreleased: true }))); + expect(pushed.pushed).toBe(1); + const pulled = await on(m, () => pullSyncStream(vopts(m))); + expect(pulled).toMatchObject({ segments: 1, staged: 1, redelivered: 0, vaultDeltas: 0 }); + expect(pulled.after).toBe(pushed.lastServerSeq); + expect(pulled.apply.applied).toBe(1); + await on(m, async () => { + const db = await storeOf(dbPath); + // The own echo was sequenced (§3.5): its transaction is no longer unsequenced. + expect( + (db.prepare('SELECT count(*) AS n FROM _sync_sequenced').get() as { n: number }).n, + ).toBe(1); + }); + const again = await on(m, () => pullSyncStream(vopts(m))); + expect(again).toMatchObject({ segments: 0, staged: 0 }); + }); + it('S4-2: a device clock ahead of the server pauses push', async () => { const { m, dbPath } = await journalMachine(); await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 4024f02c1d..52f156a309 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -112,11 +112,13 @@ import { genesisPending, readGenesisCut, } from '../store/sync/genesis.js'; +import { type PullStreamReport, pullStream, readStreamCursor } from '../store/sync/pull.js'; import { type PushStreamReport, pushStream } from '../store/sync/push.js'; import { replayPinOf } from '../store/sync/replay-pin.js'; import { activeReplica, readActiveReplicaId } from '../store/sync/replica.js'; import { ReplicaRegistry } from '../store/sync/replica-registry.js'; -import { signTxn } from '../store/sync/txn-signing.js'; +import { sealPending } from '../store/sync/sealer.js'; +import { firstBadTxnSignature, signTxn } from '../store/sync/txn-signing.js'; import { buildVaultManifest, type CarriedMachineState, @@ -145,7 +147,7 @@ import { foreignWriterLeases, storeOpenElsewhere } from '../store/writer-lease.j import { deriveKey, uuidv7 } from './crypto.js'; import { NexusError } from './http.js'; import { cursorFromCheckpoint, initialPullCursor, Journal, type PullCursor } from './journal.js'; -import type { TrustedSigners } from './keys.js'; +import { signerKeys, type TrustedSigners } from './keys.js'; import { manifestVersion } from './manifest-check.js'; import { canonicalGlobalReplicaBinder } from './nexus-attach.js'; import { NexusAccountError } from './nexus-auth.js'; @@ -798,6 +800,10 @@ const VAULT_CLEARED_JOURNAL_TABLES = [ '_sync_frame', '_sync_quarantine', '_sync_apply_intent', + // Where this device's pull stands (T12343 S5-1): a restored store resumes + // from the checkpoint it restored, never from another device's position. + '_sync_cursor', + '_sync_seen_txn', ] as const; /** @@ -1814,6 +1820,95 @@ async function pushSyncStreamImpl( } } +async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise { + const conn = await connectNexusVault(opts); + const key = await unlockNexusAccountKey(conn); + const t = await resolveTarget(conn, key.masterKey, opts, 'push'); + const replicaId = t.replicaId; + if (!replicaId) { + throw vaultError( + 'E_NEXUS_VAULT_NOT_LINKED', + 'this copy of the project is not attached from this device', + 'run `cleo project link`', + ); + } + const journal = journalFor(conn, t); + const { openDualScopeDbAtPath, getDualScopeNativeDb } = await import('../store/dual-scope-db.js'); + const db = getDualScopeNativeDb( + t.scope === 'global' + ? await openDualScopeDbAtPath('global', t.dbPath) + : await openDualScopeDbAtPath('project', t.dbPath), + ); + // Own echoes are recognised by the bound replica: it must be the stream's (T13304). + const bound = activeReplica(db, tableScopeOf(t))?.replicaId ?? null; + if (bound !== replicaId) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `this store is bound to replica ${bound ?? '(none)'}, but ${t.streamId} knows it as ${replicaId}`, + 'nothing was pulled; relink the project (`cleo project link`) so both name one replica', + ); + } + // A store with no pull position starts after the checkpoint it last synced + // (the genesis it cut, or the snapshot it restored), seeded from its signed + // replica map. + let initialCursor = readStreamCursor(db, t.streamId); + if (initialCursor === null) { + const head = await streamHead(conn, t.streamId); + const checkpoints = head.headCheckpointId ? await listCheckpoints(conn, t.streamId) : []; + const { state: synced } = syncedState(conn, t, checkpoints, head.headCheckpointId); + const from = checkpoints.find((c) => c.checkpointId === synced?.lastCheckpointId) ?? null; + if (from === null) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `this store has synced no checkpoint of ${t.streamId}, so it has no position to pull from`, + 'restore the stream (`cleo cloud restore`), or enable push (`cleo sync enable push`), first', + ); + } + journal.verifyCheckpoint(from, key.signers); + initialCursor = cursorFromCheckpoint(from); + } + return pullStream(db, { + scope: tableScopeOf(t), + stream: t.streamId, + replica: replicaId, + initialCursor, + pull: async (cursor) => { + const page = await journal.pull( + { + after: cursor.after, + knowsAllReplicas: cursor.knowsAllReplicas, + replicas: { ...cursor.replicas }, + }, + key.signers, + ); + return { + segments: page.segments.map((s) => ({ + seq: s.seq, + replicaId: s.replicaId, + replicaSeq: s.replicaSeq, + deviceId: s.deviceId, + plaintext: s.plaintext, + schemaVersion: s.meta.schemaVersion, + })), + cursor: page.cursor, + head: page.head, + }; + }, + verify: (deviceId, txns) => { + const keys = signerKeys(key.signers, deviceId); + const first = keys[0]; + if (first === undefined) return 0; + for (const k of keys) { + if (firstBadTxnSignature(k.publicKey, t.streamId, txns) === null) return null; + } + return firstBadTxnSignature(first.publicKey, t.streamId, txns); + }, + seal: () => { + sealPending(db, { scope: tableScopeOf(t), replica: replicaId }); + }, + }); +} + /** The keyed hash of an empty table (a table the parent lists that this store no longer has). */ function buildEmptyHash(t: VaultTarget, table: string): string { return emptyVaultTableHash(hashKeyOf(t.dataKey), table); @@ -2667,6 +2762,22 @@ export function pushSyncStream( return mapped(() => pushSyncStreamImpl(opts)); } +/** + * Pull this store's stream journal (journal spec §3.1; T12343 S5-1): verified + * segments after the store's cursor are decoded, signature-checked, staged + * (a re-delivered transaction never twice, a vault delta passed over) and + * applied. A store with no pull position starts after the checkpoint it last + * synced. `cleo cloud sync` (T12996) runs it with push. + * + * @param opts - Scope and overrides. + * @returns What was received, staged and applied. + * @throws {NexusAccountError} `E_NEXUS_SYNC_REFUSED` (replica mismatch, no position), + * `E_NEXUS_VAULT_NOT_LINKED`, or a mapped API error. + */ +export function pullSyncStream(opts: NexusVaultCommandOptions = {}): Promise { + return mapped(() => pullSyncStreamImpl(opts)); +} + /** * `cleo cloud pull` / `cleo cloud restore`: verify and activate a snapshot. * diff --git a/packages/core/src/store/__tests__/table-classification-gate.test.ts b/packages/core/src/store/__tests__/table-classification-gate.test.ts index 7a9a7c1627..47f983574e 100644 --- a/packages/core/src/store/__tests__/table-classification-gate.test.ts +++ b/packages/core/src/store/__tests__/table-classification-gate.test.ts @@ -199,6 +199,8 @@ const SYNC_REBASE_DDL = 'packages/core/migrations/sync-journal/20261006120000_t13193-rebase-undo/migration.sql'; const SYNC_SEGMENT_DDL = 'packages/core/migrations/sync-journal/20261007120000_t12343-segment-outbox/migration.sql'; +const SYNC_PULL_DDL = + 'packages/core/migrations/sync-journal/20261008120000_t12343-pull-cursor/migration.sql'; const SYNC_JOURNAL_TABLES = { _sync_capture: { class: 'local-only', ddl: SYNC_CAPTURE_DDL }, _sync_frame: { class: 'local-only', ddl: SYNC_CAPTURE_DDL }, @@ -217,6 +219,8 @@ const SYNC_JOURNAL_TABLES = { _sync_sequenced: { class: 'local-only', ddl: SYNC_REBASE_DDL }, _sync_segment: { class: 'local-only', ddl: SYNC_SEGMENT_DDL }, _sync_segment_txn: { class: 'local-only', ddl: SYNC_SEGMENT_DDL }, + _sync_cursor: { class: 'local-only', ddl: SYNC_PULL_DDL }, + _sync_seen_txn: { class: 'local-only', ddl: SYNC_PULL_DDL }, _sync_clock: { class: 'local-only', ddl: SYNC_JOURNAL_DDL }, _sync_meta: { class: 'local-only', ddl: SYNC_JOURNAL_DDL }, _sync_replica: { class: 'local-only', ddl: SYNC_JOURNAL_DDL }, diff --git a/packages/core/src/store/sync/__tests__/pull.test.ts b/packages/core/src/store/sync/__tests__/pull.test.ts new file mode 100644 index 0000000000..a33e1cdfbb --- /dev/null +++ b/packages/core/src/store/sync/__tests__/pull.test.ts @@ -0,0 +1,231 @@ +/** + * Pulling a stream's journal (T12343 S5-1; journal spec §3.1, §2.8). + * + * Two real stores: A authors and persists segments with the real sealer, + * segment builder and transaction signatures; a fake stream serves them; + * B pulls, stages and applies. + * + * @task T12343 + */ + +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import type { DatabaseSync } from 'node:sqlite'; +import { SYNC_SCHEMA_VERSION } from '@cleocode/contracts/sync-schema.js'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { generateEd25519 } from '../../../cloud/crypto.js'; +import { + _resetDualScopeDbCache, + getDualScopeNativeDb, + openDualScopeDbAtPath, +} from '../../dual-scope-db.js'; +import { finishCaptureFrame, openCaptureFrame, setCaptureEnabled } from '../capture.js'; +import { setSyncFlag } from '../flags.js'; +import { + type PulledStreamSegment, + pullStream, + readStreamCursor, + SegmentRefusedError, + type StreamCursor, +} from '../pull.js'; +import { sealPending } from '../sealer.js'; +import { buildSegment } from '../segments.js'; +import { firstBadTxnSignature, signTxn } from '../txn-signing.js'; + +const SYNC_SCHEMA = resolve(import.meta.dirname, '../../../../migrations/sync-journal'); +const STREAM = 'project:0192ffff-7f00-7000-8000-00000000000f'; +const RA = '0192aaaa-7f00-7000-8000-00000000000a'; +const RB = '0192bbbb-7f00-7000-8000-00000000000b'; +const DEV_A = 'dev-a'; +const KEY_A = generateEd25519(); +let clock = Date.now(); +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'cleo-pull-')); + mkdirSync(join(dir, 'cleo'), { recursive: true }); + vi.stubEnv('CLEO_HOME', join(dir, 'cleo')); + vi.stubEnv('CLEO_ROOT', undefined); + vi.stubEnv('CLEO_DIR', undefined); +}); + +afterEach(() => { + _resetDualScopeDbCache(); + vi.unstubAllEnvs(); + rmSync(dir, { recursive: true, force: true }); +}); + +async function store(name: string): Promise { + mkdirSync(join(dir, name, '.cleo'), { recursive: true }); + const db = getDualScopeNativeDb( + await openDualScopeDbAtPath('project', join(dir, name, '.cleo', 'cleo.db')), + ); + setCaptureEnabled(db, 'project', true, { schemaRoot: SYNC_SCHEMA }); + setSyncFlag(db, 'sync.seal', true, { schemaRoot: SYNC_SCHEMA, allowUnreleased: true }); + return db; +} + +const seal = (db: DatabaseSync, replica: string) => () => { + sealPending(db, { + scope: 'project', + replica, + now: () => ++clock, + env: {}, + allowUnreleased: true, + }); +}; + +function write(db: DatabaseSync, sql: string): void { + db.exec('BEGIN IMMEDIATE'); + const frame = openCaptureFrame(db, 'write', null); + db.exec(sql); + finishCaptureFrame(db, frame); + db.exec('COMMIT'); + seal(db, RA)(); +} + +const addTask = (id: string) => + `INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('${id}', 'title ${id}', 'task', 'pending', 'medium', 'uid-${id}', 'fp-${id}')`; + +/** A's next segment, signed by A's device, as the stream serves it (the sealer is identity here). */ +function authorSegment(a: DatabaseSync): Uint8Array { + const seg = buildSegment(a, { + stream: STREAM, + replica: RA, + scope: 'project', + project: null, + sealer: (_seq, plaintext) => Buffer.from(plaintext), + signTxn: (stream, txn) => signTxn(KEY_A, stream, txn), + nowIso: new Date(++clock).toISOString(), + }); + if (!seg) throw new Error('nothing to pack'); + return seg.sealed; +} + +/** A fake stream: segments in seq order, served after the cursor, one page at a time. */ +function fakeStream() { + const segments: PulledStreamSegment[] = []; + let pages = 0; + return { + segments, + get pages() { + return pages; + }, + append(plaintext: Uint8Array, replicaSeq: number, replicaId = RA, deviceId = DEV_A): void { + segments.push({ + seq: segments.length + 1, + replicaId, + replicaSeq, + deviceId, + plaintext, + schemaVersion: SYNC_SCHEMA_VERSION, + }); + }, + pull: async (cursor: StreamCursor) => { + pages += 1; + const page = segments.filter((s) => s.seq > cursor.after).slice(0, 2); + const replicas = { ...cursor.replicas }; + for (const s of page) + replicas[s.replicaId] = { deviceId: s.deviceId, replicaSeq: s.replicaSeq }; + return { + segments: page, + cursor: { after: page.at(-1)?.seq ?? cursor.after, knowsAllReplicas: true, replicas }, + head: segments.length, + }; + }, + }; +} + +const START: StreamCursor = { after: 0, knowsAllReplicas: true, replicas: {} }; + +const pullOpts = (b: DatabaseSync, stream: ReturnType) => ({ + scope: 'project' as const, + stream: STREAM, + replica: RB, + pull: stream.pull, + verify: (deviceId: string, txns: Parameters[2]) => + deviceId === DEV_A ? firstBadTxnSignature(KEY_A.publicKey, STREAM, txns) : 0, + initialCursor: START, + now: () => ++clock, + seal: seal(b, RB), +}); + +const n = (db: DatabaseSync, sql: string): number => + Number((db.prepare(sql).get() as { n: number }).n); + +describe('pullStream (S5-1)', () => { + it("stages and applies another replica's segments in stream order, persisting the cursor; a rerun pulls nothing", async () => { + const a = await store('a'); + const b = await store('b'); + const stream = fakeStream(); + write(a, addTask('T1')); + stream.append(authorSegment(a), 0); + write(a, addTask('T2')); + stream.append(authorSegment(a), 1); + write(a, "UPDATE tasks_tasks SET priority = 'high' WHERE id = 'T1'"); + stream.append(authorSegment(a), 2); + const r = await pullStream(b, pullOpts(b, stream)); + expect(r).toMatchObject({ segments: 3, staged: 3, redelivered: 0, after: 3, head: 3 }); + expect(r.apply.applied).toBe(3); + expect(b.prepare("SELECT priority FROM tasks_tasks WHERE id = 'T1'").get()).toEqual({ + priority: 'high', + }); + expect(readStreamCursor(b, STREAM)).toMatchObject({ after: 3 }); + const again = await pullStream(b, pullOpts(b, stream)); + expect(again).toMatchObject({ segments: 0, staged: 0, after: 3 }); + }); + + it('a re-delivered transaction is never staged or applied twice', async () => { + const a = await store('a'); + const b = await store('b'); + const stream = fakeStream(); + write(a, addTask('T1')); + const first = authorSegment(a); + stream.append(first, 0); + await pullStream(b, pullOpts(b, stream)); + // The same transaction arrives again in a later segment (a replayed or re-sealed segment). + stream.append(first, 1); + const r = await pullStream(b, pullOpts(b, stream)); + expect(r).toMatchObject({ segments: 1, staged: 0, redelivered: 1 }); + expect(n(b, 'SELECT count(*) AS n FROM _sync_inbox')).toBe(1); + expect(n(b, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'T1'")).toBe(1); + expect(n(b, 'SELECT count(*) AS n FROM _sync_conflict')).toBe(0); + }); + + it('a vault delta segment carries no ops and is passed over', async () => { + const a = await store('a'); + const b = await store('b'); + const stream = fakeStream(); + stream.append( + Buffer.from( + JSON.stringify({ kind: 'cleo-vault-delta/v1', base: 'cp-1', tables: { tasks_tasks: 2 } }), + ), + 0, + ); + write(a, addTask('T1')); + stream.append(authorSegment(a), 1); + const r = await pullStream(b, pullOpts(b, stream)); + expect(r).toMatchObject({ segments: 2, vaultDeltas: 1, staged: 1, after: 2 }); + expect(n(b, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'T1'")).toBe(1); + }); + + it('a transaction its device did not sign refuses the segment: nothing of its page is staged', async () => { + const a = await store('a'); + const b = await store('b'); + const stream = fakeStream(); + write(a, addTask('T1')); + stream.append(authorSegment(a), 0, RA, 'dev-mallory'); // served under another device's pin + await expect(pullStream(b, pullOpts(b, stream))).rejects.toBeInstanceOf(SegmentRefusedError); + expect(n(b, 'SELECT count(*) AS n FROM _sync_inbox')).toBe(0); + expect(readStreamCursor(b, STREAM)).toBeNull(); + }); + + it('a body that is neither a ledger segment nor a vault delta is refused', async () => { + const b = await store('b'); + const stream = fakeStream(); + stream.append(Buffer.from('not a segment'), 0); + await expect(pullStream(b, pullOpts(b, stream))).rejects.toThrow(/not a ledger segment/); + expect(readStreamCursor(b, STREAM)).toBeNull(); + }); +}); diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts new file mode 100644 index 0000000000..ab57f1e084 --- /dev/null +++ b/packages/core/src/store/sync/pull.ts @@ -0,0 +1,250 @@ +/** + * Pull one stream's journal into the store (journal spec §3.1 "Receive", + * §2.9; T12343 S5-1). + * + * {@link pullStream} pages through the stream after the store's persisted + * `_sync_cursor` (the journal client verifies order, gaps, the replica-to- + * device pin, trust, hashes and decryption before anything is returned), + * and for each segment, in ONE local transaction per page: + * - decodes it: `deflate-raw(canonical JSON of LedgerTxn[])`, parsed with + * the ledger schema; a vault delta segment (`cleo-vault-delta/v1`, a count + * correction a vault push appended before the stream's genesis) carries no + * ops and is passed over; + * - checks every transaction's signature against the segment's device key + * (§2.8); one bad signature refuses the segment, and the pull stops before + * staging anything of it; + * - skips every transaction id already staged from this stream + * (`_sync_seen_txn`): a re-delivered transaction is never staged twice, so + * a counter delta is never applied twice; + * - stages the rest (`stageTxns`), schema-ahead ones included (the applier + * holds them `refused-schema` and replays them after an upgrade); + * - advances the cursor. + * + * Then the applier applies what is staged, in stream order, own echoes + * included (they feed the fast path and the scoped rebase, §3.5). + * + * The page source is a port ({@link SegmentPuller}): the journal client in + * production (`cloud/nexus-vault.ts`), a fake in tests. + * + * @task T12343 + * @module store/sync/pull + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { inflateRawSync } from 'node:zlib'; +import type { TableScope } from '@cleocode/contracts'; +import { LedgerTxn } from '@cleocode/contracts/ledger'; +import { type ApplyReport, type ApplyStagedOptions, applyStagedTxns } from './apply/applier.js'; +import { withImmediateTransaction } from './clock-store.js'; +import { stageTxns } from './inbox.js'; +import { hasTable } from './schema.js'; + +/** Where a pull stands (the journal client's `PullCursor`). */ +export interface StreamCursor { + /** The last stream seq staged. */ + readonly after: number; + /** Whether `replicas` lists every replica with a segment at or below `after`. */ + readonly knowsAllReplicas: boolean; + /** Per replica: the device pinned to it and its last replicaSeq staged. */ + readonly replicas: Readonly< + Record + >; +} + +/** One verified, decrypted segment, as the journal client returns it. */ +export interface PulledStreamSegment { + readonly seq: number; + readonly replicaId: string; + readonly replicaSeq: number; + readonly deviceId: string; + readonly plaintext: Uint8Array; + readonly schemaVersion: number; +} + +/** Pulls the next verified page after `cursor`. */ +export type SegmentPuller = (cursor: StreamCursor) => Promise<{ + readonly segments: readonly PulledStreamSegment[]; + readonly cursor: StreamCursor; + readonly head: number; +}>; + +/** The index of a segment's first transaction its device did not sign, or null when every one is signed. */ +export type TxnVerifier = (deviceId: string, txns: readonly LedgerTxn[]) => number | null; + +/** The plaintext kind of a vault count-delta segment (`cloud/nexus-vault.ts`). */ +export const VAULT_DELTA_KIND = 'cleo-vault-delta/v1'; + +/** Options for {@link pullStream}. */ +export interface PullStreamOptions { + readonly scope: TableScope; + readonly stream: string; + /** This store's bound replica (own echoes are recognised by it). */ + readonly replica: string; + readonly pull: SegmentPuller; + readonly verify: TxnVerifier; + /** The cursor to start from when the store has none persisted (the checkpoint it was cut from or restored). */ + readonly initialCursor: StreamCursor; + /** Wall clock. @defaultValue Date.now */ + readonly now?: () => number; + /** Seals pending local captures before a rebase (the applier's `seal`). */ + readonly seal: ApplyStagedOptions['seal']; + /** Applier options passed through (pages, budget). */ + readonly apply?: Partial>; +} + +/** What {@link pullStream} did. */ +export interface PullStreamReport { + readonly stream: string; + /** Segments received, vault deltas passed over, transactions staged, re-deliveries skipped. */ + readonly segments: number; + readonly vaultDeltas: number; + readonly staged: number; + readonly redelivered: number; + /** The stream position the store has staged up to, and the server's head. */ + readonly after: number; + readonly head: number; + readonly apply: ApplyReport; +} + +/** A segment the store refuses to stage (a malformed body or a bad transaction signature). */ +export class SegmentRefusedError extends Error { + readonly code = 'E_SYNC_SEGMENT_REFUSED'; +} + +/** The store's persisted cursor for `stream`, or null. */ +export function readStreamCursor(db: DatabaseSync, stream: string): StreamCursor | null { + if (!hasTable(db, '_sync_cursor')) return null; + const row = db.prepare('SELECT cursor_json FROM _sync_cursor WHERE stream = ?').get(stream) as + | { cursor_json: string } + | undefined; + return row ? (JSON.parse(row.cursor_json) as StreamCursor) : null; +} + +/** Decode a segment's transactions, or `null` for a vault delta segment. */ +function decode(seg: PulledStreamSegment): LedgerTxn[] | null { + let text: string; + try { + text = inflateRawSync(seg.plaintext).toString('utf8'); + } catch { + // Not deflated: a vault delta segment is plain JSON. + try { + const raw: unknown = JSON.parse(Buffer.from(seg.plaintext).toString('utf8')); + if ( + typeof raw === 'object' && + raw !== null && + 'kind' in raw && + raw.kind === VAULT_DELTA_KIND + ) { + return null; + } + } catch { + // fall through to the refusal + } + // @sync-invariant none:input-shape a segment body that is neither a ledger segment nor a vault delta is never staged + throw new SegmentRefusedError( + `segment ${seg.seq} of replica ${seg.replicaId}: not a ledger segment`, + ); + } + const parsed = LedgerTxn.array().safeParse(JSON.parse(text)); + if (!parsed.success) { + // @sync-invariant none:input-shape a ledger segment that does not parse is never staged + throw new SegmentRefusedError( + `segment ${seg.seq} of replica ${seg.replicaId}: transactions do not parse`, + ); + } + return parsed.data; +} + +/** + * Pull and stage a stream's new segments, then apply (module docs). Must run + * outside a transaction. A refused segment stops the pull with + * {@link SegmentRefusedError} after the pages before it were staged. + * + * @param db - The store. + * @param o - {@link PullStreamOptions}. + * @returns What was received, staged and applied. + */ +export async function pullStream( + db: DatabaseSync, + o: PullStreamOptions, +): Promise { + const now = o.now ?? Date.now; + let cursor = readStreamCursor(db, o.stream) ?? o.initialCursor; + let segments = 0; + let vaultDeltas = 0; + let staged = 0; + let redelivered = 0; + let head = cursor.after; + for (;;) { + const page = await o.pull(cursor); + head = page.head; + const nowIso = new Date(now()).toISOString(); + // Decode and verify the whole page before writing anything of it. + const decoded = page.segments.map((seg) => { + const txns = decode(seg); + if (txns !== null) { + const bad = o.verify(seg.deviceId, txns); + if (bad !== null) { + // @sync-invariant none:input-shape a transaction its device did not sign is never staged (§2.8) + throw new SegmentRefusedError( + `segment ${seg.seq} of replica ${seg.replicaId}: transaction ${bad} is not signed by device ${seg.deviceId}`, + ); + } + } + return { seg, txns }; + }); + withImmediateTransaction(db, () => { + const seen = db.prepare( + 'INSERT INTO _sync_seen_txn (stream, txn, seq) VALUES (?, ?, ?) ON CONFLICT (stream, txn) DO NOTHING', + ); + for (const { seg, txns } of decoded) { + segments += 1; + if (txns === null) { + vaultDeltas += 1; + continue; + } + // A transaction already staged under an earlier segment is re-delivered: skip it. + const fresh = txns.filter((t) => Number(seen.run(o.stream, t.txn, seg.seq).changes) === 1); + redelivered += txns.length - fresh.length; + if (fresh.length === 0) continue; + staged += stageTxns( + db, + o.stream, + { + seq: seg.seq, + replicaId: seg.replicaId, + replicaSeq: seg.replicaSeq, + deviceId: seg.deviceId, + schemaVersion: seg.schemaVersion, + txns: fresh, + }, + nowIso, + ); + } + db.prepare( + 'INSERT INTO _sync_cursor (stream, cursor_json, updated_at) VALUES (?, ?, ?) ' + + 'ON CONFLICT (stream) DO UPDATE SET cursor_json = excluded.cursor_json, updated_at = excluded.updated_at', + ).run(o.stream, JSON.stringify(page.cursor), nowIso); + }); + cursor = page.cursor; + if (page.segments.length === 0 || cursor.after >= page.head) break; + } + const apply = applyStagedTxns(db, { + scope: o.scope, + stream: o.stream, + replica: o.replica, + now, + seal: o.seal, + ...(o.apply ?? {}), + }); + return { + stream: o.stream, + segments, + vaultDeltas, + staged, + redelivered, + after: cursor.after, + head, + apply, + }; +} diff --git a/packages/core/src/store/table-classification.ts b/packages/core/src/store/table-classification.ts index 4bdc8ac176..96861a961c 100644 --- a/packages/core/src/store/table-classification.ts +++ b/packages/core/src/store/table-classification.ts @@ -196,6 +196,18 @@ const PROJECT_TABLES: Readonly> = { source: 'journal spec t12342-t12343-journal-design §2.8 (T12343)', note: 'the transactions each persisted segment carries, in order. This device only. Created lazily by store/sync/schema.ts when a sync.* flag is first enabled (T12343)', }, + _sync_cursor: { + class: 'local-only', + status: 'optional-transient', + source: 'journal spec t12342-t12343-journal-design §3.1 Receive (T12343)', + note: "where this store's pull stands per stream: the PullCursor the journal client verifies the next page against, advanced in the transaction that stages what it covers. This device only. Created lazily by store/sync/schema.ts when a sync.* flag is first enabled (T12343)", + }, + _sync_seen_txn: { + class: 'local-only', + status: 'optional-transient', + source: 'journal spec t12342-t12343-journal-design §3.1 Receive (T12343 S5-1)', + note: 'every transaction id staged from a stream, so a re-delivered transaction is never staged twice (a counter delta would double). This device only. Created lazily by store/sync/schema.ts when a sync.* flag is first enabled (T12343)', + }, _sync_row_meta: { class: 'local-only', status: 'optional-transient', @@ -1430,6 +1442,18 @@ const GLOBAL_TABLES: Readonly> = { source: 'journal spec t12342-t12343-journal-design §2.8 (T12343)', note: 'the transactions each persisted segment carries, in order. This device only. Created lazily by store/sync/schema.ts when a sync.* flag is first enabled (T12343)', }, + _sync_cursor: { + class: 'local-only', + status: 'optional-transient', + source: 'journal spec t12342-t12343-journal-design §3.1 Receive (T12343)', + note: "where this store's pull stands per stream: the PullCursor the journal client verifies the next page against, advanced in the transaction that stages what it covers. This device only. Created lazily by store/sync/schema.ts when a sync.* flag is first enabled (T12343)", + }, + _sync_seen_txn: { + class: 'local-only', + status: 'optional-transient', + source: 'journal spec t12342-t12343-journal-design §3.1 Receive (T12343 S5-1)', + note: 'every transaction id staged from a stream, so a re-delivered transaction is never staged twice (a counter delta would double). This device only. Created lazily by store/sync/schema.ts when a sync.* flag is first enabled (T12343)', + }, _sync_row_meta: { class: 'local-only', status: 'optional-transient', diff --git a/scripts/lint-no-raw-table-writes.mjs b/scripts/lint-no-raw-table-writes.mjs index 2fb3fcbf73..32872db65f 100644 --- a/scripts/lint-no-raw-table-writes.mjs +++ b/scripts/lint-no-raw-table-writes.mjs @@ -396,6 +396,18 @@ export const EXEMPT = { 'capture undo (local-only): dropped when a failed or raced genesis cut is undone and no stream is cut, so undo is off again (T13296)', }, }, + 'packages/core/src/store/sync/pull.ts': { + _sync_cursor: { + count: 1, + reason: + 'pull cursor (local-only): advanced in the transaction that stages the segments it covers (§3.1 step 3, T12343)', + }, + _sync_seen_txn: { + count: 1, + reason: + 'staged transaction ids (local-only): a re-delivered transaction is never staged twice (§3.1, T12343 S5-1)', + }, + }, 'packages/core/src/store/sync/push.ts': { _sync_meta: { count: 2, From 0dbc899e662735a80a4320c966b3e627b18f7fc2 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 18:40:15 -0700 Subject: [PATCH 10/23] fix(sync): pull refuses while sync.pull is off (T12343 S5-1) pullStream ran regardless of the sync.pull flag; it now refuses (refused: 'sync.pull is off', nothing pulled or staged) unless the flag is on, as push does for sync.push. PullStreamReport gains `refused` and its apply report is null on a refusal. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-pull.md | 3 ++- .../src/cloud/__tests__/nexus-vault.test.ts | 5 ++++- .../src/store/sync/__tests__/pull.test.ts | 17 +++++++++++++- packages/core/src/store/sync/pull.ts | 22 ++++++++++++++++++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.changeset/t12343-pull.md b/.changeset/t12343-pull.md index f5af7912e9..5a4531ccc5 100644 --- a/.changeset/t12343-pull.md +++ b/.changeset/t12343-pull.md @@ -7,7 +7,8 @@ summary: Sync pull - stage and apply a stream's verified segments, never a trans A store can now pull its stream's journal (journal spec §3.1, §2.8; T12343 S5-1). -- **`pullStream(db, …)`** (`store/sync/pull.ts`) pages through the stream after the store's persisted cursor. For each page, in one +- **`pullStream(db, …)`** (`store/sync/pull.ts`) refuses while `sync.pull` is off. Otherwise it pages through the stream after the + store's persisted cursor. For each page, in one local transaction: - **Decode:** `deflate-raw` canonical JSON of `LedgerTxn[]`, parsed with the ledger schema. A vault count-delta segment (`cleo-vault-delta/v1`) carries no ops and is passed over. diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index f95aa138dd..9de695d88f 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5138,10 +5138,13 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { }); const pushed = await on(m, () => pushSyncStream(vopts(m, { allowUnreleased: true }))); expect(pushed.pushed).toBe(1); + await on(m, async () => { + setSyncFlag(await storeOf(dbPath), 'sync.pull', true, { allowUnreleased: true }); + }); const pulled = await on(m, () => pullSyncStream(vopts(m))); expect(pulled).toMatchObject({ segments: 1, staged: 1, redelivered: 0, vaultDeltas: 0 }); expect(pulled.after).toBe(pushed.lastServerSeq); - expect(pulled.apply.applied).toBe(1); + expect(pulled.apply?.applied).toBe(1); await on(m, async () => { const db = await storeOf(dbPath); // The own echo was sequenced (§3.5): its transaction is no longer unsequenced. diff --git a/packages/core/src/store/sync/__tests__/pull.test.ts b/packages/core/src/store/sync/__tests__/pull.test.ts index a33e1cdfbb..d6e74a6f91 100644 --- a/packages/core/src/store/sync/__tests__/pull.test.ts +++ b/packages/core/src/store/sync/__tests__/pull.test.ts @@ -63,6 +63,7 @@ async function store(name: string): Promise { ); setCaptureEnabled(db, 'project', true, { schemaRoot: SYNC_SCHEMA }); setSyncFlag(db, 'sync.seal', true, { schemaRoot: SYNC_SCHEMA, allowUnreleased: true }); + setSyncFlag(db, 'sync.pull', true, { schemaRoot: SYNC_SCHEMA, allowUnreleased: true }); return db; } @@ -148,6 +149,7 @@ const pullOpts = (b: DatabaseSync, stream: ReturnType) => ({ deviceId === DEV_A ? firstBadTxnSignature(KEY_A.publicKey, STREAM, txns) : 0, initialCursor: START, now: () => ++clock, + env: {}, seal: seal(b, RB), }); @@ -167,7 +169,7 @@ describe('pullStream (S5-1)', () => { stream.append(authorSegment(a), 2); const r = await pullStream(b, pullOpts(b, stream)); expect(r).toMatchObject({ segments: 3, staged: 3, redelivered: 0, after: 3, head: 3 }); - expect(r.apply.applied).toBe(3); + expect(r.apply?.applied).toBe(3); expect(b.prepare("SELECT priority FROM tasks_tasks WHERE id = 'T1'").get()).toEqual({ priority: 'high', }); @@ -228,4 +230,17 @@ describe('pullStream (S5-1)', () => { await expect(pullStream(b, pullOpts(b, stream))).rejects.toThrow(/not a ledger segment/); expect(readStreamCursor(b, STREAM)).toBeNull(); }); + + it('refuses with sync.pull off: nothing pulled, nothing staged', async () => { + const a = await store('a'); + const b = await store('b'); + const stream = fakeStream(); + write(a, addTask('T1')); + stream.append(authorSegment(a), 0); + setSyncFlag(b, 'sync.pull', false, { schemaRoot: SYNC_SCHEMA }); + const r = await pullStream(b, pullOpts(b, stream)); + expect(r).toMatchObject({ refused: 'sync.pull is off', segments: 0, apply: null }); + expect(stream.pages).toBe(0); + expect(n(b, 'SELECT count(*) AS n FROM _sync_inbox')).toBe(0); + }); }); diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts index ab57f1e084..2f19fa7f8f 100644 --- a/packages/core/src/store/sync/pull.ts +++ b/packages/core/src/store/sync/pull.ts @@ -36,6 +36,7 @@ import type { TableScope } from '@cleocode/contracts'; import { LedgerTxn } from '@cleocode/contracts/ledger'; import { type ApplyReport, type ApplyStagedOptions, applyStagedTxns } from './apply/applier.js'; import { withImmediateTransaction } from './clock-store.js'; +import { isSyncFlagOn } from './flags.js'; import { stageTxns } from './inbox.js'; import { hasTable } from './schema.js'; @@ -90,11 +91,15 @@ export interface PullStreamOptions { readonly seal: ApplyStagedOptions['seal']; /** Applier options passed through (pages, budget). */ readonly apply?: Partial>; + /** Environment for the `sync.pull` kill switch. @defaultValue process.env */ + readonly env?: NodeJS.ProcessEnv; } /** What {@link pullStream} did. */ export interface PullStreamReport { readonly stream: string; + /** Why nothing was pulled (`sync.pull` off), or null. */ + readonly refused: string | null; /** Segments received, vault deltas passed over, transactions staged, re-deliveries skipped. */ readonly segments: number; readonly vaultDeltas: number; @@ -103,7 +108,8 @@ export interface PullStreamReport { /** The stream position the store has staged up to, and the server's head. */ readonly after: number; readonly head: number; - readonly apply: ApplyReport; + /** What the applier did, or null when the pull was refused. */ + readonly apply: ApplyReport | null; } /** A segment the store refuses to stage (a malformed body or a bad transaction signature). */ @@ -169,6 +175,19 @@ export async function pullStream( o: PullStreamOptions, ): Promise { const now = o.now ?? Date.now; + if (!isSyncFlagOn(db, 'sync.pull', o.env ?? process.env)) { + return { + stream: o.stream, + refused: 'sync.pull is off', + segments: 0, + vaultDeltas: 0, + staged: 0, + redelivered: 0, + after: readStreamCursor(db, o.stream)?.after ?? o.initialCursor.after, + head: 0, + apply: null, + }; + } let cursor = readStreamCursor(db, o.stream) ?? o.initialCursor; let segments = 0; let vaultDeltas = 0; @@ -239,6 +258,7 @@ export async function pullStream( }); return { stream: o.stream, + refused: null, segments, vaultDeltas, staged, From 300130b72583375af74925291572337b68a46ee9 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 19:11:31 -0700 Subject: [PATCH 11/23] feat(cloud): cleo cloud sync - seal, push, pull and apply each attached stream (T12996) cloudSync (cloud/nexus-vault.ts) runs pushSyncStream then pullSyncStream for each attached stream (the project's and the account's global store, or the one --scope names) and returns one CloudSyncResult: per stream a status (synced, disabled, paused, not-attached, refused) and sealed, sent, received, staged, applied, held and conflict counts with the staged position against the server's head. E_SYNC_DISABLED names `cleo sync enable push` when no attached stream has a journal flag on; pull now refuses on sync.pull before it looks for a position. An interrupted run resumes without sending or applying anything twice. `cleo cloud sync [--scope]` is the CLI verb. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12996-cloud-sync.md | 21 +++ packages/cleo/src/cli/commands/cloud.ts | 21 +++ .../cli/lib/__tests__/nexus-vault-cli.test.ts | 35 +++++ packages/cleo/src/cli/lib/nexus-vault-cli.ts | 27 ++++ packages/contracts/src/index.ts | 2 + packages/contracts/src/nexus-account.ts | 2 + packages/contracts/src/nexus-vault.ts | 37 +++++ .../src/cloud/__tests__/nexus-vault.test.ts | 81 ++++++++++ packages/core/src/cloud/nexus-vault.ts | 142 +++++++++++++++++- 9 files changed, 365 insertions(+), 3 deletions(-) create mode 100644 .changeset/t12996-cloud-sync.md diff --git a/.changeset/t12996-cloud-sync.md b/.changeset/t12996-cloud-sync.md new file mode 100644 index 0000000000..0d2ee6083e --- /dev/null +++ b/.changeset/t12996-cloud-sync.md @@ -0,0 +1,21 @@ +--- +id: t12996-cloud-sync +tasks: [T12996] +kind: feat +summary: "`cleo cloud sync` - seal, push, pull and apply each attached stream in one call, one result per stream" +--- + +`cleo cloud sync [--scope project|global]` runs the change journal end to end for each attached stream: the project's and the +account's global store, or just the one `--scope` names (T12996). + +- **Per stream:** it seals pending writes, pushes them as segments (`pushSyncStream`), pulls the stream's new segments, and applies + them (`pullSyncStream`). +- **One LAFS envelope** (`CloudSyncResult`) reports, per stream: + - `status`: `synced`, `disabled`, `paused` (device clock ahead), `not-attached` or `refused`; + - sealed, built, sent, duplicates, received, staged, redelivered, applied, held and conflict counts; + - the staged position against the server's head. +- **`E_SYNC_DISABLED`** is returned, naming `cleo sync enable push`, when no attached stream has `sync.push` or `sync.pull` on. A store + with `sync.pull` off now refuses its pull before it looks for a pull position. +- **An interrupted run resumes on the next:** a segment is resent with the same bytes, and a transaction is never staged or applied + twice. +- `sync.push` and `sync.pull` stay unreleased: the CLI never opts in, so stores can't turn them on yet. diff --git a/packages/cleo/src/cli/commands/cloud.ts b/packages/cleo/src/cli/commands/cloud.ts index d1bb8f1908..b7c7164848 100644 --- a/packages/cleo/src/cli/commands/cloud.ts +++ b/packages/cleo/src/cli/commands/cloud.ts @@ -41,6 +41,7 @@ import { runCloudPull, runCloudPush, runCloudRestore, + runCloudSync, runCloudVault, runCloudVerify, } from '../lib/nexus-vault-cli.js'; @@ -128,6 +129,25 @@ const projectsSubCommand = defineCommand({ }, }); +const syncSubCommand = defineCommand({ + meta: { + name: 'sync', + description: + "Sync the change journal with Cleo Nexus: seal pending writes, push them as segments, pull the stream's new segments and apply them, for each attached stream (the project's and the account's global store; --scope picks one). One envelope with per-stream sealed, sent, received, applied, held and conflict counts and the staged position against the server's head. Refused with E_SYNC_DISABLED until `cleo sync enable push`. An interrupted run resumes on the next: nothing is sent or applied twice.", + }, + args: { + scope: { + type: 'string', + description: "One stream: 'project' or 'global' (default: every attached stream).", + }, + 'api-url': NEXUS_API_URL_ARG, + json: JSON_ARG, + }, + async run({ args }) { + await runCloudSync(args as Record); + }, +}); + const pushSubCommand = defineCommand({ meta: { name: 'push', @@ -311,6 +331,7 @@ export const cloudCommand = defineCommand({ projects: projectsSubCommand, activity: activitySubCommand, conflicts: conflictsSubCommand, + sync: syncSubCommand, push: pushSubCommand, pull: pullSubCommand, restore: restoreSubCommand, diff --git a/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts b/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts index a196417dc1..c8c0cc24c3 100644 --- a/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts +++ b/packages/cleo/src/cli/lib/__tests__/nexus-vault-cli.test.ts @@ -13,6 +13,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const pushNexusVault = vi.fn(); const enableSyncPush = vi.fn(); +const cloudSync = vi.fn(); const restoreNexusVault = vi.fn(); const verifyNexusVault = vi.fn(); const nexusVaultStatus = vi.fn(); @@ -23,6 +24,7 @@ const resolveNexusProjectRef = vi.fn(); const assertNexusRestoreTarget = vi.fn(); vi.mock('@cleocode/core/cloud/nexus-vault.js', () => ({ + cloudSync, enableSyncPush, pushNexusVault, restoreNexusVault, @@ -46,6 +48,7 @@ const { runCloudPull, runCloudPush, runCloudRestore, + runCloudSync, runCloudVault, runCloudVerify, runSyncEnablePush, @@ -177,6 +180,38 @@ describe('flags reach the core calls', () => { expect(written()).toContain('cp-1'); }); + it('cloud sync passes the API URL, and a scope only when one is given (T12996)', async () => { + cloudSync.mockReset().mockResolvedValue({ + apiUrl: API, + streams: [ + { + scope: 'project', + streamId: 'project:p', + status: 'synced', + refused: null, + sealed: 1, + built: 1, + sent: 1, + duplicates: 0, + received: 2, + staged: 2, + redelivered: 0, + applied: 2, + held: 0, + conflicts: 0, + after: 7, + head: 7, + }, + ], + warnings: [], + }); + await runCloudSync({ 'api-url': API }); + expect(opts(cloudSync)).toEqual({ apiUrl: API }); + await runCloudSync({ scope: 'global' }); + expect(cloudSync.mock.calls[1]?.[0]).toEqual({ apiUrl: undefined, scope: 'global' }); + expect(written()).toContain('project:p'); + }); + it('restore passes checkpoint, project, into and force', async () => { await runCloudRestore({ scope: 'project', diff --git a/packages/cleo/src/cli/lib/nexus-vault-cli.ts b/packages/cleo/src/cli/lib/nexus-vault-cli.ts index b2c25cb40f..ba638f3b85 100644 --- a/packages/cleo/src/cli/lib/nexus-vault-cli.ts +++ b/packages/cleo/src/cli/lib/nexus-vault-cli.ts @@ -18,6 +18,7 @@ import type { CloudPushResult, CloudRestoreResult, CloudSyncPushEnableResult, + CloudSyncResult, CloudVaultScope, CloudVaultStatusResult, CloudVerifyResult, @@ -106,6 +107,32 @@ export async function runSyncEnablePush(args: Args): Promise { ); } +/** + * `cleo cloud sync [--scope]`: seal, push, pull and apply each attached + * stream (T12996). Without `--scope`, every attached stream. + * + * @param args - Parsed args. + */ +export async function runCloudSync(args: Args): Promise { + const scope = stringArg(args, 'scope') === undefined ? undefined : scopeArg(args, 'cloud.sync'); + await runCloudRead( + 'cloud.sync', + async () => + (await vaultModule()).cloudSync({ + apiUrl: nexusApiUrlArg(args), + ...(scope !== undefined ? { scope } : {}), + }), + (r) => + r.streams + .map((st) => + st.status === 'synced' + ? `${st.streamId}: sent ${st.sent} segment(s), received ${st.received}, applied ${st.applied}${st.held > 0 ? `, ${st.held} held` : ''}${st.conflicts > 0 ? `, ${st.conflicts} in conflict` : ''} (at ${st.after} of ${st.head}).` + : `${st.streamId ?? st.scope}: ${st.status}${st.refused ? ` (${st.refused})` : ''}.`, + ) + .join('\n'), + ); +} + /** * `cleo cloud pull [--scope] [--force]`. * diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 2518c8c505..77502e08f4 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -3137,6 +3137,8 @@ export { type CloudPushResult, type CloudRestoreResult, type CloudSyncPushEnableResult, + type CloudSyncResult, + type CloudSyncStreamResult, type CloudVaultLease, type CloudVaultScope, type CloudVaultSnapshot, diff --git a/packages/contracts/src/nexus-account.ts b/packages/contracts/src/nexus-account.ts index f5ef38f2e8..fbc3b384fa 100644 --- a/packages/contracts/src/nexus-account.ts +++ b/packages/contracts/src/nexus-account.ts @@ -106,6 +106,8 @@ export const NEXUS_ACCOUNT_ERROR_CODES = [ 'E_NEXUS_SYNC_REFUSED', /** Another device already started this stream's change journal: this store joins it by pulling. */ 'E_NEXUS_SYNC_STREAM_JOURNALED', + /** `cleo cloud sync` found no stream with `sync.push` or `sync.pull` on (`cleo sync enable`). */ + 'E_SYNC_DISABLED', // Projects by name (`cleo cloud restore `, T13102). /** No project of the account has that name, label or id. */ 'E_NEXUS_PROJECT_NOT_FOUND', diff --git a/packages/contracts/src/nexus-vault.ts b/packages/contracts/src/nexus-vault.ts index cf2ab80b25..d357991ef6 100644 --- a/packages/contracts/src/nexus-vault.ts +++ b/packages/contracts/src/nexus-vault.ts @@ -204,6 +204,43 @@ export interface CloudSyncPushEnableResult { warnings: CloudWarning[]; } +/** One stream of `cleo cloud sync` (T12996). */ +export interface CloudSyncStreamResult { + scope: CloudVaultScope; + streamId: string | null; + /** + * `synced`: pushed and pulled what was enabled; `disabled`: neither `sync.push` nor `sync.pull` is on; + * `paused`: push paused for a device clock ahead of the server's; `not-attached`: this machine has no + * store on that stream; `refused`: a precondition or the server refused (see `refused`). + */ + status: 'synced' | 'disabled' | 'paused' | 'not-attached' | 'refused'; + /** Why this stream did not sync fully, or null. */ + refused: string | null; + /** Transactions sealed, segments persisted, segments the server stored (`duplicates` of them retries). */ + sealed: number; + built: number; + sent: number; + duplicates: number; + /** Segments received, transactions staged, re-deliveries skipped. */ + received: number; + staged: number; + redelivered: number; + /** Transactions applied; still held (pending, clock skew, or a newer schema); in conflict. */ + applied: number; + held: number; + conflicts: number; + /** The stream position this store has staged up to, and the server's head (null when not pulled). */ + after: number | null; + head: number | null; +} + +/** `cleo cloud sync` (T12996): seal, push, pull and apply each attached stream. */ +export interface CloudSyncResult { + apiUrl: string; + streams: CloudSyncStreamResult[]; + warnings: CloudWarning[]; +} + /** `cleo cloud pull` / `cleo cloud restore`. */ export interface CloudRestoreResult { apiUrl: string; diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 9de695d88f..78ff68c5c8 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -118,6 +118,7 @@ import { hasUnsyncedNexusBackup, runNexusFirstRun } from '../nexus-first-run.js' import { linkProjectToNexus } from '../nexus-link.js'; import { listNexusNamedProjects, resolveNexusProjectRef } from '../nexus-project-names.js'; import { + cloudSync, enableSyncPush, nexusVaultStatus, pullSyncStream, @@ -5156,6 +5157,86 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { expect(again).toMatchObject({ segments: 0, staged: 0 }); }); + it('C-1: cloud sync seals, pushes, pulls and applies the stream in one call; a rerun does nothing', async () => { + const { m, dbPath } = await journalMachine(); + await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + await on(m, async () => { + const db = await storeOf(dbPath); + setSyncFlag(db, 'sync.pull', true, { allowUnreleased: true }); + db.exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('C1', 'title C1', 'task', 'pending', 'medium', 'uid-C1', 'fp-C1')", + ); + }); + const r = await on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))); + expect(r.streams).toHaveLength(1); + expect(r.streams[0]).toMatchObject({ + scope: 'project', + streamId: STREAM, + status: 'synced', + refused: null, + sealed: 1, + sent: 1, + received: 1, + staged: 1, + applied: 1, + held: 0, + conflicts: 0, + }); + expect(r.streams[0]?.after).toBe(r.streams[0]?.head); + const again = await on(m, () => + cloudSync(vopts(m, { scope: 'project', allowUnreleased: true })), + ); + expect(again.streams[0]).toMatchObject({ status: 'synced', sealed: 0, sent: 0, received: 0 }); + }); + + it('C-1: an interrupted sync resumes on the next run with nothing sent or applied twice', async () => { + const { m, dbPath } = await journalMachine(); + await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); + await on(m, async () => { + const db = await storeOf(dbPath); + setSyncFlag(db, 'sync.pull', true, { allowUnreleased: true }); + db.exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('C2', 'title C2', 'task', 'pending', 'medium', 'uid-C2', 'fp-C2')", + ); + }); + fake.beforeSegment = { + deviceId: DEVICE_A, + run: async () => { + throw new ApiFail(403, 'E_FORBIDDEN'); + }, + }; + await expect( + on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))), + ).rejects.toThrow(); + const before = fake.stream(STREAM).segments.length; + const r = await on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))); + expect(r.streams[0]).toMatchObject({ + status: 'synced', + built: 0, + sent: 1, + received: 1, + applied: 1, + }); + expect(fake.stream(STREAM).segments).toHaveLength(before + 1); + await on(m, async () => { + const db = await storeOf(dbPath); + expect((db.prepare('SELECT count(*) AS n FROM _sync_inbox').get() as { n: number }).n).toBe( + 1, + ); + expect( + (db.prepare("SELECT count(*) AS n FROM tasks_tasks WHERE id = 'C2'").get() as { n: number }) + .n, + ).toBe(1); + }); + }); + + it('C-1: with the journal flags off, cloud sync refuses with E_SYNC_DISABLED naming the remedy', async () => { + const { m } = await journalMachine(); + const refused = await failure(on(m, () => cloudSync(vopts(m, { scope: 'project' })))); + expect(refused.code).toBe('E_SYNC_DISABLED'); + expect(refused.fix).toContain('cleo sync enable push'); + }); + it('S4-2: a device clock ahead of the server pauses push', async () => { const { m, dbPath } = await journalMachine(); await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 52f156a309..b9e88a0eb1 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -62,6 +62,8 @@ import type { CloudPushResult, CloudRestoreResult, CloudSyncPushEnableResult, + CloudSyncResult, + CloudSyncStreamResult, CloudVaultLease, CloudVaultScope, CloudVaultSnapshot, @@ -103,7 +105,7 @@ import { sha256File, } from '../store/portable-bundle-scan.js'; import { FIRST_OPEN_LOCK_SUFFIX } from '../store/sqlite.js'; -import { UNRELEASED_FLAGS } from '../store/sync/flags.js'; +import { isSyncFlagOn, UNRELEASED_FLAGS } from '../store/sync/flags.js'; import { completeGenesis, cutGenesisWithSnapshot, @@ -1852,7 +1854,8 @@ async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise< // (the genesis it cut, or the snapshot it restored), seeded from its signed // replica map. let initialCursor = readStreamCursor(db, t.streamId); - if (initialCursor === null) { + // With sync.pull off, pullStream refuses before it needs a position. + if (initialCursor === null && isSyncFlagOn(db, 'sync.pull')) { const head = await streamHead(conn, t.streamId); const checkpoints = head.headCheckpointId ? await listCheckpoints(conn, t.streamId) : []; const { state: synced } = syncedState(conn, t, checkpoints, head.headCheckpointId); @@ -1871,7 +1874,7 @@ async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise< scope: tableScopeOf(t), stream: t.streamId, replica: replicaId, - initialCursor, + initialCursor: initialCursor ?? initialPullCursor(), pull: async (cursor) => { const page = await journal.pull( { @@ -1909,6 +1912,120 @@ async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise< }); } +/** A `cleo cloud sync` stream result with nothing done yet. */ +const syncStreamResult = ( + scope: CloudVaultScope, + fields: Partial, +): CloudSyncStreamResult => ({ + scope, + streamId: null, + status: 'synced', + refused: null, + sealed: 0, + built: 0, + sent: 0, + duplicates: 0, + received: 0, + staged: 0, + redelivered: 0, + applied: 0, + held: 0, + conflicts: 0, + after: null, + head: null, + ...fields, +}); + +/** Codes meaning this machine has no store on the stream of a scope. */ +const NOT_ATTACHED_CODES: ReadonlySet = new Set([ + 'E_NEXUS_VAULT_NOT_LINKED', + 'E_NEXUS_NOT_A_PROJECT', +]); + +/** Seal, push, pull and apply one scope's stream (T12996). */ +async function syncOneStream( + opts: NexusVaultCommandOptions & { allowUnreleased?: boolean }, + scope: CloudVaultScope, +): Promise { + const failed = (err: unknown): CloudSyncStreamResult | null => { + if (!(err instanceof NexusAccountError)) return null; + return NOT_ATTACHED_CODES.has(err.code) + ? syncStreamResult(scope, { status: 'not-attached', refused: err.message }) + : syncStreamResult(scope, { status: 'refused', refused: `${err.code}: ${err.message}` }); + }; + let push: PushStreamReport; + try { + push = await pushSyncStreamImpl({ ...opts, scope }); + } catch (err) { + const r = failed(err); + if (r) return r; + throw err; + } + let pull: PullStreamReport; + try { + pull = await pullSyncStreamImpl({ ...opts, scope }); + } catch (err) { + const r = failed(err); + if (r) + return { + ...r, + streamId: push.stream, + sealed: push.sealed, + built: push.built, + sent: push.pushed, + }; + throw err; + } + const pushOff = push.refused === 'sync.push is off'; + const pullOff = pull.refused === 'sync.pull is off'; + const refused = [push.refused, pull.refused].filter( + (r): r is string => r !== null && r !== 'sync.push is off' && r !== 'sync.pull is off', + ); + const apply = pull.apply; + return syncStreamResult(scope, { + streamId: push.stream, + status: + pushOff && pullOff + ? 'disabled' + : refused.length > 0 + ? 'refused' + : push.clockAhead + ? 'paused' + : 'synced', + refused: refused.length > 0 ? refused.join('; ') : null, + sealed: push.sealed, + built: push.built, + sent: push.pushed, + duplicates: push.duplicates, + received: pull.segments, + staged: pull.staged, + redelivered: pull.redelivered, + applied: apply?.applied ?? 0, + held: (apply?.pending ?? 0) + (apply?.heldSkew ?? 0) + (apply?.refusedSchema ?? 0), + conflicts: apply?.conflict ?? 0, + after: pull.refused === null ? pull.after : null, + head: pull.refused === null ? pull.head : null, + }); +} + +async function cloudSyncImpl( + opts: NexusVaultCommandOptions & { allowUnreleased?: boolean } = {}, +): Promise { + const conn = await connectNexusVault(opts); + const scopes: CloudVaultScope[] = opts.scope ? [opts.scope] : ['project', 'global']; + const streams: CloudSyncStreamResult[] = []; + for (const scope of scopes) streams.push(await syncOneStream(opts, scope)); + const worked = streams.filter((r) => r.status !== 'disabled' && r.status !== 'not-attached'); + if (worked.length === 0 && streams.some((r) => r.status === 'disabled')) { + throw vaultError( + 'E_SYNC_DISABLED', + 'no attached stream has sync.push or sync.pull on', + 'turn the change journal on first: `cleo sync enable push`', + ); + } + return { apiUrl: conn.apiUrl, streams, warnings: [...conn.warnings] }; +} + /** The keyed hash of an empty table (a table the parent lists that this store no longer has). */ function buildEmptyHash(t: VaultTarget, table: string): string { return emptyVaultTableHash(hashKeyOf(t.dataKey), table); @@ -2778,6 +2895,25 @@ export function pullSyncStream(opts: NexusVaultCommandOptions = {}): Promise pullSyncStreamImpl(opts)); } +/** + * `cleo cloud sync`: seal, push, pull and apply the change journal of each + * attached stream (the project's `project:`, the account's `home:`; + * T12996). One result per stream: sealed, persisted, sent, received, staged, + * applied, held and in conflict, and how far the store has staged against + * the server's head. An interrupted run resumes on the next: a segment is + * resent with the same bytes, and a transaction is never staged twice. + * + * @param opts - `scope` (one stream; default every attached one), overrides; + * `allowUnreleased` for tests and staging only. + * @returns Every stream's result. + * @throws {NexusAccountError} `E_SYNC_DISABLED` when no attached stream has a journal flag on. + */ +export function cloudSync( + opts: NexusVaultCommandOptions & { allowUnreleased?: boolean } = {}, +): Promise { + return mapped(() => cloudSyncImpl(opts)); +} + /** * `cleo cloud pull` / `cleo cloud restore`: verify and activate a snapshot. * From 4bd20cd28b436d8cc366352015a51236c7e60799 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 19:43:00 -0700 Subject: [PATCH 12/23] fix(sync): pull refuses to cross a journal genesis and applies what precedes a refused segment (T13306, T13307) review-hotfix on #1959: - T13306: a store whose last synced checkpoint is a vault snapshot from before the stream's journal genesis was seeded from it and pulled across the genesis, missing everything folded into it: silent divergence. It is now refused, nothing pulled, with the remedy to restore the journal checkpoint and join with `cleo sync enable push` (T12999); #1955's journaled-stream remedy says the same instead of "join by pulling". - T13307: a refused segment threw before apply, stranding earlier staged pages and sticking every retry. The pull now stops at the refused segment: what precedes it (in the same page too) is staged and applied, the cursor stops just before it, and the report's `refused` names its seq, replica and device. - Pruning: _sync_seen_txn rows first staged at or below the latest verified checkpoint's coversSeq (capped at the store's own position) are dropped at the start of each pull. - Nit: with sync.pull off, pullSyncStream refuses before any network call. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-pull.md | 12 +- packages/contracts/src/nexus-account.ts | 5 +- .../src/cloud/__tests__/nexus-vault.test.ts | 35 ++++++ packages/core/src/cloud/nexus-vault.ts | 40 +++++- .../src/store/sync/__tests__/pull.test.ts | 49 ++++++-- packages/core/src/store/sync/pull.ts | 115 ++++++++++++++---- scripts/lint-no-raw-table-writes.mjs | 4 +- 7 files changed, 219 insertions(+), 41 deletions(-) diff --git a/.changeset/t12343-pull.md b/.changeset/t12343-pull.md index 5a4531ccc5..a1e2b9cff7 100644 --- a/.changeset/t12343-pull.md +++ b/.changeset/t12343-pull.md @@ -1,6 +1,6 @@ --- id: t12343-pull -tasks: [T12343] +tasks: [T12343, T13306, T13307] kind: feat summary: Sync pull - stage and apply a stream's verified segments, never a transaction twice (S5-1) --- @@ -13,14 +13,20 @@ A store can now pull its stream's journal (journal spec §3.1, §2.8; T12343 S5- - **Decode:** `deflate-raw` canonical JSON of `LedgerTxn[]`, parsed with the ledger schema. A vault count-delta segment (`cleo-vault-delta/v1`) carries no ops and is passed over. - **Verify:** every transaction's signature is checked against the segment's device key, before anything of the page is written. - A bad signature or a malformed body refuses the segment (`SegmentRefusedError`). + A bad signature or a malformed body refuses the segment. The pull stops there: what precedes it is staged and applied, the + cursor stops just before it, and the report's `refused` names its seq, replica and device (T13307). - **Deduplicate:** every transaction id already staged from the stream is skipped (`_sync_seen_txn`), so a re-delivered transaction is never staged or applied twice and a counter delta never doubles. - **Stage and advance:** the rest are staged, and the cursor advances (`_sync_cursor`). Then the applier applies, in stream order, own echoes included (they feed the fast path and the scoped rebase). +- **`_sync_seen_txn` stays bounded.** Rows first staged at or below the latest verified checkpoint's coversSeq (capped at the + store's own position) are pruned at the start of each pull (`pruneSeenTxns`). - **`pullSyncStream(opts)`** (`cloud/nexus-vault.ts`) wires `pullStream` to the journal client and the trusted device keys. It refuses - a store bound to another replica than the link. A store with no pull position starts after the checkpoint it last synced (its + a store bound to another replica than the link. With `sync.pull` off it refuses before any network call. A store whose last synced checkpoint is a vault snapshot from + before the stream's journal genesis is refused, with nothing pulled (T13306). The changes folded into that genesis never travel + as segments, so pulling across it would silently diverge. The remedy is to restore the journal checkpoint, then join with + `cleo sync enable push`. A store with no pull position starts after the checkpoint it last synced (its genesis, or the snapshot it restored), seeded from that checkpoint's signed replica map. - **New sync-journal migration** `t12343-pull-cursor`, local-only: `_sync_cursor` and `_sync_seen_txn`. Both are cleared from vault bundles, so a restored store resumes from its own checkpoint. diff --git a/packages/contracts/src/nexus-account.ts b/packages/contracts/src/nexus-account.ts index f5ef38f2e8..443e9c7ffa 100644 --- a/packages/contracts/src/nexus-account.ts +++ b/packages/contracts/src/nexus-account.ts @@ -104,7 +104,10 @@ export const NEXUS_ACCOUNT_ERROR_CODES = [ // Change-journal push (`cleo sync enable push`, T12343 S4-1b). /** The store is not fit to reach genesis (a precondition failed); nothing was cut. */ 'E_NEXUS_SYNC_REFUSED', - /** Another device already started this stream's change journal: this store joins it by pulling. */ + /** + * Another device already started this stream's change journal: this store restores its journal + * checkpoint, then joins the stream (T12999), never writing a second genesis. + */ 'E_NEXUS_SYNC_STREAM_JOURNALED', // Projects by name (`cleo cloud restore `, T13102). /** No project of the account has that name, label or id. */ diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 9de695d88f..5d16ca7140 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5156,6 +5156,41 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { expect(again).toMatchObject({ segments: 0, staged: 0 }); }); + it('S5-1: a store that synced a vault snapshot from before the journal genesis refuses to pull across it (T13306)', async () => { + const { m: a, dbPath } = await journalMachine(); + await on(a, () => pushNexusVault(vopts(a))); // a vault snapshot, before any journal + // B restores that vault snapshot and is attached under its own bound replica. + const b = await machine('b', DEVICE_B, REPLICA_B); + fake.replicas.get(REMOTE_PROJECT)?.set(REPLICA_B, DEVICE_B); + await restoreOntoB(b); + const bDb = path.join(b.root, '.cleo', 'cleo.db'); + const bound = await on(b, async () => { + const db = await storeOf(bDb); + setSyncFlag(db, 'sync.pull', true, { allowUnreleased: true }); + return ensureProjectReplica(db, { dbPath: bDb, mode: 'live' }).replicaId; + }); + b.replicaId = bound; + fake.replicas.get(REMOTE_PROJECT)?.set(bound, DEVICE_B); + link(b); + // A changes the store, then starts the journal: that change is folded into the genesis, never a segment. + await on(a, async () => { + (await storeOf(dbPath)).exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('G1', 'title G1', 'task', 'pending', 'medium', 'uid-G1', 'fp-G1')", + ); + }); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + const refused = await failure(on(b, () => pullSyncStream(vopts(b)))); + expect(refused.code).toBe('E_NEXUS_SYNC_REFUSED'); + expect(refused.message).toMatch(/started its change journal .* would silently diverge/); + expect(refused.fix).toMatch(/restore the journal checkpoint first .*cleo sync enable push/); + await on(b, async () => { + const db = await storeOf(bDb); + expect((db.prepare('SELECT count(*) AS n FROM _sync_inbox').get() as { n: number }).n).toBe( + 0, + ); + }); + }); + it('S4-2: a device clock ahead of the server pauses push', async () => { const { m, dbPath } = await journalMachine(); await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 52f156a309..99617058ac 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -103,7 +103,7 @@ import { sha256File, } from '../store/portable-bundle-scan.js'; import { FIRST_OPEN_LOCK_SUFFIX } from '../store/sqlite.js'; -import { UNRELEASED_FLAGS } from '../store/sync/flags.js'; +import { isSyncFlagOn, UNRELEASED_FLAGS } from '../store/sync/flags.js'; import { completeGenesis, cutGenesisWithSnapshot, @@ -1568,7 +1568,7 @@ async function enableSyncPushImpl( throw vaultError( 'E_NEXUS_SYNC_STREAM_JOURNALED', `${t.streamId} already carries the change journal: its head snapshot ${parent?.checkpointId} is a journal checkpoint`, - 'nothing was cut; this store joins the journal by pulling it (`cleo cloud sync`, once pull ships)', + 'nothing was cut; restore the journal checkpoint (`cleo cloud restore`), then join the stream with `cleo sync enable push` (T12999)', ); } const { state: synced } = syncedState(conn, t, parent ? [parent] : [], head.headCheckpointId); @@ -1848,13 +1848,25 @@ async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise< 'nothing was pulled; relink the project (`cleo project link`) so both name one replica', ); } + // With sync.pull off, pullStream refuses at once: no network round trip. + if (!isSyncFlagOn(db, 'sync.pull')) { + return pullStream(db, { + scope: tableScopeOf(t), + stream: t.streamId, + replica: replicaId, + initialCursor: initialPullCursor(), + pull: async () => ({ segments: [], cursor: initialPullCursor(), head: 0 }), + verify: () => 0, + seal: () => {}, + }); + } + const head = await streamHead(conn, t.streamId); + const checkpoints = head.headCheckpointId ? await listCheckpoints(conn, t.streamId) : []; // A store with no pull position starts after the checkpoint it last synced // (the genesis it cut, or the snapshot it restored), seeded from its signed // replica map. let initialCursor = readStreamCursor(db, t.streamId); if (initialCursor === null) { - const head = await streamHead(conn, t.streamId); - const checkpoints = head.headCheckpointId ? await listCheckpoints(conn, t.streamId) : []; const { state: synced } = syncedState(conn, t, checkpoints, head.headCheckpointId); const from = checkpoints.find((c) => c.checkpointId === synced?.lastCheckpointId) ?? null; if (from === null) { @@ -1864,14 +1876,34 @@ async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise< 'restore the stream (`cleo cloud restore`), or enable push (`cleo sync enable push`), first', ); } + // T13306: a vault snapshot from before the stream's journal genesis. What + // the genesis device changed between that snapshot and its cut was folded + // into the genesis checkpoint, never sent as segments: pulling across it + // would silently diverge. + const genesis = isJournalSnapshot(from) + ? undefined + : checkpoints.find((c) => isJournalSnapshot(c) && c.coversSeq >= from.coversSeq); + if (genesis) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `${t.streamId} started its change journal at checkpoint ${genesis.checkpointId}, after the vault snapshot ${from.checkpointId} this store last synced: the changes folded into that journal checkpoint never travel as segments, so pulling from here would silently diverge`, + 'nothing was pulled; restore the journal checkpoint first (`cleo cloud restore`), then join the stream with `cleo sync enable push` (T12999)', + ); + } journal.verifyCheckpoint(from, key.signers); initialCursor = cursorFromCheckpoint(from); } + // Seen-txn rows past the latest verified checkpoint can no longer be re-delivered. + const headCheckpoint = checkpoints.find((c) => c.checkpointId === head.headCheckpointId) ?? null; + if (headCheckpoint) journal.verifyCheckpoint(headCheckpoint, key.signers); + const pruneSeenUpTo = + headCheckpoint === null ? null : Math.min(headCheckpoint.coversSeq, initialCursor.after); return pullStream(db, { scope: tableScopeOf(t), stream: t.streamId, replica: replicaId, initialCursor, + pruneSeenUpTo, pull: async (cursor) => { const page = await journal.pull( { diff --git a/packages/core/src/store/sync/__tests__/pull.test.ts b/packages/core/src/store/sync/__tests__/pull.test.ts index d6e74a6f91..ebca26a7c7 100644 --- a/packages/core/src/store/sync/__tests__/pull.test.ts +++ b/packages/core/src/store/sync/__tests__/pull.test.ts @@ -26,7 +26,6 @@ import { type PulledStreamSegment, pullStream, readStreamCursor, - SegmentRefusedError, type StreamCursor, } from '../pull.js'; import { sealPending } from '../sealer.js'; @@ -212,23 +211,53 @@ describe('pullStream (S5-1)', () => { expect(n(b, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'T1'")).toBe(1); }); - it('a transaction its device did not sign refuses the segment: nothing of its page is staged', async () => { + it('a refused segment stops the pull there: what precedes it is applied, the cursor stops before it, a retry re-stages nothing (T13307)', async () => { const a = await store('a'); const b = await store('b'); const stream = fakeStream(); write(a, addTask('T1')); - stream.append(authorSegment(a), 0, RA, 'dev-mallory'); // served under another device's pin - await expect(pullStream(b, pullOpts(b, stream))).rejects.toBeInstanceOf(SegmentRefusedError); - expect(n(b, 'SELECT count(*) AS n FROM _sync_inbox')).toBe(0); - expect(readStreamCursor(b, STREAM)).toBeNull(); + stream.append(authorSegment(a), 0); + write(a, addTask('T2')); + stream.append(authorSegment(a), 1); + write(a, addTask('T3')); + stream.append(authorSegment(a), 2, RA, 'dev-mallory'); // page 2: served under another device's pin + const r = await pullStream(b, pullOpts(b, stream)); + expect(r.refused).toMatch(/segment 3 of replica .*not signed by device dev-mallory/); + expect(r).toMatchObject({ segments: 2, staged: 2, after: 2 }); + expect(r.apply?.applied).toBe(2); + expect(n(b, "SELECT count(*) AS n FROM tasks_tasks WHERE id IN ('T1', 'T2')")).toBe(2); + expect(n(b, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'T3'")).toBe(0); + expect(readStreamCursor(b, STREAM)).toMatchObject({ after: 2 }); + const retry = await pullStream(b, pullOpts(b, stream)); + expect(retry).toMatchObject({ segments: 0, staged: 0, after: 2 }); + expect(retry.refused).toMatch(/segment 3/); + }); + + it('a refused segment mid-page: the segments before it in that page are staged and applied', async () => { + const a = await store('a'); + const b = await store('b'); + const stream = fakeStream(); + write(a, addTask('T1')); + stream.append(authorSegment(a), 0); + stream.append(Buffer.from('not a segment'), 1); // same page as seq 1 + const r = await pullStream(b, pullOpts(b, stream)); + expect(r.refused).toMatch(/segment 2 of replica .*not a ledger segment/); + expect(r).toMatchObject({ segments: 1, staged: 1, after: 1 }); + expect(n(b, "SELECT count(*) AS n FROM tasks_tasks WHERE id = 'T1'")).toBe(1); }); - it('a body that is neither a ledger segment nor a vault delta is refused', async () => { + it('seen-txn rows at or below the prune floor are dropped, later ones kept', async () => { + const a = await store('a'); const b = await store('b'); const stream = fakeStream(); - stream.append(Buffer.from('not a segment'), 0); - await expect(pullStream(b, pullOpts(b, stream))).rejects.toThrow(/not a ledger segment/); - expect(readStreamCursor(b, STREAM)).toBeNull(); + write(a, addTask('T1')); + stream.append(authorSegment(a), 0); + write(a, addTask('T2')); + stream.append(authorSegment(a), 1); + await pullStream(b, pullOpts(b, stream)); + expect(n(b, 'SELECT count(*) AS n FROM _sync_seen_txn')).toBe(2); + await pullStream(b, { ...pullOpts(b, stream), pruneSeenUpTo: 1 }); + expect(b.prepare('SELECT seq FROM _sync_seen_txn').all()).toEqual([{ seq: 2 }]); }); it('refuses with sync.pull off: nothing pulled, nothing staged', async () => { diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts index 2f19fa7f8f..cd66eb154f 100644 --- a/packages/core/src/store/sync/pull.ts +++ b/packages/core/src/store/sync/pull.ts @@ -11,8 +11,9 @@ * correction a vault push appended before the stream's genesis) carries no * ops and is passed over; * - checks every transaction's signature against the segment's device key - * (§2.8); one bad signature refuses the segment, and the pull stops before - * staging anything of it; + * (§2.8); one bad signature refuses the segment, and the pull stops there: + * the segments before it are staged and applied, the segment and what + * follows are not; * - skips every transaction id already staged from this stream * (`_sync_seen_txn`): a re-delivered transaction is never staged twice, so * a counter delta is never applied twice; @@ -93,12 +94,22 @@ export interface PullStreamOptions { readonly apply?: Partial>; /** Environment for the `sync.pull` kill switch. @defaultValue process.env */ readonly env?: NodeJS.ProcessEnv; + /** + * Drop seen-txn rows first staged at or below this stream seq (the latest + * verified checkpoint's coversSeq), or null to keep every row. + */ + readonly pruneSeenUpTo?: number | null; } /** What {@link pullStream} did. */ export interface PullStreamReport { readonly stream: string; - /** Why nothing was pulled (`sync.pull` off), or null. */ + /** + * Why the pull stopped short, or null: `sync.pull` off (nothing pulled), or + * a refused segment (a malformed body or a transaction its device did not + * sign), naming its seq, replica and device. Everything before a refused + * segment was staged and applied, and the cursor stops just before it. + */ readonly refused: string | null; /** Segments received, vault deltas passed over, transactions staged, re-deliveries skipped. */ readonly segments: number; @@ -161,10 +172,61 @@ function decode(seg: PulledStreamSegment): LedgerTxn[] | null { return parsed.data; } +/** A segment's transactions (null for a vault delta), or why it is refused. */ +function refusalOf( + seg: PulledStreamSegment, + verify: TxnVerifier, +): { readonly txns: LedgerTxn[] | null } | string { + let txns: LedgerTxn[] | null; + try { + txns = decode(seg); + } catch (err) { + if (err instanceof SegmentRefusedError) return err.message; + throw err; + } + if (txns !== null) { + const bad = verify(seg.deviceId, txns); + if (bad !== null) { + return `segment ${seg.seq} of replica ${seg.replicaId}: transaction ${bad} is not signed by device ${seg.deviceId}`; + } + } + return { txns }; +} + +/** `cursor` advanced over `segs` (a page cut short by a refused segment). */ +function advance(cursor: StreamCursor, segs: readonly PulledStreamSegment[]): StreamCursor { + const replicas = { ...cursor.replicas }; + for (const s of segs) replicas[s.replicaId] = { deviceId: s.deviceId, replicaSeq: s.replicaSeq }; + return { + after: segs.at(-1)?.seq ?? cursor.after, + knowsAllReplicas: cursor.knowsAllReplicas, + replicas, + }; +} + +/** + * Drop the seen-txn rows of `stream` first staged at or below `seq`: past a + * verified checkpoint's coversSeq no segment re-delivers them (§3.1). Keeps + * `_sync_seen_txn` bounded. + * + * @param db - The store. + * @param stream - The stream. + * @param seq - The floor, inclusive. + * @returns How many rows were dropped. + */ +export function pruneSeenTxns(db: DatabaseSync, stream: string, seq: number): number { + if (!hasTable(db, '_sync_seen_txn')) return 0; + return Number( + db.prepare('DELETE FROM _sync_seen_txn WHERE stream = ? AND seq <= ?').run(stream, seq).changes, + ); +} + /** * Pull and stage a stream's new segments, then apply (module docs). Must run - * outside a transaction. A refused segment stops the pull with - * {@link SegmentRefusedError} after the pages before it were staged. + * outside a transaction. A refused segment stops the pull: everything + * before it is staged and applied, the cursor stops just before it, and the + * report's `refused` names it (T13307), so the next pull reaches it again + * without re-staging anything. * * @param db - The store. * @param o - {@link PullStreamOptions}. @@ -194,24 +256,34 @@ export async function pullStream( let staged = 0; let redelivered = 0; let head = cursor.after; + let refusedSegment: string | null = null; + // Seen-txn rows at or below the floor can no longer be re-delivered. + if (o.pruneSeenUpTo !== undefined && o.pruneSeenUpTo !== null) { + pruneSeenTxns(db, o.stream, o.pruneSeenUpTo); + } for (;;) { const page = await o.pull(cursor); head = page.head; const nowIso = new Date(now()).toISOString(); - // Decode and verify the whole page before writing anything of it. - const decoded = page.segments.map((seg) => { - const txns = decode(seg); - if (txns !== null) { - const bad = o.verify(seg.deviceId, txns); - if (bad !== null) { - // @sync-invariant none:input-shape a transaction its device did not sign is never staged (§2.8) - throw new SegmentRefusedError( - `segment ${seg.seq} of replica ${seg.replicaId}: transaction ${bad} is not signed by device ${seg.deviceId}`, - ); - } + // Decode and verify the page before writing anything of it, up to the + // first segment refused (T13307): what precedes it is staged and applied, + // the cursor stops just before it, and the refusal is reported. + const decoded: Array<{ seg: PulledStreamSegment; txns: LedgerTxn[] | null }> = []; + for (const seg of page.segments) { + const why = refusalOf(seg, o.verify); + if (typeof why === 'string') { + refusedSegment = why; + break; } - return { seg, txns }; - }); + decoded.push({ seg, txns: why.txns }); + } + const pageCursor = + refusedSegment === null + ? page.cursor + : advance( + cursor, + decoded.map((d) => d.seg), + ); withImmediateTransaction(db, () => { const seen = db.prepare( 'INSERT INTO _sync_seen_txn (stream, txn, seq) VALUES (?, ?, ?) ON CONFLICT (stream, txn) DO NOTHING', @@ -243,9 +315,10 @@ export async function pullStream( db.prepare( 'INSERT INTO _sync_cursor (stream, cursor_json, updated_at) VALUES (?, ?, ?) ' + 'ON CONFLICT (stream) DO UPDATE SET cursor_json = excluded.cursor_json, updated_at = excluded.updated_at', - ).run(o.stream, JSON.stringify(page.cursor), nowIso); + ).run(o.stream, JSON.stringify(pageCursor), nowIso); }); - cursor = page.cursor; + cursor = pageCursor; + if (refusedSegment !== null) break; if (page.segments.length === 0 || cursor.after >= page.head) break; } const apply = applyStagedTxns(db, { @@ -258,7 +331,7 @@ export async function pullStream( }); return { stream: o.stream, - refused: null, + refused: refusedSegment, segments, vaultDeltas, staged, diff --git a/scripts/lint-no-raw-table-writes.mjs b/scripts/lint-no-raw-table-writes.mjs index 32872db65f..eb9a9820f5 100644 --- a/scripts/lint-no-raw-table-writes.mjs +++ b/scripts/lint-no-raw-table-writes.mjs @@ -403,9 +403,9 @@ export const EXEMPT = { 'pull cursor (local-only): advanced in the transaction that stages the segments it covers (§3.1 step 3, T12343)', }, _sync_seen_txn: { - count: 1, + count: 2, reason: - 'staged transaction ids (local-only): a re-delivered transaction is never staged twice (§3.1, T12343 S5-1)', + 'staged transaction ids (local-only): a re-delivered transaction is never staged twice, and rows past the latest verified checkpoint are pruned (§3.1, T12343 S5-1)', }, }, 'packages/core/src/store/sync/push.ts': { From 244b21114a153031d3c9bbb7a8e1049321433cc6 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:07:44 -0700 Subject: [PATCH 13/23] fix(cloud): cloud sync isolates streams, shares one session, skips idle legs (T12996 review) review-hotfix LOWs on #1962: - LOW-1: any error on a stream is reported on that stream (status 'failed', with its code), never thrown, so the next stream still syncs. - LOW-2: one connection and one account-key unlock serve every stream of a run (push and pull take a shared session) instead of five connects and four keychain reads. - LOW-3: a flag off, or push before this store has its own genesis, is reported in `skipped`, not as `refused`, so a store that only pulls shows synced. Co-Authored-By: Claude Opus 5.5 --- .changeset/t12996-cloud-sync.md | 5 +- packages/contracts/src/nexus-vault.ts | 10 +- .../src/cloud/__tests__/nexus-vault.test.ts | 8 +- packages/core/src/cloud/nexus-vault.ts | 128 +++++++++++++----- 4 files changed, 110 insertions(+), 41 deletions(-) diff --git a/.changeset/t12996-cloud-sync.md b/.changeset/t12996-cloud-sync.md index 0d2ee6083e..31c23205ff 100644 --- a/.changeset/t12996-cloud-sync.md +++ b/.changeset/t12996-cloud-sync.md @@ -11,11 +11,14 @@ account's global store, or just the one `--scope` names (T12996). - **Per stream:** it seals pending writes, pushes them as segments (`pushSyncStream`), pulls the stream's new segments, and applies them (`pullSyncStream`). - **One LAFS envelope** (`CloudSyncResult`) reports, per stream: - - `status`: `synced`, `disabled`, `paused` (device clock ahead), `not-attached` or `refused`; + - `status`: `synced`, `disabled`, `paused` (device clock ahead), `not-attached`, `refused`, or `failed` (an error stopped this + stream, reported, never thrown, so the next stream still syncs); + - `skipped`: legs with nothing to do (a flag off, or push before this store has its own genesis), which never count as refused; - sealed, built, sent, duplicates, received, staged, redelivered, applied, held and conflict counts; - the staged position against the server's head. - **`E_SYNC_DISABLED`** is returned, naming `cleo sync enable push`, when no attached stream has `sync.push` or `sync.pull` on. A store with `sync.pull` off now refuses its pull before it looks for a pull position. - **An interrupted run resumes on the next:** a segment is resent with the same bytes, and a transaction is never staged or applied twice. +- **One connection and one account-key unlock serve every stream of a run.** - `sync.push` and `sync.pull` stay unreleased: the CLI never opts in, so stores can't turn them on yet. diff --git a/packages/contracts/src/nexus-vault.ts b/packages/contracts/src/nexus-vault.ts index d357991ef6..ae78c7dfc6 100644 --- a/packages/contracts/src/nexus-vault.ts +++ b/packages/contracts/src/nexus-vault.ts @@ -211,11 +211,17 @@ export interface CloudSyncStreamResult { /** * `synced`: pushed and pulled what was enabled; `disabled`: neither `sync.push` nor `sync.pull` is on; * `paused`: push paused for a device clock ahead of the server's; `not-attached`: this machine has no - * store on that stream; `refused`: a precondition or the server refused (see `refused`). + * store on that stream; `refused`: a precondition or the server refused (see `refused`); `failed`: an + * error stopped this stream (see `refused`); the other streams still synced. */ - status: 'synced' | 'disabled' | 'paused' | 'not-attached' | 'refused'; + status: 'synced' | 'disabled' | 'paused' | 'not-attached' | 'refused' | 'failed'; /** Why this stream did not sync fully, or null. */ refused: string | null; + /** + * Legs with nothing to do, not failures: a flag off, or push before this store has its own genesis + * (a store that only pulls). + */ + skipped: string[]; /** Transactions sealed, segments persisted, segments the server stored (`duplicates` of them retries). */ sealed: number; built: number; diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index deb03bfde7..7cae0f777a 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5205,9 +5205,11 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { throw new ApiFail(403, 'E_FORBIDDEN'); }, }; - await expect( - on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))), - ).rejects.toThrow(); + // The failure is this stream's, reported, never thrown: every other stream still syncs. + const failed = await on(m, () => cloudSync(vopts(m, { allowUnreleased: true }))); + expect(failed.streams.map((x) => x.scope)).toEqual(['project', 'global']); + expect(failed.streams[0]).toMatchObject({ status: 'failed', sent: 0 }); + expect(failed.streams[0]?.refused).toMatch(/E_FORBIDDEN/); const before = fake.stream(STREAM).segments.length; const r = await on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))); expect(r.streams[0]).toMatchObject({ diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 944a328fb1..32754733e5 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -158,6 +158,7 @@ import { nexusApiErrorToAccountError } from './nexus-enrol.js'; import { readNexusProjectLink } from './nexus-link.js'; import { connectNexusVault, + type NexusAccountKey, type NexusVaultConnection, type NexusVaultOptions, nexusHomeDataKey, @@ -1749,15 +1750,35 @@ async function enableSyncPushImpl( } } +/** One connection, the account key and one scope's store, shared by a stream's push and pull (T12996). */ +interface StreamSession { + readonly conn: NexusVaultConnection; + readonly key: NexusAccountKey; + readonly t: VaultTarget; +} + +/** Connect, unlock the account key and resolve the scope's store, once. */ +async function openStreamSession(opts: NexusVaultCommandOptions): Promise { + const conn = await connectNexusVault(opts); + const key = await unlockNexusAccountKey(conn); + const t = await resolveTarget(conn, key.masterKey, opts, 'push'); + return { conn, key, t }; +} + async function pushSyncStreamImpl( opts: NexusVaultCommandOptions & { /** Push although `sync.push` is unreleased (tests and staging only). Never set from user input. */ allowUnreleased?: boolean; } = {}, ): Promise { - const conn = await connectNexusVault(opts); - const key = await unlockNexusAccountKey(conn); - const t = await resolveTarget(conn, key.masterKey, opts, 'push'); + return pushWithSession(await openStreamSession(opts), opts); +} + +async function pushWithSession( + session: StreamSession, + opts: { readonly allowUnreleased?: boolean }, +): Promise { + const { conn, t } = session; const replicaId = t.replicaId; if (!replicaId) { throw vaultError( @@ -1823,9 +1844,11 @@ async function pushSyncStreamImpl( } async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise { - const conn = await connectNexusVault(opts); - const key = await unlockNexusAccountKey(conn); - const t = await resolveTarget(conn, key.masterKey, opts, 'push'); + return pullWithSession(await openStreamSession(opts)); +} + +async function pullWithSession(session: StreamSession): Promise { + const { conn, key, t } = session; const replicaId = t.replicaId; if (!replicaId) { throw vaultError( @@ -1952,6 +1975,7 @@ const syncStreamResult = ( streamId: null, status: 'synced', refused: null, + skipped: [], sealed: 0, built: 0, sent: 0, @@ -1973,48 +1997,79 @@ const NOT_ATTACHED_CODES: ReadonlySet = new Set([ 'E_NEXUS_NOT_A_PROJECT', ]); -/** Seal, push, pull and apply one scope's stream (T12996). */ +/** Push or pull refusals that mean "nothing to do here", not a failure (LOW-3 on #1962). */ +const SKIP_REASONS: readonly RegExp[] = [ + /^sync\.push is off$/, + /^sync\.pull is off$/, + /has no genesis cut on this store/, + /genesis checkpoint is not stored yet/, +]; + +/** The code and message of a failure, for a stream result. */ +function failureText(err: unknown): string { + if (err instanceof NexusAccountError || err instanceof NexusError) + return `${err.code}: ${err.message}`; + return err instanceof Error ? err.message : String(err); +} + +/** + * Seal, push, pull and apply one scope's stream (T12996). The run's one + * connection and account key serve both legs. A failure is reported on this stream + * (`failed`), never thrown, so the next stream still syncs. + */ async function syncOneStream( + shared: { readonly conn: NexusVaultConnection; readonly key: NexusAccountKey }, opts: NexusVaultCommandOptions & { allowUnreleased?: boolean }, scope: CloudVaultScope, ): Promise { - const failed = (err: unknown): CloudSyncStreamResult | null => { - if (!(err instanceof NexusAccountError)) return null; - return NOT_ATTACHED_CODES.has(err.code) - ? syncStreamResult(scope, { status: 'not-attached', refused: err.message }) - : syncStreamResult(scope, { status: 'refused', refused: `${err.code}: ${err.message}` }); - }; + let session: StreamSession; + try { + session = { + ...shared, + t: await resolveTarget(shared.conn, shared.key.masterKey, { ...opts, scope }, 'push'), + }; + } catch (err) { + if (err instanceof NexusAccountError && NOT_ATTACHED_CODES.has(err.code)) { + return syncStreamResult(scope, { status: 'not-attached', refused: err.message }); + } + return syncStreamResult(scope, { status: 'failed', refused: failureText(err) }); + } + const streamId = session.t.streamId; + if (!session.t.replicaId) { + return syncStreamResult(scope, { + streamId, + status: 'not-attached', + refused: 'this copy of the project is not attached from this device', + }); + } let push: PushStreamReport; try { - push = await pushSyncStreamImpl({ ...opts, scope }); + push = await pushWithSession(session, opts); } catch (err) { - const r = failed(err); - if (r) return r; - throw err; + return syncStreamResult(scope, { streamId, status: 'failed', refused: failureText(err) }); } let pull: PullStreamReport; try { - pull = await pullSyncStreamImpl({ ...opts, scope }); + pull = await pullWithSession(session); } catch (err) { - const r = failed(err); - if (r) - return { - ...r, - streamId: push.stream, - sealed: push.sealed, - built: push.built, - sent: push.pushed, - }; - throw err; + return syncStreamResult(scope, { + streamId, + status: 'failed', + refused: failureText(err), + sealed: push.sealed, + built: push.built, + sent: push.pushed, + duplicates: push.duplicates, + }); } + const isSkip = (r: string | null): boolean => r !== null && SKIP_REASONS.some((x) => x.test(r)); + const skipped = [push.refused, pull.refused].filter((r): r is string => isSkip(r)); + const refused = [push.refused, pull.refused].filter((r): r is string => r !== null && !isSkip(r)); const pushOff = push.refused === 'sync.push is off'; const pullOff = pull.refused === 'sync.pull is off'; - const refused = [push.refused, pull.refused].filter( - (r): r is string => r !== null && r !== 'sync.push is off' && r !== 'sync.pull is off', - ); const apply = pull.apply; return syncStreamResult(scope, { - streamId: push.stream, + streamId, status: pushOff && pullOff ? 'disabled' @@ -2024,6 +2079,7 @@ async function syncOneStream( ? 'paused' : 'synced', refused: refused.length > 0 ? refused.join('; ') : null, + skipped, sealed: push.sealed, built: push.built, sent: push.pushed, @@ -2034,18 +2090,20 @@ async function syncOneStream( applied: apply?.applied ?? 0, held: (apply?.pending ?? 0) + (apply?.heldSkew ?? 0) + (apply?.refusedSchema ?? 0), conflicts: apply?.conflict ?? 0, - after: pull.refused === null ? pull.after : null, - head: pull.refused === null ? pull.head : null, + after: pull.refused === null || !isSkip(pull.refused) ? pull.after : null, + head: pull.refused === null || !isSkip(pull.refused) ? pull.head : null, }); } async function cloudSyncImpl( opts: NexusVaultCommandOptions & { allowUnreleased?: boolean } = {}, ): Promise { + // One connection and one account-key unlock for every stream (LOW-2 on #1962). const conn = await connectNexusVault(opts); + const key = await unlockNexusAccountKey(conn); const scopes: CloudVaultScope[] = opts.scope ? [opts.scope] : ['project', 'global']; const streams: CloudSyncStreamResult[] = []; - for (const scope of scopes) streams.push(await syncOneStream(opts, scope)); + for (const scope of scopes) streams.push(await syncOneStream({ conn, key }, opts, scope)); const worked = streams.filter((r) => r.status !== 'disabled' && r.status !== 'not-attached'); if (worked.length === 0 && streams.some((r) => r.status === 'disabled')) { throw vaultError( From b319c4859be57a79277f861f8795538aaab1eab2 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:31:07 -0700 Subject: [PATCH 14/23] feat(sync): a restored store joins a journaled stream; two-device sync end to end (T13312, T12999, T12996) A store restored from another device's journal checkpoint had no row meta and no join state, so enable push refused it and a second device could not enter the stream. - Restore keeps _sync_row_meta and _sync_field_leave on a journal checkpoint (carryMachineState snapshotCarried). - joinStream: refuses unless every row matches its restored meta, there are no waiting captures and the triggers are present, then cuts at the capture position, baselines the ledgers, turns on push/pull/undo and seeds the pull cursor from the checkpoint. Sealing is enabled only after the check. - enableSyncPush joins (status 'joined') when the head journal checkpoint is the one this store synced and its manifest matches; a changed store is refused with the restore --force remedy, never folded; any other store gets E_NEXUS_SYNC_STREAM_JOURNALED. - Tests: the two-device headline (A and B each write, sync, see the other's write; a concurrent done/cancel converges by the typed rules), pull-only joiner reports synced with the push leg skipped, the three refusals, the T13306 remedy followed through to a join. All 7 mutations killed. Co-Authored-By: Claude Opus 5.5 --- .changeset/t13312-journal-join.md | 28 +++ packages/contracts/src/nexus-vault.ts | 6 +- .../src/cloud/__tests__/nexus-vault.test.ts | 216 +++++++++++++++++- packages/core/src/cloud/nexus-vault.ts | 78 ++++++- packages/core/src/store/sync/genesis.ts | 112 ++++++++- packages/core/src/store/sync/pull.ts | 27 ++- packages/core/src/store/vault-manifest.ts | 11 + scripts/lint-no-raw-table-writes.mjs | 4 +- 8 files changed, 455 insertions(+), 27 deletions(-) create mode 100644 .changeset/t13312-journal-join.md diff --git a/.changeset/t13312-journal-join.md b/.changeset/t13312-journal-join.md new file mode 100644 index 0000000000..dce06f66ed --- /dev/null +++ b/.changeset/t13312-journal-join.md @@ -0,0 +1,28 @@ +--- +id: t13312-journal-join +tasks: [T13312, T12999, T12996] +kind: feat +summary: "`cleo sync enable push` joins a stream another device journaled, from the journal checkpoint this store restored" +--- + +A second device can now enter a stream whose change journal another device already started (T13312, T12999). Before this, a +store restored from the journal checkpoint carried no row meta and no join state, so `cleo sync enable push` refused it and +there was no way in. + +- **Restore keeps the journal's row state.** On a journal checkpoint, the restore keeps the snapshot's `_sync_row_meta` and + `_sync_field_leave` rows instead of clearing them as machine-local. +- **The join.** On a stream whose head is a journal checkpoint this store restored, `cleo sync enable push` returns status + `joined`: + - the store's cut is placed at its current capture position; + - the ledgers are baselined; + - push, pull and undo are turned on; + - pulling resumes right after the restored checkpoint. + No second genesis is pushed. +- **A changed store is refused, never folded.** The join is refused with `E_NEXUS_SYNC_REFUSED` when: + - any table's rows differ from the checkpoint's manifest; + - any row no longer matches its restored row meta; + - a capture is still waiting to seal. + The remedy is to restore the checkpoint again (`cleo cloud restore --force`), then join. A refused join changes nothing. +- **A store that never restored the journal checkpoint** gets `E_NEXUS_SYNC_STREAM_JOURNALED`, naming the restore-then-join + path. The pull refusal for a store behind the journal genesis (T13306) names the same path. +- `sync.push` and `sync.pull` stay unreleased. diff --git a/packages/contracts/src/nexus-vault.ts b/packages/contracts/src/nexus-vault.ts index ae78c7dfc6..21bae1ada3 100644 --- a/packages/contracts/src/nexus-vault.ts +++ b/packages/contracts/src/nexus-vault.ts @@ -186,9 +186,11 @@ export interface CloudSyncPushEnableResult { streamId: string; /** * `enabled`: the cut was recorded and its genesis checkpoint stored; `resumed`: a cut recorded - * by an earlier run whose checkpoint push failed was pushed now; `already`: push was already on. + * by an earlier run whose checkpoint push failed was pushed now; `already`: push was already on; + * `joined`: this store holds the stream's journal checkpoint another device cut, and now pushes its + * own writes and pulls the stream's after it, with no genesis of its own (T13312). */ - status: 'enabled' | 'resumed' | 'already'; + status: 'enabled' | 'resumed' | 'already' | 'joined'; /** The genesis cut: the highest capture seq the checkpoint carries. */ cut: number; /** Transactions sealed before the cut, folded into the checkpoint, and rows given genesis row meta. */ diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 7cae0f777a..09b27973f2 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -67,6 +67,7 @@ import { import { setCaptureEnabled } from '../../store/sync/capture.js'; import { isSyncFlagOn, setSyncFlag } from '../../store/sync/flags.js'; import { cutGenesis, genesisCutOf, genesisPending } from '../../store/sync/genesis.js'; +import { readStreamCursor } from '../../store/sync/pull.js'; import { ensureProjectReplica, storeHwm } from '../../store/sync/replica.js'; import { ensureSyncSchema } from '../../store/sync/schema.js'; import { @@ -4861,9 +4862,15 @@ describe('cloud project link stores a new project key with its registration (onb describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { const SYNC_JOURNAL = path.resolve(import.meta.dirname, '../../../migrations/sync-journal'); - /** A machine whose project store is a real cleo.db with capture and seal on, bound and linked. */ - async function journalMachine(): Promise<{ m: Machine; dbPath: string }> { - const m = await machine('a', DEVICE_A, REPLICA_A); + /** + * A machine whose project store is a real cleo.db with capture and seal on, bound and linked. + * Device A by default; device B attaches to the project A registered. + */ + async function journalMachine(device: 'a' | 'b' = 'a'): Promise<{ m: Machine; dbPath: string }> { + const m = + device === 'a' + ? await machine('a', DEVICE_A, REPLICA_A) + : await machine('b', DEVICE_B, REPLICA_B); const cleo = path.join(m.root, '.cleo'); fs.mkdirSync(cleo, { recursive: true }); fs.writeFileSync(path.join(cleo, 'project-id'), `${LOCAL_PROJECT}\n`); @@ -4887,7 +4894,8 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { return ensureProjectReplica(db, { dbPath, mode: 'live' }).replicaId; }); m.replicaId = replicaId; - fake.addProject(REMOTE_PROJECT, { [replicaId]: DEVICE_A }); + if (device === 'a') fake.addProject(REMOTE_PROJECT, { [replicaId]: DEVICE_A }); + else fake.replicas.get(REMOTE_PROJECT)?.set(replicaId, DEVICE_B); link(m); return { m, dbPath }; } @@ -4895,6 +4903,25 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { const storeOf = async (dbPath: string) => getDualScopeNativeDb(await openDualScopeDbAtPath('project', dbPath)); + /** + * Device B with A's journal checkpoint restored, attached under its own bound + * replica, and not yet joined. + */ + async function restoredJoiner(): Promise<{ b: Machine; bDb: string }> { + const b = await machine('b', DEVICE_B, REPLICA_B); + fake.replicas.get(REMOTE_PROJECT)?.set(REPLICA_B, DEVICE_B); + await restoreOntoB(b); + const bDb = path.join(b.root, '.cleo', 'cleo.db'); + const bound = await on( + b, + async () => ensureProjectReplica(await storeOf(bDb), { dbPath: bDb, mode: 'live' }).replicaId, + ); + b.replicaId = bound; + fake.replicas.get(REMOTE_PROJECT)?.set(bound, DEVICE_B); + link(b); + return { b, bDb }; + } + it('cuts the store and pushes a checkpoint/v3 genesis the server accepts; push turns on and the vault steps aside', async () => { const { m, dbPath } = await journalMachine(); const r = await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true }))); @@ -4925,13 +4952,22 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { const vault = await failure(on(m, () => pushNexusVault(vopts(m)))); expect(vault.code).toBe('E_NEXUS_VAULT_STREAM_UPGRADED'); expect(s.checkpoints).toHaveLength(1); - // A store that never cut this stream (its cut forgotten) meets a journal head: it joins by - // pulling, never with a second genesis. + // A store that holds the journal checkpoint but forgot its cut joins it, never with a + // second genesis (T13312). await on(m, async () => { (await storeOf(dbPath)).exec("DELETE FROM _sync_meta WHERE key LIKE 'genesis_%'"); }); - const second = await failure(on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true })))); + expect((await on(m, () => enableSyncPush(vopts(m, { allowUnreleased: true })))).status).toBe( + 'joined', + ); + expect(s.checkpoints).toHaveLength(1); + // A store that never restored that checkpoint is told to restore it, then join. + const { m: other } = await journalMachine('b'); + const second = await failure( + on(other, () => enableSyncPush(vopts(other, { allowUnreleased: true }))), + ); expect(second.code).toBe('E_NEXUS_SYNC_STREAM_JOURNALED'); + expect(second.fix).toContain('cleo cloud restore'); expect(s.checkpoints).toHaveLength(1); }); @@ -5272,6 +5308,172 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { 0, ); }); + // The remedy works: B restores the journal checkpoint, joins, and holds A's folded change. + const restored = await on(b, () => restoreNexusVault(vopts(b, { mode: 'pull', force: true }))); + expect(restored.status).toBe('restored'); + expect((await on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))).status).toBe( + 'joined', + ); + expect( + await on(b, async () => + (await storeOf(bDb)).prepare("SELECT title FROM tasks_tasks WHERE id = 'G1'").get(), + ), + ).toEqual({ title: 'title G1' }); + expect((await on(b, () => pullSyncStream(vopts(b)))).refused).toBeNull(); + }); + + it("two devices (T13312, T12996): B restores A's genesis and joins; cloud sync carries each device's writes to the other and converges a concurrent status change", async () => { + const { m: a, dbPath: aDb } = await journalMachine(); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + await on(a, async () => { + setSyncFlag(await storeOf(aDb), 'sync.pull', true, { allowUnreleased: true }); + }); + const { b, bDb } = await restoredJoiner(); + const joined = await on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true }))); + expect(joined.status).toBe('joined'); + const journalCps = fake + .stream(STREAM) + .checkpoints.filter((c) => manifestVersion(c.manifest) === 3); + expect(journalCps).toHaveLength(1); + // B pulls on from the checkpoint it restored. + const genesisCp = journalCps[0]; + if (!genesisCp) throw new Error('no journal checkpoint'); + expect(await on(b, async () => readStreamCursor(await storeOf(bDb), STREAM))).toEqual( + cursorFromCheckpoint(genesisCp), + ); + + const sync = async (m: Machine) => { + const r = await on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))); + expect(r.streams[0]?.refused ?? null, `${m.name}: ${r.streams[0]?.refused}`).toBeNull(); + return r.streams[0]; + }; + const value = async (m: Machine, dbFile: string, sql: string) => + on(m, async () => (await storeOf(dbFile)).prepare(sql).get()); + + // Each device edits; both sync; each sees the other's write. + await on(a, async () => { + (await storeOf(aDb)).exec("UPDATE tasks_tasks SET title = 'from A' WHERE id = 'T1'"); + }); + await on(b, async () => { + (await storeOf(bDb)).exec( + "INSERT INTO tasks_tasks (id, title, type, status, priority, uid, birth_fp) VALUES ('T9', 'from B', 'task', 'pending', 'medium', 'uid-T9', 'fp-T9')", + ); + }); + await sync(a); + await sync(b); + await sync(a); + expect(await value(b, bDb, "SELECT title FROM tasks_tasks WHERE id = 'T1'")).toEqual({ + title: 'from A', + }); + expect(await value(a, aDb, "SELECT title FROM tasks_tasks WHERE id = 'T9'")).toEqual({ + title: 'from B', + }); + + // A concurrent status change of one task: the typed rules decide it the same way on both. + await on(a, async () => { + (await storeOf(aDb)).exec( + "UPDATE tasks_tasks SET status = 'done', pipeline_stage = 'contribution', completed_at = '2026-10-07T00:00:00.000Z' WHERE id = 'T2'", + ); + }); + await on(b, async () => { + (await storeOf(bDb)).exec( + "UPDATE tasks_tasks SET status = 'cancelled', pipeline_stage = 'cancelled', cancelled_at = '2026-10-07T00:00:01.000Z', cancellation_reason = 'dup' WHERE id = 'T2'", + ); + }); + await sync(a); + await sync(b); + await sync(a); + await sync(b); + const statusSql = + "SELECT status, pipeline_stage, completed_at, cancelled_at, cancellation_reason FROM tasks_tasks WHERE id = 'T2'"; + const onA = await value(a, aDb, statusSql); + const onB = await value(b, bDb, statusSql); + expect(onB).toEqual(onA); + // Both are absorbing, so the later write (B's cancel) wins its LWW; the + // stamps move with it as one group and the stage follows the status. + expect(onA).toEqual({ + status: 'cancelled', + pipeline_stage: 'cancelled', + completed_at: null, + cancelled_at: '2026-10-07T00:00:01.000Z', + cancellation_reason: 'dup', + }); + }); + + it('a joined store with push off syncs pull-only: synced, the push leg skipped (T13312)', async () => { + const { m: a, dbPath: aDb } = await journalMachine(); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + const { b, bDb } = await restoredJoiner(); + expect((await on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))).status).toBe( + 'joined', + ); + await on(b, async () => { + setSyncFlag(await storeOf(bDb), 'sync.push', false, { allowUnreleased: true }); + }); + await on(a, async () => { + (await storeOf(aDb)).exec("UPDATE tasks_tasks SET title = 'from A' WHERE id = 'T1'"); + }); + await on(a, () => pushSyncStream(vopts(a, { allowUnreleased: true }))); + const r = await on(b, () => cloudSync(vopts(b, { scope: 'project', allowUnreleased: true }))); + expect(r.streams[0]).toMatchObject({ status: 'synced', skipped: ['sync.push is off'] }); + expect( + await on(b, async () => + (await storeOf(bDb)).prepare("SELECT title FROM tasks_tasks WHERE id = 'T1'").get(), + ), + ).toEqual({ title: 'from A' }); + }); + + it('a join is refused, never folded, when the store changed since its restore (T13312)', async () => { + const { m: a } = await journalMachine(); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + const { b, bDb } = await restoredJoiner(); + await on(b, async () => { + (await storeOf(bDb)).exec("UPDATE tasks_tasks SET title = 'edited on B' WHERE id = 'T1'"); + }); + const err = await failure(on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))); + expect(err.code).toBe('E_NEXUS_SYNC_REFUSED'); + expect(err.message).toContain('tasks_tasks'); + expect(err.fix).toContain('cleo cloud restore --force'); + await on(b, async () => { + const db = await storeOf(bDb); + expect(genesisCutOf(db, STREAM)).toBeUndefined(); + expect(isSyncFlagOn(db, 'sync.push', {})).toBe(false); + }); + expect(fake.stream(STREAM).segments).toHaveLength(0); + }); + + it('a join is refused, never sealed past, while a capture waits to seal (T13312)', async () => { + const { m: a } = await journalMachine(); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + const { b, bDb } = await restoredJoiner(); + // A write captured but not sealed: joining would seal it as if the checkpoint authored it. + await on(b, async () => { + (await storeOf(bDb)).exec( + "INSERT INTO _sync_capture (tbl, op, rk, uid, img, at_ms) VALUES ('tasks_tasks', 'U', 'T1', 'uid-T1', '{}', 0)", + ); + }); + const err = await failure(on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))); + expect(err.code).toBe('E_NEXUS_SYNC_REFUSED'); + expect(err.message).toContain('captures still waiting to seal'); + }); + + it('a join is refused when the restored row meta does not match the rows (T13312)', async () => { + const { m: a } = await journalMachine(); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + const { b, bDb } = await restoredJoiner(); + await on(b, async () => { + (await storeOf(bDb)).exec( + "DELETE FROM _sync_row_meta WHERE tbl = 'tasks_tasks' AND uid = (SELECT uid FROM _sync_row_meta WHERE tbl = 'tasks_tasks' ORDER BY uid LIMIT 1)", + ); + }); + const err = await failure(on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))); + expect(err.code).toBe('E_NEXUS_SYNC_REFUSED'); + expect(err.message).toContain('tasks_tasks'); + await on(b, async () => { + const db = await storeOf(bDb); + expect(genesisCutOf(db, STREAM)).toBeUndefined(); + expect(isSyncFlagOn(db, 'sync.seal', {})).toBe(false); + }); }); it('S4-2: a device clock ahead of the server pauses push', async () => { diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 32754733e5..e6bd37a637 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -112,6 +112,7 @@ import { GenesisRacedError, genesisCutOf, genesisPending, + joinStream, readGenesisCut, } from '../store/sync/genesis.js'; import { type PullStreamReport, pullStream, readStreamCursor } from '../store/sync/pull.js'; @@ -797,6 +798,17 @@ function compareWithSynced( * hold values the snapshot strips, and a restore takes these tables from the * live store anyway. */ +/** + * The merge state a journal checkpoint carries for the rows it holds (journal + * spec §2.10: row meta travels inside checkpoint bundles): a restore keeps + * the snapshot's, never this machine's, so the restored rows keep their HLCs + * and a JOIN can push and pull them (T13312). + */ +const JOURNAL_SNAPSHOT_CARRIED: ReadonlySet = new Set([ + '_sync_row_meta', + '_sync_field_leave', +]); + const VAULT_CLEARED_JOURNAL_TABLES = [ '_sync_capture', '_sync_undo', @@ -1565,16 +1577,60 @@ async function enableSyncPushImpl( }; } } - // Another device already started this stream's journal: this store joins it - // by pulling (S5), never with a second genesis. - if (isJournalSnapshot(parent)) { - throw vaultError( - 'E_NEXUS_SYNC_STREAM_JOURNALED', - `${t.streamId} already carries the change journal: its head snapshot ${parent?.checkpointId} is a journal checkpoint`, - 'nothing was cut; restore the journal checkpoint (`cleo cloud restore`), then join the stream with `cleo sync enable push` (T12999)', - ); - } const { state: synced } = syncedState(conn, t, parent ? [parent] : [], head.headCheckpointId); + // Another device already started this stream's journal: never a second + // genesis. A store that holds exactly that journal checkpoint JOINS it + // (T13312); any other store restores it first. + if (parent !== null && isJournalSnapshot(parent)) { + if (synced?.lastCheckpointId !== parent.checkpointId) { + throw vaultError( + 'E_NEXUS_SYNC_STREAM_JOURNALED', + `${t.streamId} already carries the change journal: its head snapshot ${parent.checkpointId} is a journal checkpoint this store has not restored`, + 'nothing was cut; restore the journal checkpoint (`cleo cloud restore`), then join the stream with `cleo sync enable push`', + ); + } + // Unchanged since the restore: the rows match the checkpoint's manifest + // (row data) here, and joinStream checks every row against its meta. + const git = gitTracking(t); + const local = await localManifest(t, git, untrackedComparison(t, git, synced)); + const changed = + local === null + ? ['(no store)'] + : compareVaultManifests(local.manifest, comparable(parent.manifest, git)) + .filter( + (r) => r.table !== VAULT_FILES_KEY && !isVaultAnnotationKey(r.table) && !r.match, + ) + .map((r) => r.table); + if (changed.length > 0) { + throw vaultError( + 'E_NEXUS_SYNC_REFUSED', + `this store changed since it restored ${parent.checkpointId} (${changed.slice(0, 8).join(', ')}); joining would lose those changes`, + 'nothing was joined; restore the journal checkpoint again (`cleo cloud restore --force`, a safety backup is taken first), then `cleo sync enable push`', + ); + } + const joined = joinStream(db, { + scope: tableScopeOf(t), + stream: t.streamId, + cursor: cursorFromCheckpoint(parent), + ...(opts.allowUnreleased ? { allowUnreleased: true } : {}), + }); + if (joined.refused !== null) { + throw vaultError('E_NEXUS_SYNC_REFUSED', `the join was refused: ${joined.refused}`); + } + warnings.push(...conn.state.drainWarnings()); + return { + ...base, + status: 'joined', + cut: joined.cut ?? 0, + sealed: 0, + folded: 0, + baselined: {}, + snapshot: snapshotOf(parent, await deviceNames(conn)), + deltaSegmentSeq: null, + replicaSeqFloor: null, + warnings, + }; + } if (head.headCheckpointId !== null && head.headCheckpointId !== synced?.lastCheckpointId) { throw vaultError( 'E_NEXUS_VAULT_BEHIND', @@ -2594,6 +2650,10 @@ async function restoreNexusVaultImpl(opts: NexusVaultRestoreOptions): Promise = new Set([ + 'table does not exist', + 'not in the sync set', + 'table has no uid column', +]); + +/** + * Join a stream another device already started (T13312; journal spec + * §2.11 §10, §3.5 Rule 2): the store holds exactly the journal checkpoint it + * restored, with that checkpoint's row meta, and from here on pushes its own + * writes and pulls the stream's after it. No genesis, no fold: the checkpoint + * is the store's base. + * + * Every sync-set row must carry meta that matches it (the restore carried + * the checkpoint's merge state, and nothing changed since); a row without + * meta, or one whose content left its meta behind, refuses the join. Then, + * capture and seal turned on, in ONE `BEGIN IMMEDIATE`: the restore's + * suspect marks are cleared (the check just proved the rows clean), the + * ledgers set, `genesis_cut:` records the join point (the stream has + * started here: T13217's meta-less-row rule applies from now), `undo_enabled`, + * `sync.push` and `sync.pull` are set, and the pull position is seeded from + * the checkpoint. + * + * @param db - The store, outside a transaction, bound to its own replica. + * @param o - Scope, stream, the checkpoint's pull position, and the unreleased opt-in. + * @returns The join point, or why not. + */ +export function joinStream( + db: DatabaseSync, + o: { + readonly scope: TableScope; + readonly stream: string; + readonly cursor: StreamCursor; + readonly now?: () => number; + readonly allowUnreleased?: boolean; + }, +): JoinStreamReport { + if (db.isTransaction) { + // @sync-invariant none:local-only programming-error guard: the join opens its own transaction + throw new Error('joinStream must run outside a transaction'); + } + const already = genesisCutOf(db, o.stream); + if (already !== undefined) return { stream: o.stream, refused: null, cut: already }; + const replica = activeReplica(db, o.scope)?.replicaId; + if (!replica) return { stream: o.stream, refused: 'no bound replica', cut: null }; + const now = o.now ?? Date.now; + // Capture installs the triggers the row check needs; sealing waits for the + // check, so a refused join seals nothing. + setCaptureEnabled(db, o.scope, true); + // Every row must still match the meta the checkpoint carried. + const view = sealerRowView(db, o.scope); + for (const table of syncSetTables(o.scope)) { + const plan = planRepair(db, o.scope, table, view); + if (plan.skipped !== null && !JOIN_BENIGN_SKIPS.has(plan.skipped)) { + return { stream: o.stream, refused: `${table}: ${plan.skipped}`, cut: null }; + } + if (plan.skipped !== null) continue; + const off = + plan.inserts.length + plan.updates.length + plan.deletes.length + plan.unbaselined.length; + if (off > 0) { + return { + stream: o.stream, + refused: `${table}: ${off} row(s) without matching row meta; the store is not the journal checkpoint it restored (restore it again)`, + cut: null, + }; + } + } + setSyncFlag(db, 'sync.seal', true, o.allowUnreleased ? { allowUnreleased: true } : {}); + const at = now(); + const atIso = new Date(at).toISOString(); + db.exec('BEGIN IMMEDIATE'); + try { + db.prepare("DELETE FROM _sync_meta WHERE key LIKE 'suspect:%'").run(); + // Ledgers (count + held) and baseline keys; every row already has meta. + for (const table of syncSetTables(o.scope)) baselineRowMeta(db, o.scope, table, replica, at); + const cut = capturePosition(db); + setMeta(db, `${GENESIS_CUT_KEY_PREFIX}${o.stream}`, String(cut), atIso); + setMeta(db, `${GENESIS_SOURCE_SEQ_KEY_PREFIX}${o.stream}`, String(cut), atIso); + setMeta(db, UNDO_ENABLED_KEY, '1', atIso); + raiseMinWriterVersion(db); + const flag = { now: new Date(at), ...(o.allowUnreleased ? { allowUnreleased: true } : {}) }; + setSyncFlag(db, 'sync.push', true, flag); + setSyncFlag(db, 'sync.pull', true, flag); + writeStreamCursor(db, o.stream, o.cursor, atIso); + db.exec('COMMIT'); + return { stream: o.stream, refused: null, cut }; + } catch (err) { + if (db.isTransaction) db.exec('ROLLBACK'); + throw err; + } +} diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts index cd66eb154f..7e0357d1dd 100644 --- a/packages/core/src/store/sync/pull.ts +++ b/packages/core/src/store/sync/pull.ts @@ -137,6 +137,28 @@ export function readStreamCursor(db: DatabaseSync, stream: string): StreamCursor return row ? (JSON.parse(row.cursor_json) as StreamCursor) : null; } +/** + * Persist the store's pull position for `stream`, in the caller's + * transaction (the pull's page, or a JOIN seeding it from the checkpoint it + * restored). + * + * @param db - The store. + * @param stream - The stream. + * @param cursor - The position. + * @param nowIso - The time. + */ +export function writeStreamCursor( + db: DatabaseSync, + stream: string, + cursor: StreamCursor, + nowIso: string, +): void { + db.prepare( + 'INSERT INTO _sync_cursor (stream, cursor_json, updated_at) VALUES (?, ?, ?) ' + + 'ON CONFLICT (stream) DO UPDATE SET cursor_json = excluded.cursor_json, updated_at = excluded.updated_at', + ).run(stream, JSON.stringify(cursor), nowIso); +} + /** Decode a segment's transactions, or `null` for a vault delta segment. */ function decode(seg: PulledStreamSegment): LedgerTxn[] | null { let text: string; @@ -312,10 +334,7 @@ export async function pullStream( nowIso, ); } - db.prepare( - 'INSERT INTO _sync_cursor (stream, cursor_json, updated_at) VALUES (?, ?, ?) ' + - 'ON CONFLICT (stream) DO UPDATE SET cursor_json = excluded.cursor_json, updated_at = excluded.updated_at', - ).run(o.stream, JSON.stringify(pageCursor), nowIso); + writeStreamCursor(db, o.stream, pageCursor, nowIso); }); cursor = pageCursor; if (refusedSegment !== null) break; diff --git a/packages/core/src/store/vault-manifest.ts b/packages/core/src/store/vault-manifest.ts index 81573379e6..9fdb597443 100644 --- a/packages/core/src/store/vault-manifest.ts +++ b/packages/core/src/store/vault-manifest.ts @@ -638,6 +638,13 @@ export interface CarryMachineStateOptions { * the relocation rewrites them. `null` (a global store): no path is relocated. */ snapshotRoot: string | null; + /** + * Local-only tables whose rows come from the SNAPSHOT, not this machine: a + * journal checkpoint carries the merge state of the rows it holds (row + * meta, field frontiers; journal spec §2.10), which describes those rows + * and nothing on this machine (T13312). + */ + snapshotCarried?: ReadonlySet; } type Cell = string | number | bigint | null | Uint8Array; @@ -832,6 +839,10 @@ export function carryMachineState( out.preserved.push(t); }; + if (c.class === 'local-only' && opts.snapshotCarried?.has(t)) { + out.skipped.push(t); // left as the snapshot had it + continue; + } if (c.class === 'local-only') { // Machine state never comes from another machine: a table this store // does not have yet is emptied, not filled with the pusher's rows. diff --git a/scripts/lint-no-raw-table-writes.mjs b/scripts/lint-no-raw-table-writes.mjs index eb9a9820f5..969adc2b7c 100644 --- a/scripts/lint-no-raw-table-writes.mjs +++ b/scripts/lint-no-raw-table-writes.mjs @@ -377,8 +377,8 @@ export const EXEMPT = { }, 'packages/core/src/store/sync/genesis.ts': { _sync_meta: { - count: 3, - reason: `${SYNC_BOOKKEEPING}; plus dropping a stream's genesis keys when a failed or raced cut is undone (T13296), and clearing genesis_pending once the genesis checkpoint is stored (T13300)`, + count: 4, + reason: `${SYNC_BOOKKEEPING}; plus dropping a stream's genesis keys when a failed or raced cut is undone (T13296), clearing genesis_pending once the genesis checkpoint is stored (T13300), and clearing the restored suspect marks when a store joins a journal it restored (T13312)`, }, _sync_txn: { count: 2, From c0a42bba8e8db1d0ba529035ecf3cbe973c2f20b Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:37:02 -0700 Subject: [PATCH 15/23] test(sync): two idle replicas exchange no segments after the first round (T13256) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §3.5 R7-7: after one real write each and a first sync round, four more idle rounds of cloud sync on both devices send nothing and the stream's segment count holds; both stores converge. Mutation-checked: disabling the sealer's apply-echo subtraction makes A send a segment in round 0. Co-Authored-By: Claude Opus 5.5 --- .changeset/t13256-idle-no-pingpong.md | 16 +++++++ .../src/cloud/__tests__/nexus-vault.test.ts | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .changeset/t13256-idle-no-pingpong.md diff --git a/.changeset/t13256-idle-no-pingpong.md b/.changeset/t13256-idle-no-pingpong.md new file mode 100644 index 0000000000..8f1703180f --- /dev/null +++ b/.changeset/t13256-idle-no-pingpong.md @@ -0,0 +1,16 @@ +--- +id: t13256-idle-no-pingpong +tasks: [T13256] +kind: test +summary: "Two idle replicas syncing in a loop exchange no segments after the first round (journal spec §3.5 R7-7)" +--- + +A new two-device test runs on the fake server. Device B joins the journal device A started, and each device makes one real +write. One round of `cleo cloud sync` carries both writes across. Then four more rounds with no writes run on each device. + +- No device sends a segment in those rounds, and the stream's segment count does not move. +- Applying the other device's segment never makes a device seal or send one back: the apply's echo is subtracted (§3.3). +- Both stores end with the same rows. + +This test guards the R7-7 rule for when receive watermarks land: a watermark rides on an ordinary segment and is never sent +alone after a pull. diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 09b27973f2..96ce889583 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5400,6 +5400,50 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { }); }); + it('two idle replicas syncing in a loop exchange no segments after the first round (T13256, §3.5 R7-7)', async () => { + const { m: a, dbPath: aDb } = await journalMachine(); + await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); + await on(a, async () => { + setSyncFlag(await storeOf(aDb), 'sync.pull', true, { allowUnreleased: true }); + }); + const { b, bDb } = await restoredJoiner(); + expect((await on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))).status).toBe( + 'joined', + ); + const sync = async (m: Machine) => { + const r = await on(m, () => cloudSync(vopts(m, { scope: 'project', allowUnreleased: true }))); + expect(r.streams[0]?.status, `${m.name}: ${r.streams[0]?.refused}`).toBe('synced'); + return r.streams[0]; + }; + // One real write each, then a first round carries them both ways. + await on(a, async () => { + (await storeOf(aDb)).exec("UPDATE tasks_tasks SET title = 'from A' WHERE id = 'T1'"); + }); + await on(b, async () => { + (await storeOf(bDb)).exec("UPDATE tasks_tasks SET title = 'from B' WHERE id = 'T2'"); + }); + await sync(a); + await sync(b); + await sync(a); + const settled = fake.stream(STREAM).segments.length; + expect(settled).toBe(2); + // Idle from here on: applying the other's segment never makes either side send one back. + for (let round = 0; round < 4; round++) { + const ra = await sync(a); + const rb = await sync(b); + expect(ra?.sent, `round ${round}: A sent`).toBe(0); + expect(rb?.sent, `round ${round}: B sent`).toBe(0); + } + expect(fake.stream(STREAM).segments).toHaveLength(settled); + const titles = "SELECT id, title FROM tasks_tasks WHERE id IN ('T1', 'T2') ORDER BY id"; + const onA = await on(a, async () => (await storeOf(aDb)).prepare(titles).all()); + expect(await on(b, async () => (await storeOf(bDb)).prepare(titles).all())).toEqual(onA); + expect(onA).toEqual([ + { id: 'T1', title: 'from A' }, + { id: 'T2', title: 'from B' }, + ]); + }); + it('a joined store with push off syncs pull-only: synced, the push leg skipped (T13312)', async () => { const { m: a, dbPath: aDb } = await journalMachine(); await on(a, () => enableSyncPush(vopts(a, { allowUnreleased: true }))); From f1460b1c4a4f5e028942e9e4c31fd41434c5e45b Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:46:33 -0700 Subject: [PATCH 16/23] fix(sync): the cloud pull does not prune seen-txn rows until a refusal floor exists (T13256) A seen txn can return in a NEW segment above any checkpoint (a retired replica's late segment, a rebind re-pushing a stored-but-unrecorded upload); with its first-delivery row pruned by checkpoint coversSeq it would stage and apply twice (an $inc column would double). pullSyncStream no longer passes pruneSeenUpTo; pruneSeenTxns stays, documented as safe only below a floor the applier refuses under (the receive watermark). Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-pull.md | 6 ++++-- packages/core/src/cloud/nexus-vault.ts | 13 +++++++------ packages/core/src/store/sync/pull.ts | 14 +++++++++----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.changeset/t12343-pull.md b/.changeset/t12343-pull.md index a1e2b9cff7..912d961a62 100644 --- a/.changeset/t12343-pull.md +++ b/.changeset/t12343-pull.md @@ -20,8 +20,10 @@ A store can now pull its stream's journal (journal spec §3.1, §2.8; T12343 S5- - **Stage and advance:** the rest are staged, and the cursor advances (`_sync_cursor`). Then the applier applies, in stream order, own echoes included (they feed the fast path and the scoped rebase). -- **`_sync_seen_txn` stays bounded.** Rows first staged at or below the latest verified checkpoint's coversSeq (capped at the - store's own position) are pruned at the start of each pull (`pruneSeenTxns`). +- **`_sync_seen_txn` is not pruned yet.** A transaction already seen can come back in a new segment above any checkpoint, for + example a retired replica's late segment or a rebind's re-push. If its first-delivery row had been dropped, it would apply + twice. `pruneSeenTxns` exists, but the cloud pull calls it only once the receive watermark (T13256) provides a floor the + applier refuses to go below. - **`pullSyncStream(opts)`** (`cloud/nexus-vault.ts`) wires `pullStream` to the journal client and the trusted device keys. It refuses a store bound to another replica than the link. With `sync.pull` off it refuses before any network call. A store whose last synced checkpoint is a vault snapshot from before the stream's journal genesis is refused, with nothing pulled (T13306). The changes folded into that genesis never travel diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index 99617058ac..412760df5c 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -1893,17 +1893,18 @@ async function pullSyncStreamImpl(opts: NexusVaultCommandOptions = {}): Promise< journal.verifyCheckpoint(from, key.signers); initialCursor = cursorFromCheckpoint(from); } - // Seen-txn rows past the latest verified checkpoint can no longer be re-delivered. - const headCheckpoint = checkpoints.find((c) => c.checkpointId === head.headCheckpointId) ?? null; - if (headCheckpoint) journal.verifyCheckpoint(headCheckpoint, key.signers); - const pruneSeenUpTo = - headCheckpoint === null ? null : Math.min(headCheckpoint.coversSeq, initialCursor.after); + // Seen-txn rows are never pruned here yet: a txn already seen can come + // back in a NEW segment above any checkpoint (a retired replica's late + // segment, a rebind re-pushing an upload that was stored but never + // recorded), and with its first-delivery row gone it would apply twice. + // Pruning waits for a floor the applier refuses below anyway, such as the + // receive watermark (T13256), and then prunes by that floor, not by + // stream seq. return pullStream(db, { scope: tableScopeOf(t), stream: t.streamId, replica: replicaId, initialCursor, - pruneSeenUpTo, pull: async (cursor) => { const page = await journal.pull( { diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts index cd66eb154f..7a8c394007 100644 --- a/packages/core/src/store/sync/pull.ts +++ b/packages/core/src/store/sync/pull.ts @@ -95,8 +95,9 @@ export interface PullStreamOptions { /** Environment for the `sync.pull` kill switch. @defaultValue process.env */ readonly env?: NodeJS.ProcessEnv; /** - * Drop seen-txn rows first staged at or below this stream seq (the latest - * verified checkpoint's coversSeq), or null to keep every row. + * Drop seen-txn rows first staged at or below this stream seq, or null to + * keep every row. Only a floor the applier refuses below is safe + * ({@link pruneSeenTxns}); the cloud pull passes none yet (T13256). */ readonly pruneSeenUpTo?: number | null; } @@ -205,9 +206,12 @@ function advance(cursor: StreamCursor, segs: readonly PulledStreamSegment[]): St } /** - * Drop the seen-txn rows of `stream` first staged at or below `seq`: past a - * verified checkpoint's coversSeq no segment re-delivers them (§3.1). Keeps - * `_sync_seen_txn` bounded. + * Drop the seen-txn rows of `stream` first staged at or below `seq`, keeping + * `_sync_seen_txn` bounded (§3.1). Only safe below a floor the applier + * refuses any txn under: a seen txn can return in a NEW segment above a + * checkpoint (a retired replica's late segment, a rebind's re-push), so a + * checkpoint's coversSeq alone is not such a floor. No caller prunes until + * that floor exists (T13256). * * @param db - The store. * @param stream - The stream. From 7c6889b8043c2209392f1a5b0c28f14172639f3e Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:51:46 -0700 Subject: [PATCH 17/23] fix(sync): a write between the join's row check and its cut refuses the join; a refused join restores capture (T13312 review) MED (#1967): planRepair runs outside the cut's transaction, so a write in that gap was captured live at seq <= cut and baselined as checkpoint state. Capture is on before the check, so any such write is a live capture: inside the cut's BEGIN IMMEDIATE the join now refuses on one (the store changed since its restore). Sealing turns on inside that transaction too. LOW: a refused join puts capture back as it found it, so a store that had it off does not pile up captures that refuse every retry. Tests (genesis.test.ts): a join of a meta-complete store; a write in the gap (via the now() hook) refuses, no cut, push off, the row unbaselined; a refused join leaves capture off and no capture triggers. Both mutations killed. Co-Authored-By: Claude Opus 5.5 --- .changeset/t13312-journal-join.md | 6 +- .../src/store/sync/__tests__/genesis.test.ts | 66 ++++++++++++++++++- packages/core/src/store/sync/genesis.ts | 41 ++++++++---- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/.changeset/t13312-journal-join.md b/.changeset/t13312-journal-join.md index dce06f66ed..9cd01f5539 100644 --- a/.changeset/t13312-journal-join.md +++ b/.changeset/t13312-journal-join.md @@ -21,8 +21,10 @@ there was no way in. - **A changed store is refused, never folded.** The join is refused with `E_NEXUS_SYNC_REFUSED` when: - any table's rows differ from the checkpoint's manifest; - any row no longer matches its restored row meta; - - a capture is still waiting to seal. - The remedy is to restore the checkpoint again (`cleo cloud restore --force`), then join. A refused join changes nothing. + - a capture is still waiting to seal; + - a write landed after the check, which is caught inside the cut's own transaction, so it is never baselined as + checkpoint state. + The remedy is to restore the checkpoint again (`cleo cloud restore --force`), then join. A refused join changes nothing: capture is left as it was found, and sealing stays off. - **A store that never restored the journal checkpoint** gets `E_NEXUS_SYNC_STREAM_JOURNALED`, naming the restore-then-join path. The pull refusal for a store behind the journal genesis (T13306) names the same path. - `sync.push` and `sync.pull` stay unreleased. diff --git a/packages/core/src/store/sync/__tests__/genesis.test.ts b/packages/core/src/store/sync/__tests__/genesis.test.ts index 7daac48f44..e5169f68e1 100644 --- a/packages/core/src/store/sync/__tests__/genesis.test.ts +++ b/packages/core/src/store/sync/__tests__/genesis.test.ts @@ -25,7 +25,7 @@ import { ROW_IDENTITY_RECIPE_KEY, } from '../../row-identity.js'; import { finishCaptureFrame, openCaptureFrame, setCaptureEnabled } from '../capture.js'; -import { isSyncFlagOn, setSyncFlag } from '../flags.js'; +import { isSyncFlagOn, readSyncFlags, setSyncFlag } from '../flags.js'; import { cutGenesis, cutGenesisWithSnapshot, @@ -35,6 +35,7 @@ import { GenesisRacedError, genesisCutOf, genesisPending, + joinStream, UNDO_ENABLED_KEY, } from '../genesis.js'; import { streamStarted } from '../repair.js'; @@ -529,3 +530,66 @@ describe('a cut that crashed before its snapshot (T13301), and per-cut folds (LO expect(isSyncFlagOn(db, 'sync.push', {})).toBe(true); }); }); + +describe('joining a journaled stream (T13312)', () => { + /** A store whose rows all carry row meta (a cut), with the cut forgotten: what a restore leaves to join. */ + async function restoredLike(): Promise { + const { db } = await store(); + expect(cut(db).refused).toBeNull(); + db.exec( + "DELETE FROM _sync_meta WHERE key LIKE 'genesis_%' OR key IN ('undo_enabled', 'sync.push', 'sync.pull')", + ); + expect(genesisCutOf(db, STREAM)).toBeUndefined(); + return db; + } + const join = (db: DatabaseSync, now: () => number = () => ++clock) => + joinStream(db, { + scope: 'project', + stream: STREAM, + cursor: { after: 0, knowsAllReplicas: true, replicas: {} }, + now, + allowUnreleased: true, + }); + + it('joins a store whose rows match their meta: cut at the capture position, push and pull on', async () => { + const db = await restoredLike(); + const r = join(db); + expect(r.refused).toBeNull(); + expect(genesisCutOf(db, STREAM)).toBe(r.cut); + expect(isSyncFlagOn(db, 'sync.push', {})).toBe(true); + expect(isSyncFlagOn(db, 'sync.pull', {})).toBe(true); + }); + + it('a write between the row check and the cut refuses the join; its row is never baselined as checkpoint state', async () => { + const db = await restoredLike(); + let raced = false; + const r = join(db, () => { + // The last step before the cut's transaction opens. + if (!raced) { + raced = true; + db.exec(addTask('T5')); + } + return ++clock; + }); + expect(raced).toBe(true); + expect(r.refused).toMatch(/a write landed while the join checked the store/); + expect(genesisCutOf(db, STREAM)).toBeUndefined(); + expect(isSyncFlagOn(db, 'sync.push', {})).toBe(false); + expect(n(db, "SELECT count(*) AS n FROM _sync_row_meta WHERE uid = 'uid-T5'")).toBe(0); + }); + + it('a refused join leaves capture off when it was off', async () => { + const db = await restoredLike(); + setCaptureEnabled(db, 'project', false, { schemaRoot: SYNC_SCHEMA }); + db.exec("UPDATE tasks_tasks SET title = 'changed' WHERE id = 'T0'"); // rows no longer match meta + const r = join(db); + expect(r.refused).toMatch(/without matching row meta/); + expect(readSyncFlags(db)['sync.capture']).toBe(false); + expect( + n( + db, + "SELECT count(*) AS n FROM sqlite_master WHERE type = 'trigger' AND name LIKE '_sync_cap%'", + ), + ).toBe(0); + }); +}); diff --git a/packages/core/src/store/sync/genesis.ts b/packages/core/src/store/sync/genesis.ts index 46bdfabb0e..34fb56ddd3 100644 --- a/packages/core/src/store/sync/genesis.ts +++ b/packages/core/src/store/sync/genesis.ts @@ -37,7 +37,7 @@ import { UID_COLUMN, } from '../row-identity.js'; import { captureTriggerDrift, setCaptureEnabled, syncSetTables } from './capture.js'; -import { isSyncFlagOn, setSyncFlag } from './flags.js'; +import { isSyncFlagOn, readSyncFlags, setSyncFlag } from './flags.js'; import { type StreamCursor, writeStreamCursor } from './pull.js'; import { baselineRowMeta, planRepair } from './repair.js'; import { activeReplica, persistStoreSeq } from './replica.js'; @@ -594,8 +594,10 @@ const JOIN_BENIGN_SKIPS: ReadonlySet = new Set([ * * Every sync-set row must carry meta that matches it (the restore carried * the checkpoint's merge state, and nothing changed since); a row without - * meta, or one whose content left its meta behind, refuses the join. Then, - * capture and seal turned on, in ONE `BEGIN IMMEDIATE`: the restore's + * meta, or one whose content left its meta behind, refuses the join, and so + * does a write captured after the check (seen inside the cut's transaction). + * A refusal leaves capture as it found it. Then, in ONE `BEGIN IMMEDIATE` + * that also turns sealing on: the restore's * suspect marks are cleared (the check just proved the rows clean), the * ledgers set, `genesis_cut:` records the join point (the stream has * started here: T13217's meta-less-row rule applies from now), `undo_enabled`, @@ -625,32 +627,47 @@ export function joinStream( const replica = activeReplica(db, o.scope)?.replicaId; if (!replica) return { stream: o.stream, refused: 'no bound replica', cut: null }; const now = o.now ?? Date.now; - // Capture installs the triggers the row check needs; sealing waits for the - // check, so a refused join seals nothing. + // Capture installs the triggers the row check needs, and from then on + // every write is a live capture, which the cut below checks for. A refusal + // puts capture back as it was, so a store that had it off does not pile up + // captures that would refuse every retry. + const captureWasOn = readSyncFlags(db)['sync.capture']; setCaptureEnabled(db, o.scope, true); + const refuse = (why: string): JoinStreamReport => { + if (!captureWasOn) setCaptureEnabled(db, o.scope, false); + return { stream: o.stream, refused: why, cut: null }; + }; // Every row must still match the meta the checkpoint carried. const view = sealerRowView(db, o.scope); for (const table of syncSetTables(o.scope)) { const plan = planRepair(db, o.scope, table, view); if (plan.skipped !== null && !JOIN_BENIGN_SKIPS.has(plan.skipped)) { - return { stream: o.stream, refused: `${table}: ${plan.skipped}`, cut: null }; + return refuse(`${table}: ${plan.skipped}`); } if (plan.skipped !== null) continue; const off = plan.inserts.length + plan.updates.length + plan.deletes.length + plan.unbaselined.length; if (off > 0) { - return { - stream: o.stream, - refused: `${table}: ${off} row(s) without matching row meta; the store is not the journal checkpoint it restored (restore it again)`, - cut: null, - }; + return refuse( + `${table}: ${off} row(s) without matching row meta; the store is not the journal checkpoint it restored (restore it again)`, + ); } } - setSyncFlag(db, 'sync.seal', true, o.allowUnreleased ? { allowUnreleased: true } : {}); const at = now(); const atIso = new Date(at).toISOString(); db.exec('BEGIN IMMEDIATE'); try { + // The check ran outside this transaction: a write since then (another + // process, or this one) is a live capture, and the cut would baseline its + // row as the checkpoint's. The store changed since its restore: refuse. + if (db.prepare("SELECT 1 FROM _sync_capture WHERE state = 'live' LIMIT 1").get()) { + db.exec('ROLLBACK'); + return refuse( + 'a write landed while the join checked the store; it is no longer the journal checkpoint it restored (restore it again)', + ); + } + // Sealing starts with the join, never before the check. + setSyncFlag(db, 'sync.seal', true, o.allowUnreleased ? { allowUnreleased: true } : {}); db.prepare("DELETE FROM _sync_meta WHERE key LIKE 'suspect:%'").run(); // Ledgers (count + held) and baseline keys; every row already has meta. for (const table of syncSetTables(o.scope)) baselineRowMeta(db, o.scope, table, replica, at); From 0f7ec518b4e81e5595f971ffc77f9ba27ddc9c73 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:52:10 -0700 Subject: [PATCH 18/23] test(sync): idle rounds also seal and receive nothing (T13256 review nit) Co-Authored-By: Claude Opus 5.5 --- packages/core/src/cloud/__tests__/nexus-vault.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index 96ce889583..ad434e3ca1 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5431,8 +5431,15 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { for (let round = 0; round < 4; round++) { const ra = await sync(a); const rb = await sync(b); - expect(ra?.sent, `round ${round}: A sent`).toBe(0); - expect(rb?.sent, `round ${round}: B sent`).toBe(0); + // Nothing sealed (no echo re-sealed and netted away), sent or received. + for (const [who, r] of [ + ['A', ra], + ['B', rb], + ] as const) { + expect(r?.sealed, `round ${round}: ${who} sealed`).toBe(0); + expect(r?.sent, `round ${round}: ${who} sent`).toBe(0); + expect(r?.received, `round ${round}: ${who} received`).toBe(0); + } } expect(fake.stream(STREAM).segments).toHaveLength(settled); const titles = "SELECT id, title FROM tasks_tasks WHERE id IN ('T1', 'T2') ORDER BY id"; From f00e14a86194246517e31637bc16d16c11341eaa Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Tue, 6 Oct 2026 20:56:05 -0700 Subject: [PATCH 19/23] refactor(sync): cloud sync classifies skipped legs by a typed refusal kind (T13315) #1962 review nit: SKIP_REASONS matched refusal message text, so a reworded refusal would silently turn a skip into a refusal. Push and pull reports now carry refusedKind; classifySyncLegs decides skipped, refused, disabled and whether the pull position is meaningful from the kinds alone. Tests: reworded messages still skip; a skip-looking message with a non-skip kind is a refusal. Two mutations killed. Co-Authored-By: Claude Opus 5.5 --- .changeset/t13315-typed-skip.md | 19 +++++ .../src/cloud/__tests__/nexus-vault.test.ts | 38 +++++++++ packages/core/src/cloud/nexus-vault.ts | 81 +++++++++++++------ packages/core/src/store/sync/pull.ts | 8 ++ packages/core/src/store/sync/push.ts | 30 ++++++- 5 files changed, 151 insertions(+), 25 deletions(-) create mode 100644 .changeset/t13315-typed-skip.md diff --git a/.changeset/t13315-typed-skip.md b/.changeset/t13315-typed-skip.md new file mode 100644 index 0000000000..5234810ec2 --- /dev/null +++ b/.changeset/t13315-typed-skip.md @@ -0,0 +1,19 @@ +--- +id: t13315-typed-skip +tasks: [T13315] +kind: refactor +summary: "`cleo cloud sync` sorts skipped legs by a typed refusal kind instead of matching refusal message text" +--- + +`cleo cloud sync` used to decide whether a push or pull leg was skipped or refused by matching the refusal's message +against regexes. Rewording a refusal would silently turn a skip (`sync.push` off, no genesis yet) into a reported refusal. + +- **Push results** (`PushStreamReport`) now carry `refusedKind`, which is one of: + - `schema-missing` + - `push-off` + - `no-genesis` + - `genesis-pending` + - `store-behind` +- **Pull results** (`PullStreamReport`) now carry `refusedKind`, which is `pull-off` or `segment`. +- **Classification** is done by `classifySyncLegs`, using only these kinds. The message regexes are removed. A stream is + `disabled` when both legs are off. The pull position is reported unless the pull leg was skipped. diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index ad434e3ca1..cf02856d74 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -119,6 +119,7 @@ import { hasUnsyncedNexusBackup, runNexusFirstRun } from '../nexus-first-run.js' import { linkProjectToNexus } from '../nexus-link.js'; import { listNexusNamedProjects, resolveNexusProjectRef } from '../nexus-project-names.js'; import { + classifySyncLegs, cloudSync, enableSyncPush, nexusVaultStatus, @@ -5563,3 +5564,40 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { expect(fake.stream(STREAM).checkpoints).toHaveLength(0); }); }); + +describe('cloud sync leg classification (T13315)', () => { + const none = { refused: null, refusedKind: null } as const; + + it('classifies by refusal kind, so a reworded skip message is still a skip', () => { + const r = classifySyncLegs( + { refused: 'push is switched off for this store', refusedKind: 'push-off' }, + { refused: 'pulling is disabled', refusedKind: 'pull-off' }, + ); + expect(r).toEqual({ + skipped: ['push is switched off for this store', 'pulling is disabled'], + refused: [], + disabled: true, + pullSkipped: true, + }); + }); + + it('push before genesis is skipped; a store behind the server and a refused segment are refusals', () => { + expect( + classifySyncLegs({ refused: 'no cut yet', refusedKind: 'no-genesis' }, none), + ).toMatchObject({ skipped: ['no cut yet'], refused: [], disabled: false, pullSkipped: false }); + expect( + classifySyncLegs({ refused: 'not stored', refusedKind: 'genesis-pending' }, none).skipped, + ).toEqual(['not stored']); + expect( + classifySyncLegs( + { refused: 'behind', refusedKind: 'store-behind' }, + { refused: 'bad segment', refusedKind: 'segment' }, + ), + ).toMatchObject({ skipped: [], refused: ['behind', 'bad segment'], disabled: false }); + // A message that reads like a skip but carries no skip kind is a refusal. + expect( + classifySyncLegs({ refused: 'sync.push is off', refusedKind: 'schema-missing' }, none) + .refused, + ).toEqual(['sync.push is off']); + }); +}); diff --git a/packages/core/src/cloud/nexus-vault.ts b/packages/core/src/cloud/nexus-vault.ts index ddd72353bf..e62136d432 100644 --- a/packages/core/src/cloud/nexus-vault.ts +++ b/packages/core/src/cloud/nexus-vault.ts @@ -116,7 +116,7 @@ import { readGenesisCut, } from '../store/sync/genesis.js'; import { type PullStreamReport, pullStream, readStreamCursor } from '../store/sync/pull.js'; -import { type PushStreamReport, pushStream } from '../store/sync/push.js'; +import { type PushRefusal, type PushStreamReport, pushStream } from '../store/sync/push.js'; import { replayPinOf } from '../store/sync/replay-pin.js'; import { activeReplica, readActiveReplicaId } from '../store/sync/replica.js'; import { ReplicaRegistry } from '../store/sync/replica-registry.js'; @@ -2054,13 +2054,52 @@ const NOT_ATTACHED_CODES: ReadonlySet = new Set([ 'E_NEXUS_NOT_A_PROJECT', ]); -/** Push or pull refusals that mean "nothing to do here", not a failure (LOW-3 on #1962). */ -const SKIP_REASONS: readonly RegExp[] = [ - /^sync\.push is off$/, - /^sync\.pull is off$/, - /has no genesis cut on this store/, - /genesis checkpoint is not stored yet/, -]; +/** Push refusals that mean "nothing to do here", not a failure (LOW-3 on #1962). */ +const PUSH_SKIPS: ReadonlySet = new Set(['push-off', 'no-genesis', 'genesis-pending']); + +/** How one stream's push and pull legs ended, classified by refusal kind (T13315). */ +export interface SyncLegOutcome { + /** Refusals that mean "nothing to do here": a flag off, or push before this store's genesis. */ + readonly skipped: string[]; + /** Every other refusal. */ + readonly refused: string[]; + /** Both flags off: the stream is disabled. */ + readonly disabled: boolean; + /** The pull leg was skipped, so its position says nothing. */ + readonly pullSkipped: boolean; +} + +/** + * Classify a stream's push and pull results (T13315). Classification uses + * each report's typed `refusedKind`, never its message, so rewording a + * refusal never turns a skip into a refusal. + * + * @param push - The push leg's `refused` and `refusedKind`. + * @param pull - The pull leg's `refused` and `refusedKind`. + * @returns Skipped and refused messages, and whether the stream is disabled. + */ +export function classifySyncLegs( + push: Pick, + pull: Pick, +): SyncLegOutcome { + const pushSkipped = push.refusedKind !== null && PUSH_SKIPS.has(push.refusedKind); + const pullSkipped = pull.refusedKind === 'pull-off'; + const skipped: string[] = []; + const refused: string[] = []; + for (const [msg, skip] of [ + [push.refused, pushSkipped], + [pull.refused, pullSkipped], + ] as const) { + if (msg === null) continue; + (skip ? skipped : refused).push(msg); + } + return { + skipped, + refused, + disabled: push.refusedKind === 'push-off' && pullSkipped, + pullSkipped, + }; +} /** The code and message of a failure, for a stream result. */ function failureText(err: unknown): string { @@ -2119,22 +2158,18 @@ async function syncOneStream( duplicates: push.duplicates, }); } - const isSkip = (r: string | null): boolean => r !== null && SKIP_REASONS.some((x) => x.test(r)); - const skipped = [push.refused, pull.refused].filter((r): r is string => isSkip(r)); - const refused = [push.refused, pull.refused].filter((r): r is string => r !== null && !isSkip(r)); - const pushOff = push.refused === 'sync.push is off'; - const pullOff = pull.refused === 'sync.pull is off'; + const legs = classifySyncLegs(push, pull); + const { skipped, refused } = legs; const apply = pull.apply; return syncStreamResult(scope, { streamId, - status: - pushOff && pullOff - ? 'disabled' - : refused.length > 0 - ? 'refused' - : push.clockAhead - ? 'paused' - : 'synced', + status: legs.disabled + ? 'disabled' + : refused.length > 0 + ? 'refused' + : push.clockAhead + ? 'paused' + : 'synced', refused: refused.length > 0 ? refused.join('; ') : null, skipped, sealed: push.sealed, @@ -2147,8 +2182,8 @@ async function syncOneStream( applied: apply?.applied ?? 0, held: (apply?.pending ?? 0) + (apply?.heldSkew ?? 0) + (apply?.refusedSchema ?? 0), conflicts: apply?.conflict ?? 0, - after: pull.refused === null || !isSkip(pull.refused) ? pull.after : null, - head: pull.refused === null || !isSkip(pull.refused) ? pull.head : null, + after: legs.pullSkipped ? null : pull.after, + head: legs.pullSkipped ? null : pull.head, }); } diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts index 391f0b3edc..4ac838ff2e 100644 --- a/packages/core/src/store/sync/pull.ts +++ b/packages/core/src/store/sync/pull.ts @@ -112,6 +112,12 @@ export interface PullStreamReport { * segment was staged and applied, and the cursor stops just before it. */ readonly refused: string | null; + /** + * The kind of `refused` (callers branch on this; the message may be + * reworded): `pull-off`, or `segment` for a refused segment. Null when + * nothing was refused. + */ + readonly refusedKind: 'pull-off' | 'segment' | null; /** Segments received, vault deltas passed over, transactions staged, re-deliveries skipped. */ readonly segments: number; readonly vaultDeltas: number; @@ -267,6 +273,7 @@ export async function pullStream( return { stream: o.stream, refused: 'sync.pull is off', + refusedKind: 'pull-off', segments: 0, vaultDeltas: 0, staged: 0, @@ -355,6 +362,7 @@ export async function pullStream( return { stream: o.stream, refused: refusedSegment, + refusedKind: refusedSegment === null ? null : 'segment', segments, vaultDeltas, staged, diff --git a/packages/core/src/store/sync/push.ts b/packages/core/src/store/sync/push.ts index 3db8df7a9a..5073802521 100644 --- a/packages/core/src/store/sync/push.ts +++ b/packages/core/src/store/sync/push.ts @@ -88,11 +88,29 @@ export interface PushStreamOptions { readonly maxDriftMs?: number; } +/** + * Why {@link pushStream} sent nothing, as a kind callers branch on (the + * message in `refused` is for people and may be reworded): + * - `schema-missing`: the sync schema is not installed; + * - `push-off`: `sync.push` is off; + * - `no-genesis`: this store has no genesis cut on the stream; + * - `genesis-pending`: the genesis checkpoint is not stored yet; + * - `store-behind`: the server holds a later segment of this replica. + */ +export type PushRefusal = + | 'schema-missing' + | 'push-off' + | 'no-genesis' + | 'genesis-pending' + | 'store-behind'; + /** What {@link pushStream} did. */ export interface PushStreamReport { readonly stream: string; /** Why nothing was sent, or null. */ readonly refused: string | null; + /** The kind of `refused`, or null when nothing was refused. */ + readonly refusedKind: PushRefusal | null; /** Push paused because this device's clock is ahead of the server's. */ readonly clockAhead: boolean; /** Transactions sealed, segments packed and persisted, segments the server stored (`duplicates` of them retries). */ @@ -107,6 +125,7 @@ export interface PushStreamReport { const empty = (stream: string, fields: Partial): PushStreamReport => ({ stream, refused: null, + refusedKind: null, clockAhead: false, sealed: 0, built: 0, @@ -145,16 +164,22 @@ export async function pushStream( const env = o.env ?? process.env; const now = o.now ?? Date.now; if (!hasTable(db, '_sync_segment')) - return empty(o.stream, { refused: 'sync schema not installed' }); - if (!isSyncFlagOn(db, 'sync.push', env)) return empty(o.stream, { refused: 'sync.push is off' }); + return empty(o.stream, { + refused: 'sync schema not installed', + refusedKind: 'schema-missing', + }); + if (!isSyncFlagOn(db, 'sync.push', env)) + return empty(o.stream, { refused: 'sync.push is off', refusedKind: 'push-off' }); if (genesisCutOf(db, o.stream) === undefined) { return empty(o.stream, { refused: `${o.stream} has no genesis cut on this store: run \`cleo sync enable push\``, + refusedKind: 'no-genesis', }); } if (genesisPending(db, o.stream)) { return empty(o.stream, { refused: `${o.stream}'s genesis checkpoint is not stored yet: run \`cleo sync enable push\` again`, + refusedKind: 'genesis-pending', }); } const atIso = new Date(now()).toISOString(); @@ -171,6 +196,7 @@ export async function pushStream( if (o.serverLastReplicaSeq !== null && (local === null || o.serverLastReplicaSeq > local)) { return empty(o.stream, { refused: `the server holds replicaSeq ${o.serverLastReplicaSeq} of replica ${o.replica} on ${o.stream}, but this store persisted only ${local ?? 'none'}: the store is behind (restored or copied) and must rebind to a new replica (T12753)`, + refusedKind: 'store-behind', }); } From f36a293e7273ebf6f90af1e8c9787e518da72d6d Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Wed, 7 Oct 2026 07:26:57 -0700 Subject: [PATCH 20/23] feat(cloud): cloud status reports the seen-txn ledger size per store (T13317) Seen-txn pruning stays off (#1959) until a proven refusal floor exists (T13318). CloudStatusSyncStream.seenTxns {rows, bytes, byStream} from seenTxnReport (stream + txn text + 8-byte seq per row); the human clause names rows and KiB once non-empty. Tests: per-stream count and byte estimate, a read never prunes, the clause text; mutations on the byte estimate and the clause both fail a test. Co-Authored-By: Claude Opus 5.5 --- .changeset/t13317-seen-txn-status.md | 15 +++++++++ .../__tests__/nexus-cloud-cli-sync.test.ts | 17 ++++++++++ packages/cleo/src/cli/lib/nexus-cloud-cli.ts | 12 ++++++- packages/contracts/src/nexus-cloud.ts | 16 +++++++++ .../cloud/__tests__/cloud-sync-status.test.ts | 20 +++++++++++ packages/core/src/cloud/nexus-cloud-status.ts | 3 ++ packages/core/src/store/sync/pull.ts | 33 +++++++++++++++++++ 7 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 .changeset/t13317-seen-txn-status.md diff --git a/.changeset/t13317-seen-txn-status.md b/.changeset/t13317-seen-txn-status.md new file mode 100644 index 0000000000..7abc5d305d --- /dev/null +++ b/.changeset/t13317-seen-txn-status.md @@ -0,0 +1,15 @@ +--- +id: t13317-seen-txn-status +tasks: [T13317] +kind: feat +summary: "`cleo cloud status` reports the seen-transaction ledger's size per store, since nothing prunes it yet" +--- + +The pull records each transaction it stages in `_sync_seen_txn`, so a re-delivered transaction is never applied twice. +Nothing prunes that ledger yet. A seen transaction can come back in a new segment above any checkpoint, so pruning +waits for a proven refusal floor (T13318). In the meantime, its growth is now visible. + +- `cleo cloud status` now includes `seenTxns` in each store's sync block: `{rows, bytes, byStream}`. + - `bytes` counts the stored stream and txn text plus an 8-byte seq per row, without SQLite page overhead. +- The human line adds ` seen txn(s), about KiB` once the ledger holds rows. +- Nothing prunes `_sync_seen_txn`. diff --git a/packages/cleo/src/cli/lib/__tests__/nexus-cloud-cli-sync.test.ts b/packages/cleo/src/cli/lib/__tests__/nexus-cloud-cli-sync.test.ts index bd5f5397d2..4a493ce90d 100644 --- a/packages/cleo/src/cli/lib/__tests__/nexus-cloud-cli-sync.test.ts +++ b/packages/cleo/src/cli/lib/__tests__/nexus-cloud-cli-sync.test.ts @@ -18,9 +18,12 @@ const noUndo: CloudStatusSyncStream['undo'] = { exceededAt: null, }; +const noSeen: CloudStatusSyncStream['seenTxns'] = { rows: 0, bytes: 0, byStream: {} }; + const stream = ( held: CloudStatusSyncStream['held'], undo: CloudStatusSyncStream['undo'] = noUndo, + seenTxns: CloudStatusSyncStream['seenTxns'] = noSeen, ): CloudStatusSyncStream => ({ scope: 'project', stream: null, @@ -34,6 +37,7 @@ const stream = ( suspectTables: [], held, undo, + seenTxns, unsentOps: { known: false, needs: 'T12343', reason: 'not yet' }, lastPushedSeq: unknown, lastPulledSeq: unknown, @@ -86,3 +90,16 @@ describe('cloud status sync clause (T13271)', () => { expect(syncStreamClause(stream(none))).not.toContain('undo'); }); }); + +describe('cloud status sync clause: the seen-txn ledger (T13317)', () => { + const none = { count: 0, oldestAt: null, long: [], warnDays: 7 }; + it('names the seen-txn rows and their size, and says nothing while empty', () => { + expect( + syncStreamClause(stream(none, noUndo, { rows: 1200, bytes: 30_720, byStream: { s: 1200 } })), + ).toContain('1200 seen txn(s), about 30 KiB'); + expect( + syncStreamClause(stream(none, noUndo, { rows: 3, bytes: 90, byStream: { s: 3 } })), + ).toContain('3 seen txn(s), about 0.1 KiB'); + expect(syncStreamClause(stream(none))).not.toContain('seen txn'); + }); +}); diff --git a/packages/cleo/src/cli/lib/nexus-cloud-cli.ts b/packages/cleo/src/cli/lib/nexus-cloud-cli.ts index 7dbb5bc164..94ebb016d0 100644 --- a/packages/cleo/src/cli/lib/nexus-cloud-cli.ts +++ b/packages/cleo/src/cli/lib/nexus-cloud-cli.ts @@ -86,7 +86,8 @@ function syncSummary(r: CloudStatusResult): string { /** * One store's clause of the `cleo cloud status` line: its flags, unsealed - * and sealed counts, quarantined captures and held writes (§3.5 Rule 5). + * and sealed counts, quarantined captures, the seen-txn ledger's size + * (T13317) and held writes (§3.5 Rule 5). * * @param st - The store's sync block. * @returns The clause, with a leading space. @@ -113,6 +114,9 @@ export function syncStreamClause(st: CloudStatusSyncStream): string { ? `undo at ${Math.round((st.undo.bytes / st.undo.budget) * 100)}% of its budget: pull to drain it` : `undo budget exceeded${st.undo.exceededAt ? ` since ${st.undo.exceededAt}` : ''}: a rebind runs at the next pull`, ]), + ...(st.seenTxns.rows > 0 + ? [`${st.seenTxns.rows} seen txn(s), about ${formatKiB(st.seenTxns.bytes)}`] + : []), ...(st.held.count > 0 ? [ `${st.held.count} held by a rebase${long > 0 ? ` (${long} older than ${st.held.warnDays} days)` : ''}`, @@ -123,6 +127,12 @@ export function syncStreamClause(st: CloudStatusSyncStream): string { return ` Sync (${st.scope}${st.stream ? ` ${st.stream}` : ''}): ${parts.join('; ')}.`; } +/** Bytes as KiB, one decimal under 10 KiB. */ +function formatKiB(bytes: number): string { + const kib = bytes / 1024; + return `${kib < 10 ? kib.toFixed(1) : Math.round(kib)} KiB`; +} + /** * Run a `cleo cloud` read: call it, print warnings, emit, or fail with a LAFS error. * diff --git a/packages/contracts/src/nexus-cloud.ts b/packages/contracts/src/nexus-cloud.ts index 052297c6c8..112de96d6a 100644 --- a/packages/contracts/src/nexus-cloud.ts +++ b/packages/contracts/src/nexus-cloud.ts @@ -565,6 +565,20 @@ export interface CloudStatusUndoBudget { readonly exceededAt: string | null; } +/** + * The pull's seen-transaction ledger (`_sync_seen_txn`, T13317). Nothing + * prunes it yet: a seen txn can return in a new segment above any checkpoint, + * so pruning waits for a proven refusal floor (T13318). + */ +export interface CloudStatusSeenTxns { + /** Rows held, every stream. */ + readonly rows: number; + /** Estimated bytes: the stored text and integer payload, without SQLite page overhead. */ + readonly bytes: number; + /** Rows per stream. */ + readonly byStream: Readonly>; +} + export interface CloudStatusSyncStream { /** Which store. */ readonly scope: 'project' | 'global'; @@ -590,6 +604,8 @@ export interface CloudStatusSyncStream { readonly held: CloudStatusHeld; /** Undo held against its budget (§3.5 Rule 2, D5). */ readonly undo: CloudStatusUndoBudget; + /** The seen-transaction ledger's size; it grows until pruning has a proven floor (T13317). */ + readonly seenTxns: CloudStatusSeenTxns; /** Sealed ops not yet handed to the outbox for sending. */ readonly unsentOps: CloudSyncFact; /** The last sequence this store pushed to the server. */ diff --git a/packages/core/src/cloud/__tests__/cloud-sync-status.test.ts b/packages/core/src/cloud/__tests__/cloud-sync-status.test.ts index 6ec86bda21..8bf4d718b9 100644 --- a/packages/core/src/cloud/__tests__/cloud-sync-status.test.ts +++ b/packages/core/src/cloud/__tests__/cloud-sync-status.test.ts @@ -101,6 +101,7 @@ describe('cloud status sync block (T12998)', () => { unsealedOps: 0, lastSealedSeq: null, quarantined: {}, + seenTxns: { rows: 0, bytes: 0, byStream: {} }, }); }); @@ -152,6 +153,25 @@ describe('cloud status sync block (T12998)', () => { expect(project?.quarantined).toEqual({ tasks_tasks: 1 }); }); + it('reports the seen-txn ledger per stream with its estimated bytes; nothing prunes it (T13317)', async () => { + const db = await store(true); + const ins = db.prepare('INSERT INTO _sync_seen_txn (stream, txn, seq) VALUES (?, ?, ?)'); + ins.run('project:p1', 'r1:1', 1); + ins.run('project:p1', 'r1:2', 2); + ins.run('home:u1', 'r2:1', 1); + const { project } = await projectBlock(); + // stream + txn text + an 8-byte seq per row. + const bytes = + 2 * ('project:p1'.length + 'r1:1'.length + 8) + ('home:u1'.length + 'r2:1'.length + 8); + expect(project?.seenTxns).toEqual({ + rows: 3, + bytes, + byStream: { 'home:u1': 1, 'project:p1': 2 }, + }); + // A read never prunes: the next read sees the same rows. + expect((await projectBlock()).project?.seenTxns.rows).toBe(3); + }); + it('is read-only: the store file is unchanged by a read', async () => { const db = await store(true); addTask(db, 'T1'); diff --git a/packages/core/src/cloud/nexus-cloud-status.ts b/packages/core/src/cloud/nexus-cloud-status.ts index e0d910c4e3..65feb8b1a4 100644 --- a/packages/core/src/cloud/nexus-cloud-status.ts +++ b/packages/core/src/cloud/nexus-cloud-status.ts @@ -287,6 +287,7 @@ export async function readStoreSyncStream( { suspectTables }, { holdsReport }, { undoBudget }, + { seenTxnReport }, ] = await Promise.all([ import('../store/sync/flags.js'), import('../store/sync/schema.js'), @@ -294,6 +295,7 @@ export async function readStoreSyncStream( import('../store/sync/structural.js'), import('../store/sync/held.js'), import('../store/sync/sequencing.js'), + import('../store/sync/pull.js'), ]); const holds = holdsReport(db, Date.now()); const flags = readSyncFlags(db); @@ -344,6 +346,7 @@ export async function readStoreSyncStream( warnDays: holds.warnDays, }, undo: undoBudget(db), + seenTxns: seenTxnReport(db), unsentOps: UNSENT_UNKNOWN, lastPushedSeq: needsPush('the last pushed sequence'), lastPulledSeq: needsPush('the last pulled sequence'), diff --git a/packages/core/src/store/sync/pull.ts b/packages/core/src/store/sync/pull.ts index 4ac838ff2e..3f5f538664 100644 --- a/packages/core/src/store/sync/pull.ts +++ b/packages/core/src/store/sync/pull.ts @@ -233,6 +233,39 @@ function advance(cursor: StreamCursor, segs: readonly PulledStreamSegment[]): St }; } +/** Bytes counted for a seen-txn row's `seq` integer. */ +const SEEN_SEQ_BYTES = 8; + +/** + * The size of `_sync_seen_txn` (T13317), for `cleo cloud status`: rows per + * stream and the estimated payload bytes (stream and txn text plus the seq + * integer, without SQLite's page overhead). Read-only. + * + * @param db - The store. + * @returns Zeroes when the table is not installed. + */ +export function seenTxnReport(db: DatabaseSync): { + rows: number; + bytes: number; + byStream: Record; +} { + if (!hasTable(db, '_sync_seen_txn')) return { rows: 0, bytes: 0, byStream: {} }; + const byStream: Record = {}; + let rows = 0; + let bytes = 0; + for (const r of db + .prepare( + `SELECT stream, count(*) AS n, sum(octet_length(stream) + octet_length(txn) + ${SEEN_SEQ_BYTES}) AS b + FROM _sync_seen_txn GROUP BY stream ORDER BY stream`, + ) + .all() as Array<{ stream: string; n: number; b: number | null }>) { + byStream[r.stream] = r.n; + rows += r.n; + bytes += r.b ?? 0; + } + return { rows, bytes, byStream }; +} + /** * Drop the seen-txn rows of `stream` first staged at or below `seq`, keeping * `_sync_seen_txn` bounded (§3.1). Only safe below a floor the applier From dafeaa7949f59f6874458224b89f367f0555a77b Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Wed, 7 Oct 2026 10:00:18 -0700 Subject: [PATCH 21/23] test(sync): relink B after the rebinding restore in the T13306 remedy test Main's T13109 binds a new replica when a vault restore places a store; the CLI's relink attaches it, which this helper machine does by hand. Co-Authored-By: Claude Opus 5.5 --- packages/core/src/cloud/__tests__/nexus-vault.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/core/src/cloud/__tests__/nexus-vault.test.ts b/packages/core/src/cloud/__tests__/nexus-vault.test.ts index bb15dfd3dd..c39b8ee73c 100644 --- a/packages/core/src/cloud/__tests__/nexus-vault.test.ts +++ b/packages/core/src/cloud/__tests__/nexus-vault.test.ts @@ -5653,6 +5653,13 @@ describe('sync enable push: the genesis checkpoint (T12343 S4-1b)', () => { // The remedy works: B restores the journal checkpoint, joins, and holds A's folded change. const restored = await on(b, () => restoreNexusVault(vopts(b, { mode: 'pull', force: true }))); expect(restored.status).toBe('restored'); + // The placed store is a new instance bound to a new replica (T13109); `cleo cloud pull` + // relinks it (the CLI's relink), which this machine helper does by hand. + const rebound = restored.replica?.current; + expect(rebound).toBeDefined(); + b.replicaId = rebound ?? ''; + fake.replicas.get(REMOTE_PROJECT)?.set(b.replicaId, DEVICE_B); + link(b); expect((await on(b, () => enableSyncPush(vopts(b, { allowUnreleased: true })))).status).toBe( 'joined', ); From d0f7960590e422cc6f8c87d5acc8b013ea71d3a0 Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Wed, 7 Oct 2026 10:20:04 -0700 Subject: [PATCH 22/23] fix(sync): a genesis cut whose marker went stale or was taken over is undone; markers release only their own (T13296 merge-up delta) team-lead on #1953's merge-up: main's 1 h marker max age now applies to the genesis marker, and a stale marker lets writers in mid-snapshot. Measured the whole marker-held span on scratch copies (sandboxed HOME/CLEO_HOME): cleocode 1.3 GB 58-63 s (cut 2.3-2.4 s, export 45-49 s, extract+manifest 10-12 s), claude-todo 2.2-2.8 s. The 1 h floor is over 50x the worst case. - holdRestoreMarker: {release, intact}; release removes the marker only while it is still this holder's (content match); intact = ours and younger than RESTORE_MARKER_MAX_AGE_MS. writeRestoreMarker wraps it. - cutGenesisWithSnapshot: after the snapshot and race check, a marker that is no longer intact undoes the cut and throws GenesisRacedError (E_SYNC_GENESIS_MARKER_LOST) before the caller records or uploads. Tests: a takeover mid-snapshot undoes the cut and leaves the new holder's marker; a snapshot outliving the window undoes the cut. Three mutations killed (no intact check, unconditional release, no window). Co-Authored-By: Claude Opus 5.5 --- .changeset/t12343-genesis-snapshot.md | 11 ++- packages/core/src/store/restore-marker.ts | 72 +++++++++++++++---- .../src/store/sync/__tests__/genesis.test.ts | 37 +++++++++- packages/core/src/store/sync/genesis.ts | 23 ++++-- 4 files changed, 123 insertions(+), 20 deletions(-) diff --git a/.changeset/t12343-genesis-snapshot.md b/.changeset/t12343-genesis-snapshot.md index a98059ba44..82b20dc7bf 100644 --- a/.changeset/t12343-genesis-snapshot.md +++ b/.changeset/t12343-genesis-snapshot.md @@ -22,6 +22,15 @@ write made after the cut would travel both in the bundle and in a segment, and r The write chokepoint is `assertExodusWriteSafe`, which now calls `awaitStoreWritable`. Its callers are the task accessor's write transactions, session creation, `insertIdempotent` and `upsertIdempotent`. A writer that bypasses those is not paused; the race - check above catches its write instead. The marker is always released. + check above catches its write instead. +- **A marker that lapses undoes the cut.** The genesis cut checks its marker after the snapshot. The cut is undone and + `GenesisRacedError` (`E_SYNC_GENESIS_MARKER_LOST`) is thrown in either case: + - the marker is older than the stale window, so other processes may already have been writing; + - another process replaced the marker. + + Nothing is recorded or uploaded. +- **Marker window and release.** The stale window stays at the 1 h floor. The worst measured hold is about 63 s, on a scratch copy of + the 1.3 GB cleocode store: cut 2.4 s, export 48.7 s, extract and manifest 11.8 s. A marker is released only while it is still the + holder's own (`holdRestoreMarker`), so a lapsed holder never removes the marker of the process that took over. - **The restore-in-progress marker** (`store/restore-marker.ts`, `store/pid-alive.ts` and the open-path checks) comes from #1902 (T13258), cherry-picked unchanged. diff --git a/packages/core/src/store/restore-marker.ts b/packages/core/src/store/restore-marker.ts index 02d57a61ce..485210d756 100644 --- a/packages/core/src/store/restore-marker.ts +++ b/packages/core/src/store/restore-marker.ts @@ -71,6 +71,13 @@ function readMarker(dbPath: string): RestoreMarker | 'unreadable' | null { * A marker older than this is stale whoever wrote it: a restore holds it for * seconds (the copy of the replaced store and a rename), and a crashed holder * on another host (or this one under a changed hostname) cannot be probed. + * + * A genesis cut holds it longest: the cut plus the bundle export, measured on + * scratch copies of the largest real stores (2026-10-07, #1953 review): about + * 63 s worst on cleocode's 1.3 GB store (cut 2.4 s, export 48.7 s, extract + * and manifest 11.8 s) and 2.8 s on claude-todo. The window is the 1 h floor, + * over 50x that worst case, and a holder that outlives it never finishes: + * {@link RestoreMarkerHold.intact} turns false and the genesis cut aborts. */ export const RESTORE_MARKER_MAX_AGE_MS = 60 * 60 * 1000; @@ -236,39 +243,80 @@ export async function awaitStoreWritable( throw markerRefusal(dbPath, marker); } +/** A held marker: its release, and whether it is still this holder's and inside its window. */ +export interface RestoreMarkerHold { + /** Remove the marker, but only while it is still this holder's (idempotent). */ + readonly release: () => void; + /** + * Whether the marker on disk is still the one written here and younger than + * {@link RESTORE_MARKER_MAX_AGE_MS}. False once it went stale (other + * processes may already be writing) or another process replaced it. + */ + readonly intact: () => boolean; +} + /** - * Write the marker for a restore of `dbPath` (exclusive). Returns its release. + * Write the marker for a restore of `dbPath` (exclusive) and hold it. A holder + * whose work outlives the window must check {@link RestoreMarkerHold.intact} + * before it commits to its result, and discard the result when it is false. * - * @param dbPath - The store file about to be replaced. - * @param kind - Who is replacing it. - * @returns A function that removes the marker (idempotent). - * @throws {CleoError} `E_STORE_RESTORING` when another live restore holds it. + * @param dbPath - The store file about to be replaced or snapshotted. + * @param kind - Who is holding it. + * @returns The hold. + * @throws {CleoError} `E_STORE_RESTORING` / `E_STORE_GENESIS` when another live holder has it. * @task T13258 */ -export function writeRestoreMarker(dbPath: string, kind: RestoreMarker['kind']): () => void { +export function holdRestoreMarker(dbPath: string, kind: RestoreMarker['kind']): RestoreMarkerHold { const file = dbPath + RESTORE_MARKER_SUFFIX; const deviceId = readDeviceId(); + const startedMs = Date.now(); const marker: RestoreMarker = { pid: process.pid, host: hostname(), ...(deviceId !== null ? { deviceId } : {}), - startedAt: new Date().toISOString(), + startedAt: new Date(startedMs).toISOString(), kind, }; + const body = `${JSON.stringify(marker)}\n`; const existing = readMarker(dbPath); if (existing !== null && blocks(existing)) assertStoreNotRestoring(dbPath, 0); // A stale marker (its restorer is gone) is replaced. if (existing !== null) rmSync(file, { force: true }); const fd = openSync(file, 'wx'); try { - writeSync(fd, `${JSON.stringify(marker)}\n`); + writeSync(fd, body); } finally { closeSync(fd); } + const ours = (): boolean => { + try { + return readFileSync(file, 'utf8') === body; + } catch { + return false; + } + }; let released = false; - return () => { - if (released) return; - released = true; - rmSync(file, { force: true }); + return { + release: () => { + if (released) return; + released = true; + // Never remove a marker another process took over after ours went stale. + if (ours()) rmSync(file, { force: true }); + }, + intact: () => !released && Date.now() - startedMs <= RESTORE_MARKER_MAX_AGE_MS && ours(), }; } + +/** + * Write the marker for a restore of `dbPath` (exclusive). Returns its release + * ({@link holdRestoreMarker} for a holder that must check it still holds it). + * + * @param dbPath - The store file about to be replaced. + * @param kind - Who is replacing it. + * @returns A function that removes the marker while it is still this holder's (idempotent). + * @throws {CleoError} `E_STORE_RESTORING` when another live restore holds it. + * @task T13258 + */ +export function writeRestoreMarker(dbPath: string, kind: RestoreMarker['kind']): () => void { + return holdRestoreMarker(dbPath, kind).release; +} diff --git a/packages/core/src/store/sync/__tests__/genesis.test.ts b/packages/core/src/store/sync/__tests__/genesis.test.ts index c909f07607..adec6ce86d 100644 --- a/packages/core/src/store/sync/__tests__/genesis.test.ts +++ b/packages/core/src/store/sync/__tests__/genesis.test.ts @@ -7,7 +7,7 @@ * @task T12343 */ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; @@ -18,7 +18,7 @@ import { getDualScopeNativeDb, openDualScopeDbAtPath, } from '../../dual-scope-db.js'; -import { RESTORE_MARKER_SUFFIX } from '../../restore-marker.js'; +import { RESTORE_MARKER_MAX_AGE_MS, RESTORE_MARKER_SUFFIX } from '../../restore-marker.js'; import { ROW_IDENTITY_META_TABLE, ROW_IDENTITY_RECIPE, @@ -434,6 +434,39 @@ describe('genesis cut with its checkpoint snapshot (S4-1b; T13296, T13297)', () expect(again).toMatchObject({ refused: null, folded: 1 }); }); + it('a marker another process took over during the snapshot undoes the cut, and its marker is left in place', async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + const theirs = `${JSON.stringify({ pid: 999_999, host: 'elsewhere', startedAt: new Date().toISOString(), kind: 'restore' })}\n`; + const run = cutGenesisWithSnapshot(db, opts(), async () => { + // Our marker went stale and a restore elsewhere replaced it. + writeFileSync(dbPath + RESTORE_MARKER_SUFFIX, theirs); + }); + await expect(run).rejects.toThrow(/E_SYNC_GENESIS_MARKER_LOST/); + expectUncut(db); + // Our release never removes the new holder's marker. + expect(readFileSync(dbPath + RESTORE_MARKER_SUFFIX, 'utf8')).toBe(theirs); + }); + + it('a snapshot that outlives the marker window undoes the cut: a stale marker held nobody off', async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + const realNow = Date.now; + const run = cutGenesisWithSnapshot(db, opts(), async () => { + const late = realNow() + RESTORE_MARKER_MAX_AGE_MS + 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => late); + }); + try { + await expect(run).rejects.toThrow(/E_SYNC_GENESIS_MARKER_LOST/); + } finally { + vi.restoreAllMocks(); + } + expectUncut(db); + expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); + }); + it('a refused cut never runs the snapshot and releases the marker', async () => { const { db } = await store(); setCaptureEnabled(db, 'project', false, { schemaRoot: SYNC_SCHEMA }); diff --git a/packages/core/src/store/sync/genesis.ts b/packages/core/src/store/sync/genesis.ts index 162a8bc169..267bf1ebbb 100644 --- a/packages/core/src/store/sync/genesis.ts +++ b/packages/core/src/store/sync/genesis.ts @@ -28,7 +28,7 @@ import type { DatabaseSync } from 'node:sqlite'; import type { TableScope } from '@cleocode/contracts'; -import { writeRestoreMarker } from '../restore-marker.js'; +import { holdRestoreMarker } from '../restore-marker.js'; import { BIRTH_FP_COLUMN, ROW_IDENTITY, @@ -438,8 +438,10 @@ function uncutGenesis(db: DatabaseSync, opts: GenesisCutOptions): void { * that bypasses the chokepoint is caught after the snapshot (the capture * position moved past the cut): the cut is undone and * {@link GenesisRacedError} is thrown, so no bundle holding a post-cut write - * is ever pushed. A failing snapshot undoes the cut too. The marker is always - * released. + * is ever pushed. A failing snapshot undoes the cut too, and so does a + * marker that went stale or another process took over during the snapshot + * ({@link GenesisRacedError}, `E_SYNC_GENESIS_MARKER_LOST`). The marker is + * always released, unless another process now holds it. * * @param db - The store. * @param opts - {@link GenesisCutOptions}, plus the store file the marker guards. @@ -452,7 +454,7 @@ export async function cutGenesisWithSnapshot( opts: GenesisCutOptions & { readonly dbPath: string }, snapshot: (cut: number) => Promise, ): Promise { - const release = writeRestoreMarker(opts.dbPath, 'genesis'); + const marker = holdRestoreMarker(opts.dbPath, 'genesis'); try { const report = cutGenesis(db, opts); if (report.refused !== null || report.already || report.cut === null) return report; @@ -470,8 +472,19 @@ export async function cutGenesisWithSnapshot( 'E_SYNC_GENESIS_RACED: a write reached the store during the genesis snapshot; the cut was undone, run it again', ); } + // A marker that went stale or was taken over held nobody off for part of + // the snapshot: discard the cut, never finish it (an uncaptured write may + // be in the bundle). The bundle is complete here, so a lapse after this + // check no longer matters. + if (!marker.intact()) { + uncutGenesis(db, opts); + // @sync-invariant none:local-only the genesis marker lapsed during the snapshot; the cut is undone and nothing is pushed + throw new GenesisRacedError( + 'E_SYNC_GENESIS_MARKER_LOST: the genesis marker went stale or was taken over during the snapshot; the cut was undone, run it again', + ); + } return report; } finally { - release(); + marker.release(); } } From 21d054a3a3c1890853640801208b624594ab558b Mon Sep 17 00:00:00 2001 From: Keaton Hoskins Date: Fri, 9 Oct 2026 21:46:28 -0700 Subject: [PATCH 23/23] fix(sync): a genesis run whose marker was taken over aborts without undoing the cut (T13296) team-lead on #1953: a second genesis process that takes over our marker after it went stale (over 1 h) resumes OUR pending cut. Our marker-lost path then ran uncutGenesis and undid the cut the new holder was snapshotting. - RestoreMarkerHold.takenOver(): the file on disk is no longer ours (replaced, or replaced and already released). Only a new holder removes or replaces a held marker, and only once it is stale, so an absent marker is a takeover too, never "ours expired". - cutGenesisWithSnapshot: after the snapshot, a takeover throws E_SYNC_GENESIS_MARKER_LOST and leaves the pending cut to the new holder. A failing snapshot after a takeover rethrows without undoing it. Only a marker that is still ours but past its window (expired, untaken) undoes the cut, as before. Tests: a takeover by another holder, a second genesis holder taking over a stale marker mid-snapshot, a holder that took over and already released, and a failing snapshot after a takeover all leave the cut pending; the expired-untaken case still undoes it. Four mutations killed (no takeover check, unconditional undo on snapshot failure, takenOver always true, absent marker read as ours). Co-Authored-By: Claude Opus 5.5 --- packages/core/src/store/restore-marker.ts | 8 ++ .../src/store/sync/__tests__/genesis.test.ts | 77 +++++++++++++++++-- packages/core/src/store/sync/genesis.ts | 30 +++++--- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/packages/core/src/store/restore-marker.ts b/packages/core/src/store/restore-marker.ts index 485210d756..b149441d5c 100644 --- a/packages/core/src/store/restore-marker.ts +++ b/packages/core/src/store/restore-marker.ts @@ -253,6 +253,13 @@ export interface RestoreMarkerHold { * processes may already be writing) or another process replaced it. */ readonly intact: () => boolean; + /** + * Whether another holder took the marker over: the file on disk is no + * longer the one written here (replaced, or replaced and already released). + * Only a new holder removes or replaces a held marker, and only after it + * went stale, so a holder that sees this no longer owns what it guarded. + */ + readonly takenOver: () => boolean; } /** @@ -304,6 +311,7 @@ export function holdRestoreMarker(dbPath: string, kind: RestoreMarker['kind']): if (ours()) rmSync(file, { force: true }); }, intact: () => !released && Date.now() - startedMs <= RESTORE_MARKER_MAX_AGE_MS && ours(), + takenOver: () => !released && !ours(), }; } diff --git a/packages/core/src/store/sync/__tests__/genesis.test.ts b/packages/core/src/store/sync/__tests__/genesis.test.ts index adec6ce86d..98499f94f3 100644 --- a/packages/core/src/store/sync/__tests__/genesis.test.ts +++ b/packages/core/src/store/sync/__tests__/genesis.test.ts @@ -18,7 +18,11 @@ import { getDualScopeNativeDb, openDualScopeDbAtPath, } from '../../dual-scope-db.js'; -import { RESTORE_MARKER_MAX_AGE_MS, RESTORE_MARKER_SUFFIX } from '../../restore-marker.js'; +import { + holdRestoreMarker, + RESTORE_MARKER_MAX_AGE_MS, + RESTORE_MARKER_SUFFIX, +} from '../../restore-marker.js'; import { ROW_IDENTITY_META_TABLE, ROW_IDENTITY_RECIPE, @@ -117,6 +121,14 @@ const cut = (db: DatabaseSync) => allowUnreleased: true, }); +/** The cut is still committed and pending, as its new holder needs it. */ +function expectCutPending(db: DatabaseSync): void { + expect(genesisCutOf(db, STREAM)).toBeDefined(); + expect(meta(db, `${GENESIS_PENDING_KEY_PREFIX}${STREAM}`)).toBeDefined(); + expect(meta(db, UNDO_ENABLED_KEY)).toBeDefined(); + expect(n(db, "SELECT count(*) AS n FROM _sync_txn WHERE state = 'folded'")).toBe(1); +} + /** Nothing of the cut was written. */ function expectUncut(db: DatabaseSync): void { expect(genesisCutOf(db, STREAM)).toBeUndefined(); @@ -434,21 +446,76 @@ describe('genesis cut with its checkpoint snapshot (S4-1b; T13296, T13297)', () expect(again).toMatchObject({ refused: null, folded: 1 }); }); - it('a marker another process took over during the snapshot undoes the cut, and its marker is left in place', async () => { + it('a marker another process took over during the snapshot aborts WITHOUT undoing the cut, and its marker is left in place', async () => { const { db } = await store(); write(db, addTask('T1')); seal(db); const theirs = `${JSON.stringify({ pid: 999_999, host: 'elsewhere', startedAt: new Date().toISOString(), kind: 'restore' })}\n`; const run = cutGenesisWithSnapshot(db, opts(), async () => { - // Our marker went stale and a restore elsewhere replaced it. + // Our marker went stale and another holder replaced it. writeFileSync(dbPath + RESTORE_MARKER_SUFFIX, theirs); }); - await expect(run).rejects.toThrow(/E_SYNC_GENESIS_MARKER_LOST/); - expectUncut(db); + await expect(run).rejects.toThrow(/E_SYNC_GENESIS_MARKER_LOST: another process took over/); + // The new holder owns the pending cut: it is left committed. + expectCutPending(db); // Our release never removes the new holder's marker. expect(readFileSync(dbPath + RESTORE_MARKER_SUFFIX, 'utf8')).toBe(theirs); }); + it('a second genesis holder that takes over a stale marker keeps the cut it resumes: the first run aborts and never undoes it', async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + const realNow = Date.now; + let second: ReturnType | undefined; + const run = cutGenesisWithSnapshot(db, opts(), async () => { + // Our marker outlives its window; a second genesis process takes it over + // to resume our pending cut, and is still snapshotting when we wake. + const late = realNow() + RESTORE_MARKER_MAX_AGE_MS + 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => late); + second = holdRestoreMarker(dbPath, 'genesis'); + }); + try { + await expect(run).rejects.toThrow(/E_SYNC_GENESIS_MARKER_LOST: another process took over/); + expectCutPending(db); + expect(second?.intact()).toBe(true); + expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(true); + } finally { + second?.release(); + vi.restoreAllMocks(); + } + expect(existsSync(dbPath + RESTORE_MARKER_SUFFIX)).toBe(false); + }); + + it('a holder that took over and already released keeps the cut too: an absent marker is never read as ours expired', async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + const run = cutGenesisWithSnapshot(db, opts(), async () => { + const second = holdRestoreMarker(dbPath, 'genesis'); + second.release(); + }); + await expect(run).rejects.toThrow(/E_SYNC_GENESIS_MARKER_LOST: another process took over/); + expectCutPending(db); + }); + + it("a failing snapshot after a takeover rethrows without undoing the new holder's cut", async () => { + const { db } = await store(); + write(db, addTask('T1')); + seal(db); + let second: ReturnType | undefined; + const run = cutGenesisWithSnapshot(db, opts(), async () => { + second = holdRestoreMarker(dbPath, 'genesis'); + throw new Error('upload failed'); + }); + try { + await expect(run).rejects.toThrow('upload failed'); + expectCutPending(db); + } finally { + second?.release(); + } + }); + it('a snapshot that outlives the marker window undoes the cut: a stale marker held nobody off', async () => { const { db } = await store(); write(db, addTask('T1')); diff --git a/packages/core/src/store/sync/genesis.ts b/packages/core/src/store/sync/genesis.ts index 267bf1ebbb..39a7ca3947 100644 --- a/packages/core/src/store/sync/genesis.ts +++ b/packages/core/src/store/sync/genesis.ts @@ -439,9 +439,11 @@ function uncutGenesis(db: DatabaseSync, opts: GenesisCutOptions): void { * position moved past the cut): the cut is undone and * {@link GenesisRacedError} is thrown, so no bundle holding a post-cut write * is ever pushed. A failing snapshot undoes the cut too, and so does a - * marker that went stale or another process took over during the snapshot - * ({@link GenesisRacedError}, `E_SYNC_GENESIS_MARKER_LOST`). The marker is - * always released, unless another process now holds it. + * marker that went stale during the snapshot with nobody taking it over + * ({@link GenesisRacedError}, `E_SYNC_GENESIS_MARKER_LOST`). When another + * process took the stale marker over, it owns the pending cut: this run + * aborts with the same error and never undoes the cut. The marker is always + * released, unless another process now holds it. * * @param db - The store. * @param opts - {@link GenesisCutOptions}, plus the store file the marker guards. @@ -462,9 +464,19 @@ export async function cutGenesisWithSnapshot( try { await snapshot(cut); } catch (err) { - uncutGenesis(db, opts); + // A holder that took the marker over owns the pending cut now: leave it. + if (!marker.takenOver()) uncutGenesis(db, opts); throw err; } + // Our marker went stale and another process took it over (a genesis run + // resuming this pending cut, or a restore): the cut is its to finish or + // replace. Abort without undoing it, and push nothing from this snapshot. + if (marker.takenOver()) { + // @sync-invariant none:local-only another process took the genesis marker over during the snapshot; the pending cut is left to it and nothing is pushed + throw new GenesisRacedError( + 'E_SYNC_GENESIS_MARKER_LOST: another process took over the genesis marker during the snapshot; the pending cut is left to it and nothing is pushed', + ); + } if (capturePosition(db) > cut) { uncutGenesis(db, opts); // @sync-invariant none:local-only a local write raced the genesis snapshot; the cut is undone and nothing is pushed @@ -472,15 +484,15 @@ export async function cutGenesisWithSnapshot( 'E_SYNC_GENESIS_RACED: a write reached the store during the genesis snapshot; the cut was undone, run it again', ); } - // A marker that went stale or was taken over held nobody off for part of - // the snapshot: discard the cut, never finish it (an uncaptured write may - // be in the bundle). The bundle is complete here, so a lapse after this - // check no longer matters. + // A marker that went stale, with nobody taking it over, held nobody off + // for part of the snapshot: discard the cut, never finish it (an + // uncaptured write may be in the bundle). The bundle is complete here, so + // a lapse after this check no longer matters. if (!marker.intact()) { uncutGenesis(db, opts); // @sync-invariant none:local-only the genesis marker lapsed during the snapshot; the cut is undone and nothing is pushed throw new GenesisRacedError( - 'E_SYNC_GENESIS_MARKER_LOST: the genesis marker went stale or was taken over during the snapshot; the cut was undone, run it again', + 'E_SYNC_GENESIS_MARKER_LOST: the genesis marker went stale during the snapshot; the cut was undone, run it again', ); } return report;