From b9b88da4867a2f044a8f8caf8bb4758c54b8fedb Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Mon, 21 Sep 2026 09:13:08 +0100 Subject: [PATCH] Prove bounded search foundations and restore source navigation recall --- .github/workflows/ci.yml | 3 + GOALS.md | 83 +++ README.md | 5 + RELEASE_EVIDENCE.md | 94 ++++ benchmarks/scale/README.md | 69 +++ benchmarks/scale/corpus.mjs | 76 +++ benchmarks/scale/postings-bench.mjs | 149 ++++++ benchmarks/scale/postings.mjs | 305 +++++++++++ benchmarks/scale/postings.test.mjs | 383 ++++++++++++++ benchmarks/scale/sqlite.mjs | 485 ++++++++++++++++++ benchmarks/scale/sqlite.test.mjs | 208 ++++++++ docs/ADR-001-scalable-collections.md | 112 ++++ docs/BOUNDED-SEARCH-DESIGN.md | 27 + docs/ENTERPRISE-SCALE-REVIEW.md | 103 ++++ docs/OVERNIGHT-HANDOFF.md | 31 ++ .../context-tools/src/source-scan.test.ts | 77 +++ packages/context-tools/src/source-scan.ts | 58 ++- packages/context/src/paging/store.test.ts | 81 +++ packages/context/src/paging/store.ts | 54 ++ 19 files changed, 2397 insertions(+), 6 deletions(-) create mode 100644 GOALS.md create mode 100644 RELEASE_EVIDENCE.md create mode 100644 benchmarks/scale/README.md create mode 100644 benchmarks/scale/corpus.mjs create mode 100644 benchmarks/scale/postings-bench.mjs create mode 100644 benchmarks/scale/postings.mjs create mode 100644 benchmarks/scale/postings.test.mjs create mode 100644 benchmarks/scale/sqlite.mjs create mode 100644 benchmarks/scale/sqlite.test.mjs create mode 100644 docs/ADR-001-scalable-collections.md create mode 100644 docs/BOUNDED-SEARCH-DESIGN.md create mode 100644 docs/ENTERPRISE-SCALE-REVIEW.md create mode 100644 docs/OVERNIGHT-HANDOFF.md create mode 100644 packages/context/src/paging/store.test.ts create mode 100644 packages/context/src/paging/store.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc86e2e..54c798f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,4 +19,7 @@ jobs: - run: npm run build - run: npm test - run: npm run benchmark:tokens:check + - run: npm run benchmark:tokens:parity + - run: node --test benchmarks/scale/sqlite.test.mjs benchmarks/scale/postings.test.mjs + - run: node benchmarks/scale/postings-bench.mjs --records 10000 - run: npm run test:packages diff --git a/GOALS.md b/GOALS.md new file mode 100644 index 0000000..b7a26d6 --- /dev/null +++ b/GOALS.md @@ -0,0 +1,83 @@ +# Z1P Core release goals + +Updated: 21 September 2026. + +We want coding agents to spend less time and inference rediscovering a codebase, without making their answers or changes less reliable. The open-source core comes first. It must be useful, dependable and independently installable before private services become a dependency. + +This is an acceptance plan, not a claim that the gates have passed. Existing code and tests are a starting point. Record fresh evidence against the release commit before closing a gate. + +Current results and open blockers are recorded in [RELEASE_EVIDENCE.md](RELEASE_EVIDENCE.md). + +## Order of work + +1. Establish the release baseline and remaining gaps. +2. Prove trust boundaries, extraction and retrieval behaviour. +3. Prove installation and real consumer workflows. +4. Measure complete tasks, then publish a verified release. +5. Only then build private operational features justified by real use. + +The free/private boundary remains [OPEN_CORE.md](OPEN_CORE.md). Local extraction, verification, bounded retrieval, CLI, MCP and portable formats stay MIT licensed. Provider routing and worker execution belong in consumers. A generally useful core fix belongs here even if a private pilot discovers it. + +## G0: A reproducible baseline + +- [ ] Record the candidate commit, Node version, platform, commands and complete results in a release evidence document. +- [ ] Run `npm run check`, `npm run benchmark:tokens:check` and `npm run benchmark:tokens:parity` from a clean checkout using `npm ci --ignore-scripts`. +- [ ] Map the existing tests to G1-G3 below. Record missing cases explicitly rather than treating a green suite as complete coverage. +- [ ] Classify release blockers by correctness, security, compatibility and usability; give each an owner and a regression test where applicable. +- [ ] Verify CI for the candidate commit. If infrastructure prevents it running, record the blocker and local evidence separately; do not call CI passed. + +Exit: another developer can reproduce the baseline, and every known release blocker has a concrete acceptance condition. + +## G1: Trust boundaries we can rely on + +- [ ] Exercise valid and tampered signatures, wrong keys, malformed input, unsupported versions and invalid grants through public APIs. +- [ ] Prove collection and project isolation, scoped access and revocation on subsequent reads. Document that revocation cannot erase plaintext already obtained by an authorised reader. +- [ ] Prove graph edges, repository text and retrieved instructions cannot grant access, execute code or become trusted instructions merely by being retrieved. +- [ ] Test repository path escapes, symlinks, ignored/secret files, oversized inputs and traversal limits. Document what is excluded and what is not guaranteed to be detected. +- [ ] Check CLI, MCP, errors and logs for unintended disclosure of keys, credentials and plaintext. No network transfer without an explicit configured operation. +- [ ] Document the threat model, key custody, supported security properties and a private vulnerability-reporting route. + +Exit: negative tests protect each stated boundary, with no unresolved release-blocking security or data-isolation defect. + +## G2: Evidence that is correct enough to use + +- [ ] Prove indexed queries, refresh and resource budgets through the [enterprise scale plan](docs/ENTERPRISE-SCALE-REVIEW.md) before making whole-codebase claims or raising the 128-record limit. ADR-001 remains provisional until its query and trust-boundary gaps are resolved. +- [ ] Verify deterministic output and stable source identity on repeat scans, with fixtures for edits, deletion, renames, duplicate names and ambiguous links. +- [ ] State snapshot freshness and how to replace or invalidate stale evidence. A stale record must not silently masquerade as the current source. +- [ ] Keep TypeScript/JavaScript compiler-derived evidence separate from lexical and inferred evidence in output and documentation. +- [ ] Cover multi-repository manifests, explicit cross-repository links, missing repositories and ambiguous package names without widening access. +- [ ] Exercise retrieval budgets, truncation, empty results, corrections and path limits. Return enough provenance to inspect the source and recognise incomplete answers. +- [ ] Retain full predeclared required-source recall in both existing benchmark gates. Report raw answer evidence and navigation compression separately. + +Exit: documented behaviour matches executable fixtures, including the cases where extraction cannot answer reliably. + +## G3: A release someone else can actually use + +- [ ] Test real package tarballs outside the workspace: imports, exports, browser isolation, CLI, MCP, notices and licences. +- [ ] Walk through the documented install, scan, persist, retrieve, export and import path on a clean machine or isolated environment. +- [ ] Exercise an actual MCP client session, including initialisation, errors, cancellation and bounded output. Name the tested client and version. +- [ ] Verify compatibility fixtures for existing package names, APIs and signed wire formats. Document any required migration before release. +- [ ] Test intended consumer integration in isolated branches or fixtures. Do not silently repin or migrate live consumers. +- [ ] Reconcile README commands, package versions, changelog and extraction provenance. Distinguish a prepared tarball from an available registry release. +- [ ] Publish through the authorised release process, then install the exact published versions from the registry outside the workspace and repeat the smoke workflow. + +Exit: a newcomer can follow the published instructions successfully without local workspace links or unpublished dependencies. + +## G4: Useful savings, with quality held constant + +- [ ] Define representative tasks and acceptance tests before comparing baseline and Z1P-assisted runs. Include repository orientation, bug investigation, change impact and an accepted code change. +- [ ] Hold repository revisions, task instructions, model settings and acceptance standards constant. Separate context improvements from any later model-routing experiment. +- [ ] Count the whole task: retrieved source, input/output tokens, cached tokens where reported, retries, tool calls, failures and review time. Include scan and refresh overhead. +- [ ] Record missing provider usage as unknown, never zero. Keep observed billing, estimated token cost and fixed subscription costs distinct. +- [ ] Report accepted tasks, regressions and unsuccessful runs alongside cost. Navigation compression alone is not an inference-bill saving. +- [ ] Publish a reproducible, non-sensitive evaluation and its limitations. Keep private code, prompts, invoices and customer data out of this repository. + +Exit: the tested workflow reduces cost per accepted outcome without lowering the agreed quality bar. If it does not, record the result and fix the cause before making a savings claim. + +## Definition of done + +G0-G4 have linked evidence and reviewer sign-off for a named release. No known release-blocking defect remains. The free local route works without a Z1P account or service, and the published packages have passed an independent install check. + +For each gate, record: status (`not assessed`, `in progress`, `blocked`, `passed`), owner, commit, environment, command or workflow, result, evidence link, reviewer and remaining limitations. Unchecked items are unverified, not necessarily unimplemented. + +Start with G0. Do not broaden language support, build a graph editor or add hosted dependencies to avoid fixing the release path. diff --git a/README.md b/README.md index 482c528..5ed5303 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,11 @@ The free core is intended to be useful on its own, not a time-limited trial. See [OPEN_CORE.md](OPEN_CORE.md) for the durable product boundary and compatibility policy. +See [GOALS.md](GOALS.md) for the core-first release gates, required evidence +and whole-task inference-cost evaluation. +The current candidate results and blockers are recorded in +[RELEASE_EVIDENCE.md](RELEASE_EVIDENCE.md). + The formats and APIs are project-agnostic. A collection can describe one repository or an explicitly assembled ecosystem; graph operations never make another collection visible or turn an extracted relationship into authority. diff --git a/RELEASE_EVIDENCE.md b/RELEASE_EVIDENCE.md new file mode 100644 index 0000000..de1db26 --- /dev/null +++ b/RELEASE_EVIDENCE.md @@ -0,0 +1,94 @@ +# Z1P Core release candidate evidence ledger + +## Source shipment verification, 21 September 2026 + +The scale-foundations shipment passes 33 core tests, 28 tools tests, independent packed-package smoke, both token benchmark gates, 22 synthetic scale tests and the 10k posting-index runner. The memory blob fixture now copies Node Buffer inputs independently and rejects non-string digests before coercion. CI includes navigation recall, both scale suites and the 10k probe. These are source-shipment checks, not closure of G0-G4 or an npm publication. PR and main CI provide commit-specific remote evidence; earlier entries below remain historical snapshots. + +Updated: 21 September 2026. + +## Candidate metadata + +| Field | Value | +| --- | --- | +| Evidence date | 21 September 2026 | +| Tested commit | `5540c4d` | +| Working tree | Uncommitted `README.md`, `GOALS.md` and this ledger are documentation changes only | +| Runtime | Node v24.21.0 | +| Platform | macOS arm64 | +| Registry | `@forgesworn/context-tools@0.3.0` is not published | + +Overall status: **G0 in progress, not passed.** + +The tested commit builds and its current automated tests pass. CI has not been rerun for the eventual release commit, known release gaps remain, and no readiness or security conclusion follows from this evidence. + +## Commands and observed outcomes + +```sh +npm run check && npm run benchmark:tokens:check && npm run benchmark:tokens:parity +``` + +| Check | Outcome | Evidence | +| --- | --- | --- | +| TypeScript builds | Passed | Both workspace packages compiled | +| `@forgesworn/context` tests | Passed | 4 files, 25 tests | +| `@forgesworn/context-tools` tests | Passed | 4 files, 24 tests | +| Packed-package smoke | Passed | Independent Node import, declarations and browser bundle; independent CLI create, append and restart recovery | +| Raw evidence benchmark | Passed | 26 files, 67,087 baseline tokens, 6 queries, aggregate 28.30x reduction multiplier, 96.47% reduction, minimum declared-source recall 1.0; 10x regression floor passed | +| Navigation benchmark | Passed with material limitation | 25 files, 60,541 baseline tokens, 8 queries, average 140.71x, minimum 138.54x and median 140.47x, minimum declared-source recall 1.0; 71.5x reference target passed | + +The navigation scan retained 128 graph records and dropped 111. This makes the present collection ceiling a release and product blocker for whole-codebase claims. The navigation result measures compact pointers to required sources. It is not sufficient answer evidence and is not a measured inference-bill saving. + +## Gate status + +| Gate | Status | Evidence and remaining gaps | +| --- | --- | --- | +| G0 reproducible baseline | In progress | Local build, 49 tests, packed smoke and both benchmarks passed. The release commit and a current CI run remain outstanding. Test-to-gate coverage is recorded below. | +| G1 trust boundaries | Partially evidenced | Tests cover grant and scope rechecks, proof refusal, uploader signing boundaries, bounded traversal and scanner path/symlink handling. The full negative boundary matrix and published threat model remain outstanding. | +| G2 usable evidence | Partially evidenced | Tests cover deterministic scans, corrections, provenance, ambiguity and response bounds. Repository refresh across deletion and rename is not implemented or proven. The 128-record collection ceiling prevents whole-codebase claims. | +| G3 independent use | Partially evidenced | Real tarballs work outside the workspace. A published-registry install, documented end-to-end walkthrough and named real MCP client robustness check remain outstanding. | +| G4 cost per accepted task | Not started | No paired pilot trial has run. Current token benchmarks do not measure accepted task quality, retries, review time or billed cost. | + +## Existing test evidence + +- `retrieval.test.ts`: bounded query hits and one-hop links without external reads; current-head corrections; exact UTF-8 caps and oversized omission; source revisions are not conflated. +- `graph.test.ts`: atomic links and cycles; bounded query and path operations without crossing collections; provenance; invalid, absent and corrected endpoints; byte, depth and node caps; corrected stale topology; deterministic shortest paths. +- `index.test.ts`: grant and scope rechecks; identity sharing and restore; malformed provenance; unknown proof and personal-sharing refusal; host proof adapter; uploader signing boundaries. +- Scanner tests: deterministic JS/TS and conservative broad-language output; hidden, generated and symlinked trees ignored or rejected; UTF-8 and resource bounds; append-safe relations; ambiguous packages and links handled. +- `blossom.test.ts`: published envelope fixture compatibility. +- The package smoke builds real tarballs and checks the independent consumers described above. + +These statements describe the named tests. They do not extend to untested environments, attacks or integrations. + +## Known blockers + +1. Design and prove a route beyond 128 records while keeping each read bounded. +2. Define and test repository refresh for changed, deleted and renamed files, including explicit stale-snapshot behaviour. +3. Exercise initialisation, errors, cancellation and bounded results through a named real MCP client. +4. Publish the matching packages through the authorised process, then repeat the independent smoke against the registry versions. +5. Run CI on the eventual release commit and record its result separately from this local pass. +6. Publish the threat model and complete the negative trust-boundary test matrix. +7. Run the paired cost-per-accepted-task trial. Include retries, failures, review effort, scan/refresh cost and the actual charging basis. + +## Claim boundaries + +- Passing these checks does not establish that Z1P Core is secure, release-ready or production-ready. +- The broad-language scanner is conservative lexical navigation, not compiler-level semantic analysis. +- The benchmark corpora and declared-source checks do not prove task answer quality. +- No cost saving is claimed until the paired trial measures complete accepted tasks. +- Registry availability, current CI, a real client workflow and normal-work use remain separate gates. + +## Next action + +Follow the [enterprise scale review](docs/ENTERPRISE-SCALE-REVIEW.md): prove indexed queries and resource budgets before implementing a new cryptographic paging format. + +## Subsequent working-tree checks + +Adding the experimental paging memory stores increased the test count to 55; both builds, all tests and independent package smoke passed. Raw-evidence reduction was 28.90x with minimum declared-source recall 1.0. + +The expanded navigation corpus failed: 27 files, 128 retained records, 122 dropped records, and `retrieveView` source recall 0. The earlier passing baseline above remains historical evidence only. The current working tree does not pass the full gate. No scanner repair has been applied, and no benchmark threshold has been relaxed. + +### Worker implementation follow-up + +The scanner regression above has now been repaired by adding bounded exported-name anchors to file records and allocating symbol detail round-robin across files. Existing IDs, source fields and the 128-record limit are preserved. Full workspace verification passed: 31 core tests, 28 tools tests, independent package smoke and both unchanged token benchmark gates. Navigation minimum declared-source recall is again 1.0, with aggregate reduction 144.73x on this working tree. + +The separate [synthetic scale probe](benchmarks/scale/README.md) adds eight passing tests and successful 10k/50k and 100k/500k record/edge runs. This prototype is not integrated into Core and does not lift the current collection limit or qualify private-data storage. diff --git a/benchmarks/scale/README.md b/benchmarks/scale/README.md new file mode 100644 index 0000000..a3924fb --- /dev/null +++ b/benchmarks/scale/README.md @@ -0,0 +1,69 @@ +# Synthetic SQLite scale probe + +This isolated prototype evaluates a possible Node index adapter using public generated data. It does not index repositories, integrate with ContextVault, encrypt its database or establish enterprise readiness. No production dependency or public API has changed. + +Run on the pinned Node 24 runtime: + +```sh +node --test benchmarks/scale/sqlite.test.mjs +node benchmarks/scale/sqlite.mjs --records 10000 +node benchmarks/scale/sqlite.mjs --records 100000 +``` + +An explicit `--records 1000000` tier is available but has not yet been qualified. Each run uses a temporary database and removes its generated database on completion. The schema stores five outgoing edges per primary record, plus ten auxiliary records with different content and no edges to check scope filtering. + +The eight tests cover deterministic corpus generation, basic scoped queries, generation replacement with pinned old reads, rollback on record-generator failure and dangling edges, literal FTS input, and parameter validation. The CLI additionally tests different content under the same record IDs across scopes. + +## Initial measurements + +21 September 2026, macOS arm64, Node 24.21.0, SQLite 3.53.4. Single runs, 32 warm samples per query type. Measurements are observations, not performance thresholds. + +| Primary records / edges | Primary build | Database including auxiliary scope | Warm unique-term search p95 | +| --- | --- | --- | --- | +| 10,000 / 50,000 | 0.47 seconds | 11,735,040 bytes | 0.271 ms | +| 100,000 / 500,000 | 6.62 seconds | 118,878,208 bytes | 0.226 ms | + +Both runs found all 32 sampled expected record IDs and passed the CLI checks. These are synthetic exact-term queries, not evidence of real-task answer quality. The first query in a process is not a cold OS-cache measurement. Reported recall fields currently check record IDs despite using `ExpectedSourceHits` in their names. + +A subsequent run added 32 common-term queries matching every primary record while returning ten rows. At 100,000 records / 500,000 edges, build time was 4.80 seconds, unique-term p95 0.124 ms, and common-term p50/p95 149.66/300.68 ms. The CLI checked the ten expected IDs on every common-term query. All eight probe tests and CLI assertions passed. Variation between runs is expected; these are individual measurements, not a capacity guarantee. + +The common-term result demonstrates the next design problem: a small result count does not imply bounded internal search work. Qualify explicit posting-list work budgets and incomplete-result reporting before integrating this adapter into agent queries. + +## Adapter decision + +### Separate posting-page experiment + +`postings.mjs` evaluates the [bounded search design](../../docs/BOUNDED-SEARCH-DESIGN.md): a composite `(scope, generation, term, id)` primary key, ID-only exact-token pages, capped materialised posting rows and generation-bound continuation. It does not replace the FTS baseline. All 14 new tests pass, alongside the eight baseline tests. + +```sh +node --test benchmarks/scale/postings.test.mjs benchmarks/scale/sqlite.test.mjs +node benchmarks/scale/postings-bench.mjs --records 10000 +node benchmarks/scale/postings-bench.mjs --records 100000 +node benchmarks/scale/postings-bench.mjs --records 1000000 +``` + +Final rerun, 21 September 2026: macOS arm64, Node 24.21.0, SQLite 3.53.4. Eight warm-up rounds then 32 samples per query type; nearest-rank percentiles. Each tier passed exact expected-ID, row-count, completeness and query-plan checks. Deep pages start twenty records from the end, using keyset continuation rather than OFFSET. + +| Records (zero edges) | Build ms | Database bytes | Peak RSS bytes | Common p95 ms | Unique p95 ms | Deep p95 ms | +| --- | --- | --- | --- | --- | --- | --- | +| 10,000 | 69.964 | 1,630,208 | 56,786,944 | 0.034084 | 0.034916 | 0.036334 | +| 100,000 | 711.905 | 16,482,304 | 66,961,408 | 0.031042 | 0.031958 | 0.033041 | +| 1,000,000 | 7,689.925 | 162,115,584 | 125,386,752 | 0.039459 | 0.288500 | 0.061584 | + +The measured plan was `SEARCH postings USING PRIMARY KEY (scope=? AND generation=? AND term=? AND id>?)`, with no full scan or temporary sort. Common/deep pages each materialise ten posting rows; unique queries materialise one. Internal SQLite work remains unmeasured. A full page conservatively returns incomplete, requiring continuation to establish exhaustion. Tests cover this boundary, empty generations, duplicate IDs, failed builds, distinct scope contents, pinned generations and cursor reuse after reopening. + +These are single-run observations, not latency guarantees. This is **not** a like-for-like speedup comparison with the earlier FTS/graph baseline: it returns IDs only, stores no graph edges and does not rank or fetch payloads. It demonstrates a useful access pattern, not enterprise readiness. Query timing includes the runner's awaited synchronous call. Tokenisation lowercases text before extracting ASCII-pattern tokens; it is not production Unicode tokenisation. The exposed database handle and plain cursor are benchmark conveniences, not security boundaries. + +Remaining posting-prototype hardening includes constructor-failure handle cleanup, schema compatibility checks, adversarial cursor objects, stronger iterator-cleanup tests and interrupted-process durability. Successful reopen/transaction tests do not prove crash durability. None of this changes the plaintext-private-data prohibition below. + +Continue evaluating SQLite for the Node adapter. It provides a useful baseline for transactional generations and indexed lookup without adding a dependency to the experiment. It is not selected as the production storage implementation yet. + +Open gates: + +- Internal SQLite work remains unmeasured. SQL LIMIT bounds returned rows, not posting scans, sorting, CPU or elapsed time. +- The shared FTS table filters results by scope but does not provide tenant workload isolation. Production access domains need appropriate partitioning and authorisation before search. +- The synchronous adapter cannot cancel an in-flight SQLite query. Worker isolation and enforceable budgets remain design/implementation work. +- Full generations are rebuilt; incremental refresh, deletions, rename handling and high-degree hubs are not covered. +- Persistent reopening, empty-generation identity, durable crash recovery, duplicate CLI flags and some cleanup paths still need hardening. +- The prototype uses plaintext synthetic data only. Do not use it for private repositories before encryption and key custody are implemented and reviewed. +- The regular five-edge graph and exact identifiers need more demanding fixtures before capacity conclusions. diff --git a/benchmarks/scale/corpus.mjs b/benchmarks/scale/corpus.mjs new file mode 100644 index 0000000..2808b13 --- /dev/null +++ b/benchmarks/scale/corpus.mjs @@ -0,0 +1,76 @@ +// Public synthetic corpus only. Never reads private repositories; no network. +// Streaming generators: no million-element arrays are retained by callers. + +const MIN_COUNT = 10; +const MAX_COUNT = 1_000_000; + +function assertCount(count) { + if (!Number.isSafeInteger(count) || count < MIN_COUNT || count > MAX_COUNT) { + throw new RangeError(`count must be a safe integer in [${MIN_COUNT}, ${MAX_COUNT}], got ${count}`); + } +} + +function pad7(n) { + return String(n).padStart(7, "0"); +} + +export function recordId(index) { + return `r${pad7(index)}`; +} + +export function needleTerm(index) { + return `needle${pad7(index)}`; +} + +export const COMMON_KEYWORD = "shared"; + +export function sourceFor(index) { + return `src/file${String(index % 10000).padStart(4, "0")}.ts`; +} + +export function topicFor(index) { + return index % 100; +} + +// Revision 1 changes only record zero's text to contain "refreshed". +// All other revisions leave every record byte-identical. +export function* recordsFor(count, { revision = 0 } = {}) { + assertCount(count); + if (revision !== 0 && revision !== 1) { + throw new RangeError(`revision must be 0 or 1, got ${revision}`); + } + for (let i = 0; i < count; i++) { + const id = recordId(i); + const text = + revision === 1 && i === 0 + ? `refreshed ${COMMON_KEYWORD} topic${topicFor(i)} ${needleTerm(i)} body` + : `${COMMON_KEYWORD} topic${topicFor(i)} ${needleTerm(i)} body`; + yield { id, source: sourceFor(i), text }; + } +} + +// Exactly 5 outgoing edges per record: i -> i+1 .. i+5 (wrapping). +// With count >= 10 there are no duplicates and no self edges. +export function* edgesFor(count) { + assertCount(count); + for (let i = 0; i < count; i++) { + for (let step = 1; step <= 5; step++) { + yield { source: recordId(i), target: recordId((i + step) % count), kind: "calls" }; + } + } +} + +export function expectedEdgeTargets(count, index) { + assertCount(count); + const out = []; + for (let step = 1; step <= 5; step++) out.push(recordId((index + step) % count)); + return out; +} + +export function expectedNeedleHits(count, index) { + assertCount(count); + if (index < 0 || index >= count) throw new RangeError(`index out of range: ${index}`); + return [recordId(index)]; +} + +export const CORPUS_LIMITS = Object.freeze({ MIN_COUNT, MAX_COUNT }); diff --git a/benchmarks/scale/postings-bench.mjs b/benchmarks/scale/postings-bench.mjs new file mode 100644 index 0000000..5364e11 --- /dev/null +++ b/benchmarks/scale/postings-bench.mjs @@ -0,0 +1,149 @@ +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { PostingIndex } from './postings.mjs'; +import { recordsFor, recordId, needleTerm } from './corpus.mjs'; + +function parseArgs(argv) { + let n = 10000; + let seen = false; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--records') { + if (seen) throw new Error('duplicate --records'); + seen = true; + const v = argv[++i]; + if (v === undefined) throw new Error('--records needs value'); + const num = Number(v); + if (![10000, 100000, 1000000].includes(num)) throw new Error('invalid N'); + n = num; + } else throw new Error('unexpected argument: ' + a); + } + return n; +} + +function pct(sorted, q) { + if (sorted.length === 0) return null; + const idx = Math.min(sorted.length - 1, Math.ceil(q * sorted.length) - 1); + return sorted[idx]; +} + +function sameIds(a, b) { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} + +const N = parseArgs(process.argv.slice(2)); +let dir = null; +let index = null; +let exitCode = 0; +const out = { + n: N, + nodeVersion: process.version, + platform: process.platform, + arch: process.arch, + correctness: false, + caveat: 'ID-only exact-token pages; no ranking, payload fetch, graph, hard deadline or internal SQLite work bound', +}; + +try { + dir = await mkdtemp(join(tmpdir(), 'z1p-postings-bench-')); + const dbPath = join(dir, 'index.db'); + index = new PostingIndex(dbPath); + + const t0 = performance.now(); + await index.build('bench', 'g1', recordsFor(N)); + const buildMs = performance.now() - t0; + + await index.close(); + out.buildMs = buildMs; + out.dbBytes = (await stat(dbPath)).size; + index = null; + + index = new PostingIndex(dbPath); + out.sqliteVersion = index.db.prepare('select sqlite_version() as v').get().v; + const plan = index.explainSearchPlan('bench', 'g1', 'shared'); + out.queryPlan = plan; + const planStr = plan.join(' '); + if (!planStr.includes('SEARCH') || !planStr.includes('PRIMARY KEY')) throw new Error('plan missing SEARCH PRIMARY KEY'); + if (planStr.includes('SCAN')) throw new Error('plan uses SCAN'); + if (planStr.includes('TEMP B-TREE')) throw new Error('plan uses TEMP B-TREE'); + + for (let s = 0; s < 8; s++) { + const term = needleTerm((s * 7919) % N); + await index.search('bench', 'g1', term, { limit: 10, maxPostings: 10 }); + await index.search('bench', 'g1', 'shared', { limit: 10, maxPostings: 10 }); + await index.search('bench', 'g1', 'shared', { limit: 10, maxPostings: 10, cursor: { version: 1, scope: 'bench', generation: 'g1', term: 'shared', after: recordId(N - 21) } }); + } + + const samples = { common: [], unique: [], deep: [] }; + let ok = true; + for (let s = 0; s < 32; s++) { + { + const t = performance.now(); + const r = await index.search('bench', 'g1', 'shared', { limit: 10, maxPostings: 10 }); + samples.common.push(performance.now() - t); + if (r.internalWorkMeasured !== false) ok = false; + if (r.postingsRead !== r.ids.length || r.ids.length > 10) ok = false; + if (r.complete !== false) ok = false; + const exp = []; + for (let i = 0; i < 10; i++) exp.push(recordId(i)); + if (!sameIds(r.ids, exp)) ok = false; + } + { + const term = needleTerm((s * 7919) % N); + const t = performance.now(); + const r = await index.search('bench', 'g1', term, { limit: 10, maxPostings: 10 }); + samples.unique.push(performance.now() - t); + if (r.internalWorkMeasured !== false) ok = false; + if (r.postingsRead !== r.ids.length || r.ids.length > 10) ok = false; + if (r.complete !== true) ok = false; + if (!sameIds(r.ids, [recordId((s * 7919) % N)])) ok = false; + } + { + const t = performance.now(); + const r = await index.search('bench', 'g1', 'shared', { limit: 10, maxPostings: 10, cursor: { version: 1, scope: 'bench', generation: 'g1', term: 'shared', after: recordId(N - 21) } }); + samples.deep.push(performance.now() - t); + if (r.internalWorkMeasured !== false) ok = false; + if (r.postingsRead !== r.ids.length || r.ids.length > 10) ok = false; + if (r.complete !== false) ok = false; + const exp = []; + for (let i = N - 20; i <= N - 11; i++) exp.push(recordId(i)); + if (!sameIds(r.ids, exp)) ok = false; + } + } + if (!ok) throw new Error('assertion failure'); + + out.maxRSSBytes = process.resourceUsage().maxRSS * 1024; + out.heapUsedBytes = process.memoryUsage().heapUsed; + out.correctness = true; + for (const [name, arr] of Object.entries(samples)) { + const sorted = arr.slice().sort((a, b) => a - b); + out[name] = { + p50: pct(sorted, 0.5), + p95: pct(sorted, 0.95), + min: sorted[0], + max: sorted[sorted.length - 1], + n: sorted.length, + }; + } + out.corpus = { records: N, edges: 0 }; + out.workloadNote = 'corpus counts are records only; not equivalent to graph workload'; +} catch (e) { + exitCode = 1; + out.error = String(e && e.message ? e.message : e); +} finally { + let cleanupError; + try { if (index) await index.close(); } catch (error) { cleanupError = error; } + try { if (dir) await rm(dir, { recursive: true, force: true }); } catch (error) { if (!cleanupError) cleanupError = error; } + if (cleanupError) { + process.exitCode = 1; + out.correctness = false; + out.cleanupError = String(cleanupError); + } +} + +console.log(JSON.stringify(out)); +if (exitCode) process.exitCode = exitCode; diff --git a/benchmarks/scale/postings.mjs b/benchmarks/scale/postings.mjs new file mode 100644 index 0000000..88d777e --- /dev/null +++ b/benchmarks/scale/postings.mjs @@ -0,0 +1,305 @@ +/** + * postings.mjs + * Isolated synthetic benchmark prototype: a SQLite-backed posting index with + * keyset-paginated single-token search. NOT a production API; no FTS, no + * external dependencies, no network. + * + * Tokenisation is ASCII-only: text is lowercased and split on [a-z0-9_]+. + * Non-ASCII characters act as separators and are not indexed. + */ + +import { DatabaseSync } from 'node:sqlite'; + +// ASCII-lowercase token pattern; intentionally limited (not production Unicode). +const TOKEN_RE = /[a-z0-9_]+/g; +const ID_RE = /^r\d{7}$/; +const MAX_TEXT = 65536; +const MAX_TOKEN = 128; + +function isControlFreeString(name) { + if (typeof name !== 'string' || name.length === 0 || name.length > 128) return false; + for (let i = 0; i < name.length; i++) { + const c = name.charCodeAt(i); + if (c < 32 || c === 127) return false; + } + return true; +} + +function validateTerm(term) { + if (typeof term !== 'string') throw new TypeError('term must be a string'); + if (term.length < 1 || term.length > MAX_TOKEN) { + throw new RangeError('term length must be 1..128'); + } + if (!/^[a-z0-9_]+$/.test(term)) { + throw new TypeError('term must be a single lowercase ASCII token [a-z0-9_]'); + } + return term; +} + +function validateLimit(limit) { + if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1 || limit > 100) { + throw new RangeError('limit must be an integer in 1..100'); + } + return limit; +} + +function validateMaxPostings(maxPostings) { + if ( + typeof maxPostings !== 'number' || + !Number.isInteger(maxPostings) || + maxPostings < 1 || + maxPostings > 10000 + ) { + throw new RangeError('maxPostings must be an integer in 1..10000'); + } + return maxPostings; +} + +function validateName(name) { + if (!isControlFreeString(name)) { + throw new TypeError('scope/generation must be 1..128 chars with no control characters'); + } + return name; +} + +// Validate a cursor object and equality with the current query context. +// `cursor === undefined` means "no cursor"; `null` is rejected as invalid. +function validateCursor(cursor, { scope, generation, term }) { + if (cursor === undefined) return null; + if (cursor === null || typeof cursor !== 'object' || Array.isArray(cursor)) { + throw new TypeError('cursor must be a plain object'); + } + if (cursor.version !== 1) { + throw new TypeError('cursor.version must be 1'); + } + if (cursor.scope !== scope) { + throw new TypeError('cursor.scope mismatch'); + } + if (cursor.generation !== generation) { + throw new TypeError('cursor.generation mismatch'); + } + if (cursor.term !== term) { + throw new TypeError('cursor.term mismatch'); + } + const after = cursor.after; + if (typeof after !== 'string' || !ID_RE.test(after)) { + throw new TypeError('cursor.after must be a valid record id'); + } + return after; +} + +export class PostingIndex { + /** + * @param {string} [path] SQLite file path; defaults to an in-memory db. + */ + constructor(path = ':memory:') { + this.db = new DatabaseSync(path); + this._closed = false; + this.db.exec('PRAGMA foreign_keys = ON;'); + this._init(); + } + + _init() { + // generations: track existing (scope, generation) pairs so we can reject + // duplicate builds including empty ones, and detect unknown generations. + this.db.exec(` + CREATE TABLE IF NOT EXISTS generations ( + scope TEXT NOT NULL, + generation TEXT NOT NULL, + PRIMARY KEY (scope, generation) + ) WITHOUT ROWID + `); + // records: store record IDs only, for transactional duplicate detection + // without a corpus-sized in-memory Set. FK inline to generations. + this.db.exec(` + CREATE TABLE IF NOT EXISTS records ( + scope TEXT NOT NULL, + generation TEXT NOT NULL, + id TEXT NOT NULL, + PRIMARY KEY (scope, generation, id), + FOREIGN KEY (scope, generation) REFERENCES generations (scope, generation) + ON DELETE CASCADE + ) WITHOUT ROWID + `); + // postings: composite PRIMARY KEY without rowid; supports indexed keyset + // scan on (scope, generation, term, id) with no sort and no full scan. + // FK inline to generations (SQLite does not support ADD CONSTRAINT). + this.db.exec(` + CREATE TABLE IF NOT EXISTS postings ( + scope TEXT NOT NULL, + generation TEXT NOT NULL, + term TEXT NOT NULL, + id TEXT NOT NULL, + PRIMARY KEY (scope, generation, term, id), + FOREIGN KEY (scope, generation) REFERENCES generations (scope, generation) + ON DELETE CASCADE + ) WITHOUT ROWID + `); + } + + /** Idempotent close. */ + close() { + if (this.db && !this._closed) { + try { + this.db.close(); + } finally { + this._closed = true; + } + } + } + + /** + * Build a generation from a records iterable. + */ + build(scope, generation, recordsIterable) { + scope = validateName(scope); + generation = validateName(generation); + + const exists = this.db + .prepare('SELECT 1 FROM generations WHERE scope = ? AND generation = ?') + .get(scope, generation); + if (exists) { + throw new Error(`duplicate generation: ${scope}/${generation}`); + } + + if (recordsIterable == null || typeof recordsIterable[Symbol.iterator] !== 'function') { + throw new TypeError('recordsIterable must be iterable'); + } + + this.db.exec('BEGIN IMMEDIATE'); + try { + // Insert the generation row even for empty corpora, so empty generations + // are tracked and duplicate detection works for them too. + this.db + .prepare('INSERT INTO generations (scope, generation) VALUES (?, ?)') + .run(scope, generation); + + const insRecord = this.db.prepare( + 'INSERT INTO records (scope, generation, id) VALUES (?, ?, ?)' + ); + const insPosting = this.db.prepare( + 'INSERT OR IGNORE INTO postings (scope, generation, term, id) VALUES (?, ?, ?, ?)' + ); + + // for-of ensures the iterator's return() is invoked on any throw, so the + // caller's generator can release resources deterministically. + for (const record of recordsIterable) { + if (record == null || typeof record !== 'object') { + throw new TypeError('record must be an object'); + } + const id = record.id; + const text = record.text; + if (typeof id !== 'string' || !ID_RE.test(id)) { + throw new TypeError('record.id must match /^r\\d{7}$/'); + } + if (typeof text !== 'string') { + throw new TypeError('record.text must be a string'); + } + if (text.length > MAX_TEXT) { + throw new RangeError('record.text exceeds 65536 chars'); + } + insRecord.run(scope, generation, id); + + const seen = new Set(); + const lower = text.toLowerCase(); + TOKEN_RE.lastIndex = 0; + let m; + while ((m = TOKEN_RE.exec(lower)) !== null) { + const token = m[0]; + if (token.length > MAX_TOKEN) { + throw new RangeError('token exceeds 128 chars'); + } + if (seen.has(token)) continue; + seen.add(token); + insPosting.run(scope, generation, token, id); + } + } + + this.db.exec('COMMIT'); + } catch (err) { + this.db.exec('ROLLBACK'); + throw err; + } + } + + /** + * Keyset-paginated search over a single token. + * + * @returns {{ids: string[], postingsRead: number, complete: boolean, + * cursor: object|undefined, internalWorkMeasured: boolean}} + */ + search(scope, generation, term, { limit = 10, maxPostings = 100, cursor } = {}) { + scope = validateName(scope); + generation = validateName(generation); + term = validateTerm(term); + limit = validateLimit(limit); + maxPostings = validateMaxPostings(maxPostings); + + // Validate cursor BEFORE any SQL query. + const after = validateCursor(cursor, { scope, generation, term }); + + const genExists = this.db + .prepare('SELECT 1 FROM generations WHERE scope = ? AND generation = ?') + .get(scope, generation); + if (!genExists) { + throw new Error(`unknown generation: ${scope}/${generation}`); + } + + const cap = Math.min(limit, maxPostings); + + const stmt = this.db.prepare( + `SELECT id FROM postings + WHERE scope = ? AND generation = ? AND term = ? AND id > ? + ORDER BY id + LIMIT ?` + ); + const rows = stmt.all(scope, generation, term, after || '', cap); + const ids = rows.map((r) => r.id); + const postingsRead = rows.length; + // Conservative completion: an exactly-full page cannot prove exhaustion; + // a subsequent empty page confirms it. complete iff page < cap. + const complete = postingsRead < cap; + let nextCursor = undefined; + if (!complete) { + nextCursor = { + version: 1, + scope, + generation, + term, + after: ids[ids.length - 1], + }; + } + return { + ids, + postingsRead, + complete, + cursor: nextCursor, + internalWorkMeasured: false, + }; + } + + /** + * Return SQLite EXPLAIN QUERY PLAN detail strings for the search shape. + */ + explainSearchPlan(scope, generation, term) { + scope = validateName(scope); + generation = validateName(generation); + term = validateTerm(term); + const genExists = this.db + .prepare('SELECT 1 FROM generations WHERE scope = ? AND generation = ?') + .get(scope, generation); + if (!genExists) { + throw new Error(`unknown generation: ${scope}/${generation}`); + } + const rows = this.db + .prepare( + `EXPLAIN QUERY PLAN + SELECT id FROM postings + WHERE scope = ? AND generation = ? AND term = ? AND id > '' + ORDER BY id + LIMIT 10` + ) + .all(scope, generation, term); + return rows.map((r) => r.detail); + } +} diff --git a/benchmarks/scale/postings.test.mjs b/benchmarks/scale/postings.test.mjs new file mode 100644 index 0000000..25428db --- /dev/null +++ b/benchmarks/scale/postings.test.mjs @@ -0,0 +1,383 @@ +/** + * postings.test.mjs + * Tests for the synthetic posting-index prototype. Node 24 ESM, node:test. + * No external dependencies or network access. + */ + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PostingIndex } from './postings.mjs'; + +// 13 records; each text contains the token "shared", "needle", and a number. +function makeRecords(n = 13) { + const out = []; + for (let i = 0; i < n; i++) { + out.push({ + id: 'r' + String(i).padStart(7, '0'), + text: 'shared needle' + i, + }); + } + return out; +} + +// Collect pages until complete. Because a full page is never marked complete, +// the terminating page is the FIRST page with fewer than cap rows (possibly +// empty). We assert that final page's shape; callers can inspect earlier pages. +function collectAll(idx, scope, generation, term, { limit, maxPostings }) { + const collected = []; + const cap = Math.min(limit, maxPostings); + let page = idx.search(scope, generation, term, { limit, maxPostings }); + assert.ok(page.ids.length <= cap, 'page respects cap'); + collected.push(...page.ids); + let pageCount = 1; + while (!page.complete) { + assert.ok(page.cursor && typeof page.cursor === 'object', 'incomplete page has cursor'); + page = idx.search(scope, generation, term, { + limit, + maxPostings, + cursor: page.cursor, + }); + pageCount++; + assert.ok(page.ids.length <= cap, 'page respects cap'); + collected.push(...page.ids); + } + // Complete page has no cursor and fewer than `cap` rows. + assert.strictEqual(page.cursor, undefined, 'complete page has no cursor'); + assert.ok(page.ids.length < cap, 'complete page is not full'); + return { collected, pageCount, lastPage: page }; +} + +test('pagination over 13 shared records with limit5/maxPostings3', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g', makeRecords(13)); + const { collected, lastPage, pageCount } = collectAll(idx, 's', 'g', 'shared', { + limit: 5, + maxPostings: 3, + }); + assert.strictEqual(collected.length, 13, 'all 13 ids collected'); + // 13 = 4 full pages of 3 + one final page of 1. + assert.strictEqual(pageCount, 5, 'four full pages + one row'); + assert.strictEqual(lastPage.ids.length, 1, 'final page has ONE row'); + assert.strictEqual(lastPage.complete, true, 'short final page is complete'); + // Sorted ascending (string compare on zero-padded ids). + const sorted = [...collected].sort(); + assert.deepEqual(collected, sorted, 'ids are in ascending order'); + } finally { + idx.close(); + } +}); + +test('exact 3-row end requires empty continuation', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g', [ + { id: 'r0000001', text: 'alpha one' }, + { id: 'r0000002', text: 'alpha two' }, + { id: 'r0000003', text: 'alpha three' }, + ]); + // Inspect the FIRST page directly: exactly cap rows -> incomplete. + const first = idx.search('s', 'g', 'alpha', { limit: 3, maxPostings: 3 }); + assert.strictEqual(first.ids.length, 3, 'first page is full'); + assert.strictEqual(first.complete, false, 'exactly-full page not complete'); + assert.ok(first.cursor, 'incomplete page has cursor'); + // Continuation is empty and complete. + const cont = idx.search('s', 'g', 'alpha', { + limit: 3, + maxPostings: 3, + cursor: first.cursor, + }); + assert.deepStrictEqual(cont.ids, [], 'continuation empty'); + assert.strictEqual(cont.postingsRead, 0, 'zero rows read'); + assert.strictEqual(cont.complete, true, 'empty page complete'); + assert.strictEqual(cont.cursor, undefined, 'empty page no cursor'); + } finally { + idx.close(); + } +}); + +test('empty generation is tracked and searchable as empty', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g-empty', []); + const page = idx.search('s', 'g-empty', 'shared', { limit: 5, maxPostings: 3 }); + assert.deepStrictEqual(page.ids, []); + assert.strictEqual(page.postingsRead, 0); + assert.strictEqual(page.complete, true); + assert.strictEqual(page.cursor, undefined); + assert.throws(() => idx.search('s', 'nope', 'shared'), /unknown generation/); + } finally { + idx.close(); + } +}); + +test('duplicate empty generation is rejected', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g-empty', []); + assert.throws(() => idx.build('s', 'g-empty', []), /duplicate generation/); + } finally { + idx.close(); + } +}); + +test('scope isolation: same IDs, different tokens; missing token both ways', () => { + const idx = new PostingIndex(); + try { + // Same ids, DIFFERENT tokens per scope. + idx.build('scope-a', 'g', [ + { id: 'r0000001', text: 'alpha shared' }, + { id: 'r0000002', text: 'alpha shared' }, + ]); + idx.build('scope-b', 'g', [ + { id: 'r0000001', text: 'beta shared' }, + { id: 'r0000002', text: 'beta shared' }, + ]); + const aShared = idx.search('scope-a', 'g', 'shared', { limit: 100, maxPostings: 100 }); + const bShared = idx.search('scope-b', 'g', 'shared', { limit: 100, maxPostings: 100 }); + assert.deepEqual(aShared.ids, ['r0000001', 'r0000002']); + assert.deepEqual(bShared.ids, ['r0000001', 'r0000002']); + // "alpha" only exists in scope-a; "beta" only exists in scope-b. + const aAlpha = idx.search('scope-a', 'g', 'alpha', { limit: 100, maxPostings: 100 }); + const aBeta = idx.search('scope-a', 'g', 'beta', { limit: 100, maxPostings: 100 }); + const bAlpha = idx.search('scope-b', 'g', 'alpha', { limit: 100, maxPostings: 100 }); + const bBeta = idx.search('scope-b', 'g', 'beta', { limit: 100, maxPostings: 100 }); + assert.deepEqual(aAlpha.ids, ['r0000001', 'r0000002']); + assert.deepEqual(aBeta.ids, [], 'beta must NOT appear in scope-a'); + assert.deepEqual(bAlpha.ids, [], 'alpha must NOT appear in scope-b'); + assert.deepEqual(bBeta.ids, ['r0000001', 'r0000002']); + } finally { + idx.close(); + } +}); + +test('pinned generation remains after newer build', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g1', makeRecords(5)); + idx.build('s', 'g2', makeRecords(8)); + const g1 = idx.search('s', 'g1', 'shared', { limit: 100, maxPostings: 100 }); + const g2 = idx.search('s', 'g2', 'shared', { limit: 100, maxPostings: 100 }); + assert.strictEqual(g1.ids.length, 5); + assert.strictEqual(g2.ids.length, 8); + const g1again = idx.search('s', 'g1', 'shared', { limit: 100, maxPostings: 100 }); + assert.strictEqual(g1again.ids.length, 5); + } finally { + idx.close(); + } +}); + +test('invalid and mismatched cursors are rejected (before any SQL)', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g', makeRecords(13)); + idx.build('s2', 'g2', makeRecords(3)); + const p = idx.search('s', 'g', 'shared', { limit: 3, maxPostings: 3 }); + const cur = p.cursor; + assert.ok(cur, 'first page returns a cursor'); + + // Mismatched scope: query existing scope 's2'. + assert.throws( + () => idx.search('s2', 'g2', 'shared', { limit: 3, maxPostings: 3, cursor: cur }), + /cursor\.scope mismatch/ + ); + // Mismatched generation: query existing generation 'g2' under 's'. + assert.throws( + () => idx.search('s', 'g2', 'shared', { limit: 3, maxPostings: 3, cursor: cur }), + /cursor\.generation mismatch/ + ); + // Mismatched term. + assert.throws( + () => idx.search('s', 'g', 'needle', { limit: 3, maxPostings: 3, cursor: cur }), + /cursor\.term mismatch/ + ); + // Wrong version. + assert.throws( + () => + idx.search('s', 'g', 'shared', { + limit: 3, + maxPostings: 3, + cursor: { version: 2, scope: 's', generation: 'g', term: 'shared', after: 'r0000001' }, + }), + /cursor\.version/ + ); + // Invalid after id. + assert.throws( + () => + idx.search('s', 'g', 'shared', { + limit: 3, + maxPostings: 3, + cursor: { version: 1, scope: 's', generation: 'g', term: 'shared', after: 'bad' }, + }), + /cursor\.after/ + ); + // Non-object cursor. + assert.throws( + () => idx.search('s', 'g', 'shared', { limit: 3, maxPostings: 3, cursor: 'x' }), + /cursor must be a plain object/ + ); + // Plain null cursor is invalid (not the same as "no cursor"). + assert.throws( + () => idx.search('s', 'g', 'shared', { limit: 3, maxPostings: 3, cursor: null }), + /cursor must be a plain object/ + ); + } finally { + idx.close(); + } +}); + +test('invalid term / limits / maxPostings are rejected', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g', makeRecords(3)); + assert.throws(() => idx.search('s', 'g', 123), /term must be a string/); + assert.throws(() => idx.search('s', 'g', 'Shared'), /single lowercase ASCII token/); + assert.throws(() => idx.search('s', 'g', 'shared x'), /single lowercase ASCII token/); + assert.throws(() => idx.search('s', 'g', ''), /term length/); + assert.throws(() => idx.search('s', 'g', 'a'.repeat(129)), /term length/); + assert.throws(() => idx.search('s', 'g', 'shared', { limit: 0 }), /limit/); + assert.throws(() => idx.search('s', 'g', 'shared', { limit: 101 }), /limit/); + assert.throws(() => idx.search('s', 'g', 'shared', { limit: 1.5 }), /limit/); + assert.throws(() => idx.search('s', 'g', 'shared', { maxPostings: 0 }), /maxPostings/); + assert.throws(() => idx.search('s', 'g', 'shared', { maxPostings: 10001 }), /maxPostings/); + } finally { + idx.close(); + } +}); + +test('throwing iterator rolls back, closes iterator, and allows retry', () => { + const idx = new PostingIndex(); + try { + let returned = false; + function* throwingGenerator() { + try { + yield { id: 'r0000001', text: 'shared one' }; + throw new Error('boom mid-stream'); + } finally { + returned = true; + } + } + assert.throws(() => idx.build('s', 'g', throwingGenerator()), /boom mid-stream/); + assert.strictEqual(returned, true, 'iterator return() was invoked on error'); + // Rolled back; retry allowed. + assert.throws(() => idx.search('s', 'g', 'shared'), /unknown generation/); + idx.build('s', 'g', makeRecords(4)); + const page = idx.search('s', 'g', 'shared', { limit: 100, maxPostings: 100 }); + assert.strictEqual(page.ids.length, 4); + } finally { + idx.close(); + } +}); + +test('duplicate record id in a generation rolls back', () => { + const idx = new PostingIndex(); + try { + const records = [ + { id: 'r0000001', text: 'shared one' }, + { id: 'r0000001', text: 'shared two' }, + ]; + assert.throws( + () => idx.build('s', 'g', records), + /SQLITE_CONSTRAINT|constraint|PRIMARY/i + ); + assert.throws(() => idx.search('s', 'g', 'shared'), /unknown generation/); + idx.build('s', 'g', [ + { id: 'r0000001', text: 'shared one' }, + { id: 'r0000002', text: 'shared two' }, + ]); + const page = idx.search('s', 'g', 'shared', { limit: 100, maxPostings: 100 }); + assert.deepEqual(page.ids, ['r0000001', 'r0000002']); + } finally { + idx.close(); + } +}); + +test('repeated token appears once per record', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g', [ + { id: 'r0000001', text: 'shared shared shared' }, + { id: 'r0000002', text: 'shared needle' }, + ]); + const page = idx.search('s', 'g', 'shared', { limit: 100, maxPostings: 100 }); + assert.deepEqual(page.ids, ['r0000001', 'r0000002']); + assert.strictEqual(page.ids.length, 2, 'token deduped per record'); + } finally { + idx.close(); + } +}); + +test('overlong token is rejected, never truncated', () => { + const idx = new PostingIndex(); + try { + const longToken = 'a'.repeat(129); + assert.throws( + () => idx.build('s', 'g', [{ id: 'r0000001', text: longToken }]), + /token exceeds 128/ + ); + assert.throws(() => idx.search('s', 'g', 'a'.repeat(128)), /unknown generation/); + } finally { + idx.close(); + } +}); + +test('reopen temp file: cursor captured before close works after reopen', async () => { + const dir = await mkdtemp(join(tmpdir(), 'postings-')); + const path = join(dir, 'index.sqlite'); + let idx2; + try { + const idx = new PostingIndex(path); + idx.build('s', 'g', makeRecords(13)); + idx.build('s', 'g2', makeRecords(7)); + // Capture the cursor BEFORE closing. + const first = idx.search('s', 'g', 'shared', { limit: 3, maxPostings: 3 }); + assert.strictEqual(first.ids.length, 3); + assert.strictEqual(first.complete, false); + const captured = first.cursor; + assert.ok(captured, 'cursor captured before close'); + idx.close(); + + idx2 = new PostingIndex(path); + // Generation preserved. + const p2 = idx2.search('s', 'g2', 'shared', { limit: 100, maxPostings: 100 }); + assert.strictEqual(p2.ids.length, 7); + // Captured cursor still advances after reopen. + const next = idx2.search('s', 'g', 'shared', { + limit: 3, + maxPostings: 3, + cursor: captured, + }); + assert.strictEqual(next.ids.length, 3, 'cursor advanced with full page'); + assert.deepEqual(next.ids, ['r0000003', 'r0000004', 'r0000005']); + // Full pagination still works after reopen. + const { collected } = collectAll(idx2, 's', 'g', 'shared', { limit: 3, maxPostings: 3 }); + assert.strictEqual(collected.length, 13); + } finally { + if (idx2) idx2.close(); + await rm(dir, { recursive: true, force: true }); + } +}); + +test('explainSearchPlan: PK index SEARCH, no SCAN, no temp sort', () => { + const idx = new PostingIndex(); + try { + idx.build('s', 'g', makeRecords(13)); + const details = idx.explainSearchPlan('s', 'g', 'shared'); + assert.ok(Array.isArray(details) && details.length > 0, 'has detail strings'); + for (const d of details) assert.strictEqual(typeof d, 'string'); + const joined = details.join('\n'); + assert.match(joined, /SEARCH/, 'uses an indexed SEARCH'); + assert.match(joined, /PRIMARY KEY/, 'uses the composite PRIMARY KEY index'); + assert.doesNotMatch(joined, /\bSCAN\b/, 'no full table SCAN'); + assert.doesNotMatch(joined, /TEMP B-TREE/i, 'no temp b-tree (sort)'); + assert.doesNotMatch(joined, /USE TEMP/i, 'no temp structure for sort'); + // Input validation. + assert.throws(() => idx.explainSearchPlan('s', 'g', 'Bad'), /lowercase ASCII token/); + assert.throws(() => idx.explainSearchPlan('s', 'nope', 'shared'), /unknown generation/); + } finally { + idx.close(); + } +}); diff --git a/benchmarks/scale/sqlite.mjs b/benchmarks/scale/sqlite.mjs new file mode 100644 index 0000000..95b749b --- /dev/null +++ b/benchmarks/scale/sqlite.mjs @@ -0,0 +1,485 @@ +// Isolated, synchronous benchmark harness for Z1P Core scale evaluation. +// Public synthetic data only; no network; no private repositories. +// +// ARCHITECTURE BOUNDARY / NON-CLAIMS +// - This evaluates indexed local query over synthetic data BEFORE any v2 crypto work. +// Every SQL statement names both scope and generation explicitly; there is no +// "global query then filter in JS" path. +// - Generations are built transactionally and the current pointer is switched only +// after all records and edges are inserted. A failed build leaves the prior +// generation untouched. This is NOT a production encrypted index, and it is not +// an access-control proof. +// - The synchronous API is NOT a cancellable production worker. Deadline handling, +// if any, can only be best-effort between synchronous calls. It is never a bound +// on internal SQLite/FTS engine work; LIMIT does not bound internal engine work. +// - Counters are exact per-result (rows returned). Internal engine work is reported +// as unmeasured, and FTS relevance is not claimed to be exact top-k. + +import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { + recordsFor, + edgesFor, + recordId, + needleTerm, + expectedEdgeTargets, + CORPUS_LIMITS, +} from "./corpus.mjs"; + +const MAX_LIMIT = 100; + +function assertName(value, label) { + if (typeof value !== "string" || value.length === 0 || value.length > 128 || /[\u0000-\u001f]/.test(value)) { + throw new TypeError(`${label} must be a nonempty string of length 1..128 without control chars`); + } +} + +function assertId(value, label) { + if (typeof value !== "string" || !/^r\d{7}$/.test(value)) { + throw new TypeError(`${label} must match ^r\\d{7}$`); + } +} + +function assertLimit(limit) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIMIT) { + throw new RangeError(`limit must be a safe integer in [1, ${MAX_LIMIT}]`); + } +} + +// Treat caller text as a literal FTS phrase by wrapping in a quoted phrase and +// doubling any embedded double quotes. Never interprets raw FTS grammar. +function toLiteralFtsPhrase(term) { + if (typeof term !== "string" || term.length === 0) { + throw new TypeError("search term must be a nonempty string"); + } + return `"${term.replace(/"/g, '""')}"`; +} + +const SCHEMA = ` +PRAGMA foreign_keys = ON; +CREATE TABLE records ( + scope TEXT NOT NULL, + generation TEXT NOT NULL, + id TEXT NOT NULL, + source TEXT NOT NULL, + text TEXT NOT NULL, + PRIMARY KEY (scope, generation, id) +); +CREATE TABLE edges ( + scope TEXT NOT NULL, + generation TEXT NOT NULL, + source TEXT NOT NULL, + target TEXT NOT NULL, + kind TEXT NOT NULL, + PRIMARY KEY (scope, generation, source, target, kind), + FOREIGN KEY (scope, generation, source) REFERENCES records (scope, generation, id), + FOREIGN KEY (scope, generation, target) REFERENCES records (scope, generation, id) +); +CREATE INDEX edges_forward ON edges (scope, generation, source, kind, target); +CREATE INDEX edges_reverse ON edges (scope, generation, target, kind, source); +CREATE TABLE current ( + scope TEXT PRIMARY KEY, + generation TEXT NOT NULL +); +CREATE VIRTUAL TABLE records_fts USING fts5 ( + scope UNINDEXED, + generation UNINDEXED, + id UNINDEXED, + text +); +`; + +export class ScaleIndex { + #db; + #closed = false; + + constructor(path) { + if (typeof path !== "string" || path.length === 0) { + throw new TypeError("path must be a nonempty string"); + } + this.#db = new DatabaseSync(path); + this.#db.exec(SCHEMA); + } + + close() { + if (this.#closed) return; + this.#closed = true; + this.#db.close(); + } + + build(scope, generation, recordsIterable, edgesIterable) { + assertName(scope, "scope"); + assertName(generation, "generation"); + if (recordsIterable == null || edgesIterable == null) { + throw new TypeError("recordsIterable and edgesIterable are required"); + } + + const existing = this.#db + .prepare("SELECT 1 AS present FROM current WHERE scope = ?") + .get(scope); + + const genPresent = this.#db + .prepare("SELECT 1 AS present FROM records WHERE scope = ? AND generation = ? LIMIT 1") + .get(scope, generation); + if (genPresent) { + throw new Error(`generation already present for scope: ${scope}/${generation}`); + } + void existing; + + const insertRecord = this.#db.prepare( + "INSERT INTO records (scope, generation, id, source, text) VALUES (?, ?, ?, ?, ?)" + ); + const insertEdge = this.#db.prepare( + "INSERT INTO edges (scope, generation, source, target, kind) VALUES (?, ?, ?, ?, ?)" + ); + const insertFts = this.#db.prepare( + "INSERT INTO records_fts (scope, generation, id, text) VALUES (?, ?, ?, ?)" + ); + const knownId = this.#db.prepare( + "SELECT 1 AS present FROM records WHERE scope = ? AND generation = ? AND id = ? LIMIT 1" + ); + const switchCurrent = this.#db.prepare( + "INSERT INTO current (scope, generation) VALUES (?, ?) " + + "ON CONFLICT (scope) DO UPDATE SET generation = excluded.generation" + ); + + this.#db.exec("BEGIN IMMEDIATE"); + try { + for (const record of recordsIterable) { + if (record == null || typeof record !== "object") { + throw new TypeError("record must be an object"); + } + assertId(record.id, "record.id"); + assertName(record.source, "record.source"); + if (typeof record.text !== "string") { + throw new TypeError("record.text must be a string"); + } + insertRecord.run(scope, generation, record.id, record.source, record.text); + insertFts.run(scope, generation, record.id, record.text); + } + + for (const edge of edgesIterable) { + if (edge == null || typeof edge !== "object") { + throw new TypeError("edge must be an object"); + } + assertId(edge.source, "edge.source"); + assertId(edge.target, "edge.target"); + assertName(edge.kind, "edge.kind"); + if (!knownId.get(scope, generation, edge.source)) { + throw new Error(`dangling edge source: ${edge.source}`); + } + if (!knownId.get(scope, generation, edge.target)) { + throw new Error(`dangling edge target: ${edge.target}`); + } + insertEdge.run(scope, generation, edge.source, edge.target, edge.kind); + } + + switchCurrent.run(scope, generation); + this.#db.exec("COMMIT"); + } catch (err) { + try { + this.#db.exec("ROLLBACK"); + } catch { + // best-effort rollback; preserve original error + } + throw err; + } + } + + activeGeneration(scope) { + assertName(scope, "scope"); + const row = this.#db + .prepare("SELECT generation FROM current WHERE scope = ?") + .get(scope); + return row ? row.generation : undefined; + } + + lookup(scope, generation, id) { + assertName(scope, "scope"); + assertName(generation, "generation"); + assertId(id, "id"); + const row = this.#db + .prepare( + "SELECT id, source, text FROM records WHERE scope = ? AND generation = ? AND id = ?" + ) + .get(scope, generation, id); + return row ? { id: row.id, source: row.source, text: row.text } : undefined; + } + + search(scope, generation, term, limit) { + assertName(scope, "scope"); + assertName(generation, "generation"); + assertLimit(limit); + const phrase = toLiteralFtsPhrase(term); + const rows = this.#db + .prepare( + "SELECT r.id AS id, r.source AS source, r.text AS text " + + "FROM records_fts f JOIN records r " + + "ON r.scope = f.scope AND r.generation = f.generation AND r.id = f.id " + + "WHERE f.scope = ? AND f.generation = ? AND records_fts MATCH ? " + + "ORDER BY r.id ASC LIMIT ?" + ) + .all(scope, generation, phrase, limit); + const results = rows.map((r) => ({ id: r.id, source: r.source, text: r.text })); + return { + results, + count: results.length, + internalWorkMeasured: false, + relevanceTopKExact: false, + }; + } + + neighbours(scope, generation, id, limit) { + assertName(scope, "scope"); + assertName(generation, "generation"); + assertId(id, "id"); + assertLimit(limit); + const rows = this.#db + .prepare( + "SELECT target FROM edges " + + "WHERE scope = ? AND generation = ? AND source = ? AND kind = 'calls' " + + "ORDER BY target ASC LIMIT ?" + ) + .all(scope, generation, id, limit); + const results = rows.map((r) => r.target); + return { + results, + count: results.length, + internalWorkMeasured: false, + }; + } +} + +// --------------------------------------------------------------------------- +// CLI: node benchmarks/scale/sqlite.mjs [--records N] +// --------------------------------------------------------------------------- + +const ALLOWED_RECORDS = new Set([10000, 100000, 1000000]); +const QUERY_SAMPLES = 32; + +function parseArgs(argv) { + let records = 10000; + let sawRecords = false; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--records") { + const value = argv[i + 1]; + if (value === undefined) throw new Error("--records requires a value"); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`invalid --records value: ${value}`); + records = parsed; + sawRecords = true; + i++; + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + if (sawRecords && !ALLOWED_RECORDS.has(records)) { + throw new Error(`--records must be one of 10000, 100000, 1000000 (got ${records})`); + } + return { records }; +} + +function summariseLatencies(samples) { + if (samples.length === 0) return { n: 0, p50: null, p95: null, min: null, max: null, mean: null }; + const sorted = [...samples].sort((a, b) => a - b); + const pick = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]; + const total = samples.reduce((acc, v) => acc + v, 0); + return { + n: samples.length, + p50: pick(0.5), + p95: pick(0.95), + min: sorted[0], + max: sorted[sorted.length - 1], + mean: total / samples.length, + }; +} + +async function runCli(argv) { + const { records } = parseArgs(argv); + const edgeCount = records * 5; + const dir = await mkdtemp(join(tmpdir(), "z1p-scale-")); + const dbPath = join(dir, "scale.db"); + const out = { + nodeVersion: process.version, + sqliteVersion: null, + records, + edges: edgeCount, + buildMs: null, + databaseBytes: null, + peakRssBytes: null, + peakRssUnits: "bytes (process.resourceUsage().maxRSS is KiB on POSIX; converted)", + heapUsedBytes: null, + firstQuery: { ms: null, note: "first query in this process; not a true cold-OS-cache claim" }, + warmLookup: null, + warmSearch: null, + warmNeighbours: null, + counters: { + searchReturnedRows: null, + neighboursReturnedRows: null, + internalWorkMeasured: false, + relevanceTopKExact: false, + }, + recall: { + lookupExpectedSourceHits: 0, + lookupSampled: 0, + searchExpectedSourceHits: 0, + searchSampled: 0, + }, + scopeIsolationOk: false, + deadline: "best-effort between synchronous calls only; not a bound on internal engine work", + correctness: false, + }; + + const index = new ScaleIndex(dbPath); + try { + const sqliteRow = index && new DatabaseSync(":memory:"); + // sqlite version via a separate handle (index owns its own). + { + const probe = new DatabaseSync(":memory:"); + try { + out.sqliteVersion = probe.prepare("SELECT sqlite_version() AS v").get().v; + } finally { + probe.close(); + } + sqliteRow.close(); + } + + const buildStart = performance.now(); + index.build("bench", "gen-1", recordsFor(records), edgesFor(records)); + out.buildMs = performance.now() - buildStart; + + // Scope isolation fixture: same record IDs under a different scope. + // Uses auxiliary record text to avoid materialising the primary records. + const auxiliaryRecords = Array.from(recordsFor(10), (r) => ({ ...r, text: "scopeonlysecret" })); + index.build("bench-other", "gen-1", auxiliaryRecords, []); + out.auxiliaryRecords = 10; + out.auxiliaryEdges = 0; + out.totalStoredRecords = records + 10; + out.totalStoredEdges = edgeCount; + out.buildTimeScope = "primary"; + + const expectedCurrent = "gen-1"; + if (index.activeGeneration("bench") !== expectedCurrent) throw new Error("active generation mismatch"); + + // First query: separate from warm distribution. + const firstStart = performance.now(); + const firstLookup = index.lookup("bench", "gen-1", recordId(0)); + out.firstQuery.ms = performance.now() - firstStart; + if (!firstLookup || firstLookup.id !== recordId(0)) throw new Error("first lookup failed"); + + const warmLookup = []; + const warmSearch = []; + const warmNeighbours = []; + + for (let s = 0; s < QUERY_SAMPLES; s++) { + const target = (s * 7919) % records; + const id = recordId(target); + + let t0 = performance.now(); + const rec = index.lookup("bench", "gen-1", id); + warmLookup.push(performance.now() - t0); + out.recall.lookupSampled++; + if (rec && rec.id === id) out.recall.lookupExpectedSourceHits++; + + t0 = performance.now(); + const searchRes = index.search("bench", "gen-1", needleTerm(target), 10); + warmSearch.push(performance.now() - t0); + out.recall.searchSampled++; + out.counters.searchReturnedRows = searchRes.count; + if (searchRes.results.length === 1 && searchRes.results[0].id === id) { + out.recall.searchExpectedSourceHits++; + } + + t0 = performance.now(); + const nb = index.neighbours("bench", "gen-1", id, 5); + warmNeighbours.push(performance.now() - t0); + out.counters.neighboursReturnedRows = nb.count; + + // Scope isolation: identical IDs under other scope must be independent. + if (s === 0) { + const other = index.lookup("bench-other", "gen-1", id); + const leak = index.search("bench-other", "gen-1", needleTerm(target), 10); + if (!other || other.text === rec.text || leak.count !== 0 || index.search("bench", "gen-1", "scopeonlysecret", 10).count !== 0 || index.search("bench-other", "gen-1", "scopeonlysecret", 10).count !== 10 || index.neighbours("bench-other", "gen-1", id, 5).count !== 0) { + throw new Error("scope isolation check failed"); + } + out.scopeIsolationOk = true; + } + } + + out.warmLookup = summariseLatencies(warmLookup); + out.warmSearch = summariseLatencies(warmSearch); + out.warmNeighbours = summariseLatencies(warmNeighbours); + const commonTermSearch = []; + for (let i = 0; i < QUERY_SAMPLES; i++) { + const t0 = performance.now(); + const res = index.search("bench", "gen-1", "shared", 10); + const elapsed = performance.now() - t0; + commonTermSearch.push(elapsed); + if (res.count !== 10) { + throw new Error(`Expected 10 results, got ${res.count}`); + } + for (let j = 0; j < res.results.length; j++) { + const row = res.results[j]; + const expectedId = recordId(j); + if (row.id !== expectedId) { + throw new Error(`Expected row id ${expectedId} at position ${j}, got ${row.id}`); + } + } + } + out.commonTermSearch = summariseLatencies(commonTermSearch); + out.heapUsedBytes = process.memoryUsage().heapUsed; + + const nb0 = index.neighbours("bench", "gen-1", recordId(0), 5); + const expected = expectedEdgeTargets(records, 0); + const nbOk = + nb0.count === 5 && nb0.results.length === 5 && nb0.results.every((v, i) => v === expected[i]); + if (!nbOk) throw new Error("neighbour expectation mismatch"); + + out.correctness = + out.recall.lookupSampled === out.recall.lookupExpectedSourceHits && + out.recall.searchSampled === out.recall.searchExpectedSourceHits && + out.scopeIsolationOk === true; + + if (typeof process.report?.getReport === "function") { + const report = process.report.getReport(); + if (report && report.header && typeof report.header.heapUsed === "number") { + out.heapUsedBytes = report.header.heapUsed; + } + } + const usage = process.resourceUsage(); + // maxRSS is reported in KiB on POSIX; convert explicitly. + out.peakRssBytes = usage.maxRSS * 1024; + + index.close(); + const fileStat = await stat(dbPath); + out.databaseBytes = fileStat.size; + } finally { + try { + index.close(); + } catch {} + try { + await rm(dir, { recursive: true, force: true }); + } catch {} + } + + process.stdout.write(`${JSON.stringify(out)}\n`); + if (!out.correctness) process.exitCode = 1; +} + +const invokedDirectly = + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href; + +if (invokedDirectly) { + runCli(process.argv.slice(2)).catch((err) => { + process.stderr.write(`${err && err.stack ? err.stack : String(err)}\n`); + process.exitCode = 1; + }); +} + +export { CORPUS_LIMITS }; diff --git a/benchmarks/scale/sqlite.test.mjs b/benchmarks/scale/sqlite.test.mjs new file mode 100644 index 0000000..ddeae7f --- /dev/null +++ b/benchmarks/scale/sqlite.test.mjs @@ -0,0 +1,208 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { ScaleIndex } from "./sqlite.mjs"; +import { + recordsFor, + edgesFor, + recordId, + needleTerm, + expectedEdgeTargets, +} from "./corpus.mjs"; + +function makeIndex() { + const idx = new ScaleIndex(":memory:"); + return idx; +} + +test("corpus: deterministic, exactly 5 valid edges per record, no self edges", () => { + const count = 10; + const records = [...recordsFor(count)]; + assert.equal(records.length, count); + assert.equal(records[0].id, recordId(0)); + assert.equal(records[9].id, recordId(9)); + assert.equal(records[0].source, "src/file0000.ts"); + assert.ok(records[0].text.includes(needleTerm(0))); + + const edges = [...edgesFor(count)]; + assert.equal(edges.length, count * 5); + const ids = new Set(records.map((r) => r.id)); + const seen = new Set(); + for (const e of edges) { + assert.ok(ids.has(e.source), `unknown source ${e.source}`); + assert.ok(ids.has(e.target), `unknown target ${e.target}`); + assert.notEqual(e.source, e.target, "no self edges"); + assert.equal(e.kind, "calls"); + const key = `${e.source}->${e.target}`; + assert.ok(!seen.has(key), `duplicate edge ${key}`); + seen.add(key); + } + + // Wrap-around behaviour for the first record. + const targets = edges.filter((e) => e.source === recordId(0)).map((e) => e.target).sort(); + assert.deepEqual(targets, expectedEdgeTargets(count, 0).sort()); +}); + +test("generators are deterministic across repeated invocations", () => { + const a = [...recordsFor(13, { revision: 1 })]; + const b = [...recordsFor(13, { revision: 1 })]; + assert.deepEqual(a, b); + assert.ok(a[0].text.includes("refreshed")); + assert.ok(!a[1].text.includes("refreshed")); + const ea = [...edgesFor(13)]; + const eb = [...edgesFor(13)]; + assert.deepEqual(ea, eb); +}); + +test("exact lookup, search, and adjacency under scope+generation isolation", () => { + const idx = makeIndex(); + try { + const count = 64; + idx.build("s1", "g1", recordsFor(count), edgesFor(count)); + idx.build("s2", "g1", recordsFor(count), edgesFor(count)); + + const id = recordId(7); + const a = idx.lookup("s1", "g1", id); + const b = idx.lookup("s2", "g1", id); + assert.ok(a && b); + assert.equal(a.id, id); + assert.equal(b.id, id); + + // Search must only return this scope/generation. + const s1 = idx.search("s1", "g1", needleTerm(7), 10); + assert.equal(s1.count, 1); + assert.equal(s1.results[0].id, id); + assert.equal(s1.internalWorkMeasured, false); + + const s2 = idx.search("s2", "g1", needleTerm(7), 10); + assert.equal(s2.count, 1); + assert.equal(s2.results[0].id, id); + + // Neighbours: five deterministic targets. + const nb = idx.neighbours("s1", "g1", id, 5); + assert.equal(nb.count, 5); + assert.deepEqual(nb.results, expectedEdgeTargets(count, 7).sort()); + assert.equal(nb.internalWorkMeasured, false); + + // Missing id: undefined, not an error. + assert.equal(idx.lookup("s1", "g1", recordId(999999 - (999999 % 1)) && "r9999999"), undefined); + } finally { + idx.close(); + } +}); + +test("update generation preserves old pinned reads and switches current", () => { + const idx = makeIndex(); + try { + const count = 32; + idx.build("sc", "g1", recordsFor(count, { revision: 0 }), edgesFor(count)); + const before = idx.lookup("sc", "g1", recordId(0)); + assert.ok(before); + assert.ok(!before.text.includes("refreshed")); + + idx.build("sc", "g2", recordsFor(count, { revision: 1 }), edgesFor(count)); + assert.equal(idx.activeGeneration("sc"), "g2"); + + const pinned = idx.lookup("sc", "g1", recordId(0)); + assert.ok(pinned); + assert.equal(pinned.text, before.text, "old pinned generation must remain readable unchanged"); + + const current = idx.lookup("sc", "g2", recordId(0)); + assert.ok(current); + assert.ok(current.text.includes("refreshed")); + } finally { + idx.close(); + } +}); + +test("failed generator mid-build rolls back new generation and retains prior head", () => { + const idx = makeIndex(); + try { + const count = 24; + idx.build("rb", "g1", recordsFor(count), edgesFor(count)); + assert.equal(idx.activeGeneration("rb"), "g1"); + + function* brokenRecords() { + let i = 0; + for (const r of recordsFor(count)) { + if (i === 5) throw new Error("injected record failure"); + i++; + yield r; + } + } + + assert.throws(() => idx.build("rb", "g2", brokenRecords(), edgesFor(count)), /injected record failure/); + assert.equal(idx.activeGeneration("rb"), "g1"); + assert.equal(idx.lookup("rb", "g2", recordId(0)), undefined); + assert.ok(idx.lookup("rb", "g1", recordId(0))); + } finally { + idx.close(); + } +}); + +test("dangling edge rolls back new generation and retains prior head", () => { + const idx = makeIndex(); + try { + const count = 24; + idx.build("dg", "g1", recordsFor(count), edgesFor(count)); + assert.equal(idx.activeGeneration("dg"), "g1"); + + function* badEdges() { + yield { source: recordId(0), target: recordId(1), kind: "calls" }; + yield { source: recordId(0), target: "r9999999", kind: "calls" }; + } + + assert.throws(() => idx.build("dg", "g2", recordsFor(count), badEdges()), /dangling edge/); + assert.equal(idx.activeGeneration("dg"), "g1"); + assert.equal(idx.lookup("dg", "g2", recordId(0)), undefined); + } finally { + idx.close(); + } +}); + +test("hostile FTS query is treated as literal, no SQL injection or cross-scope leakage", () => { + const idx = makeIndex(); + try { + const count = 16; + idx.build("hx", "g1", recordsFor(count), edgesFor(count)); + idx.build("hy", "g1", recordsFor(count), edgesFor(count)); + + const hostile = ['needle0000004" OR "1"=="1', "needle0000004' OR '1'='1", "*", '"']; + for (const term of hostile) { + const r = idx.search("hx", "g1", term, 10); + assert.equal(r.count, 0, `hostile term matched unexpectedly: ${term}`); + } + + // A literal quote inside a real match still works as literal phrase. + const quoted = idx.search("hx", "g1", needleTerm(4), 10); + assert.equal(quoted.count, 1); + assert.equal(quoted.results[0].id, recordId(4)); + + // No cross-scope leakage: search for hx needle under hy returns nothing. + const cross = idx.search("hy", "g1", "definitely-not-in-corpus", 10); + assert.equal(cross.count, 0); + } finally { + idx.close(); + } +}); + +test("invalid limits and duplicate generation rejection", () => { + const idx = makeIndex(); + try { + const count = 16; + idx.build("li", "g1", recordsFor(count), edgesFor(count)); + assert.throws(() => idx.search("li", "g1", "shared", 0), RangeError); + assert.throws(() => idx.search("li", "g1", "shared", 101), RangeError); + assert.throws(() => idx.neighbours("li", "g1", recordId(0), -1), RangeError); + assert.throws(() => idx.lookup("li", "g1", "not-an-id"), TypeError); + + assert.throws( + () => idx.build("li", "g1", recordsFor(count), edgesFor(count)), + /generation already present/ + ); + + assert.throws(() => idx.build("", "gx", recordsFor(count), edgesFor(count)), TypeError); + assert.throws(() => idx.build("li", "", recordsFor(count), edgesFor(count)), TypeError); + } finally { + idx.close(); + } +}); diff --git a/docs/ADR-001-scalable-collections.md b/docs/ADR-001-scalable-collections.md new file mode 100644 index 0000000..af734b0 --- /dev/null +++ b/docs/ADR-001-scalable-collections.md @@ -0,0 +1,112 @@ +# ADR-001: Scalable signed collections + +Status: provisional; implementation sequence superseded by the [enterprise scale review](ENTERPRISE-SCALE-REVIEW.md). + +The review identifies missing query indexes, work budgets, durable publication and rollback semantics. Do not implement the cryptographic format below until those decisions are resolved. The existing memory stores remain experimental internal foundations. + +Updated: 21 September 2026. + +## Context + +The v1 collection snapshot contains every signed record event. Appending copies and re-signs the complete snapshot, validation materialises it, and a collection is capped at 128 records. The current repository benchmark retained 128 records and dropped 111. Raising the constant would preserve the full-read and full-rewrite behaviour and would not provide credible whole-codebase support. + +The target is at least 10,000 records and 50,000 edges without requiring a query to decrypt, validate or materialise the complete graph. Retrieval remains capped at 20 records and 32 KiB, and graph views at 40 nodes and 32 KiB. + +## Invariants + +- Existing v1 policies, snapshots, records, access events and signed domains remain valid and unambiguous. +- A collection remains one authorisation domain. A relation cannot silently cross a collection or project boundary. +- Record contents and relationships remain tamper-evident and encrypted at rest. +- Exact lookup and bounded traversal load only the pages required by the bounded result. +- Appends retain expected-head concurrency. A losing writer cannot become authoritative by sequence or timing. +- Core remains local-first and browser-safe. Normal reads use an explicitly supplied local blob store and never follow a URL found in graph content. +- Revocation prevents future authorised access. It cannot retract keys or plaintext already obtained by a recipient. + +## Options + +| Option | Result | +| --- | --- | +| Increase v1 limits | Rejected. Snapshots, validation, key rotation and appends still scale with the whole collection. Event and envelope limits become the next ceiling. | +| Split one project across ordinary v1 collections | Rejected as the target design. It weakens atomic relationships, complicates grants and key rotation, and would require implicit cross-collection traversal. | +| One v2 root containing every record/page and edge entry | Rejected. The index and key schedule become another unbounded snapshot. | +| Append-only event log with sparse indexes | Deferred. It is useful for multi-writer history but adds log compaction and synchronisation complexity not required by the first target. | +| Bounded v2 Merkle/B-tree indexes and immutable pages | Selected. The root and every page remain bounded while lookup grows logarithmically. | + +## Decision + +Add an explicit v2 collection format. A small signed root commits to bounded immutable encrypted pages through ciphertext SHA-256 digests. Separate paged trees locate records, outgoing edges and any historical epoch-key material. No root, key schedule, index node, edge bucket or record page grows with the total graph. + +The signed root contains only bounded metadata: the collection and policy epoch identifiers, revision, previous committed root digest, record-index root digest, edge-index root digest, key-schedule root digest and authenticated counts. Its canonical encoded content must not exceed 8 KiB before the outer signature. + +Each logical page has an opaque random page ID and a fixed page class. A minimal clear header carries the format version, collection ID, epoch, page class and page ID. The page key is derived from the epoch key, page class and page ID. AEAD encryption authenticates the complete header as associated data. + +The stored address is SHA-256 over the complete sealed bytes, including the authenticated header and ciphertext. A reader verifies that digest before decryption, then verifies the authenticated page identity, type, range and internal references. Plaintext hashes are not storage addresses and equal plaintext under different opaque page IDs must not expose an equality address. + +Index nodes contain bounded key ranges and child ciphertext digests. An exact lookup decrypts one node per tree level followed by the selected record page. One-hop traversal uses an edge tree keyed by source, relation kind and target, and stops at the existing record, node and byte budgets. Page-read limits are derived and asserted from the actual built tree height rather than assumed constants. + +Core receives pages through a narrow content-addressed `BlobStore`; graph content cannot choose an arbitrary URL. Persistent filesystem, IndexedDB and remote synchronisation adapters are later concerns with their own authority boundaries. + +## Writes, rotation and recovery + +Writers first store immutable sealed pages. They then create and sign a root whose previous digest is the currently authorised head. Publication uses a narrow `RootStore.compareAndSwap(expectedHeadDigest, newRootDigest)` operation. A losing writer receives a conflict, reads the committed head, revalidates and deliberately retries. Sequence numbers do not resolve siblings. + +A crash before root publication leaves the previous root authoritative and may leave orphan pages. A crash after successful compare-and-swap leaves the new committed root authoritative. Recovery starts from the last committed valid root supplied by `RootStore`, never the highest uncommitted candidate. Garbage collection removes unreferenced pages only after an explicit grace period or complete reference scan from every retained committed root. + +Policy rotation creates a new epoch key for future pages. Old pages require retained old epoch keys unless a deliberate full rewrite and re-encryption occurs. Historical key material is itself stored in bounded encrypted pages committed by the root, not an ever-growing inline list. A recipient holding an old key may retain old ciphertext or plaintext but cannot derive a new epoch key. + +Deletion and rename will be represented by a new committed root with deterministic tombstones and replacement index entries. Their repository-refresh semantics are not part of the first implementation slice. + +## Compatibility + +V1 objects keep their existing parsers, validation and limits. V2 uses distinct versioned root and page objects and does not reinterpret a v1 signature. The first implementation slice adds no automatic migration and does not change `ContextVault`, persistence or public v1 behaviour. + +A later migration must read and fully validate a v1 snapshot, emit v2 pages and publish a separate v2 root without modifying the original. Existing compatibility fixtures remain regression gates. + +## First implementation slice + +Build and test the storage/index primitives before connecting them to `ContextVault`: + +- bounded canonical codecs for a root, page header, index node, record page and edge bucket; +- seal/open helpers using the existing qualified cryptographic primitives where they fit; +- bounded B-tree construction, exact record lookup and one-hop neighbour lookup; +- a `BlobStore` interface and in-memory content-addressed implementation; +- a `RootStore` interface and test-only in-memory compare-and-swap implementation; and +- a deterministic 10,000-record, 50,000-edge fixture with page-read instrumentation. + +Tentative interfaces, subject to implementation review: + +```ts +interface BlobStore { + put(bytes: Uint8Array): Promise + get(digest: string): Promise +} + +interface RootStore { + readHead(): Promise + compareAndSwap(expectedHead: string | null, nextHead: string): Promise +} +``` + +The slice should live under `packages/context/src/paging/` with focused codec, crypto, index and store tests. No dependency is added merely for convenience. + +## Acceptance + +- The encoded root is no more than 8 KiB; all page types and fan-out have enforced bounds. +- The 10,000-record and 50,000-edge fixture builds deterministically. +- Exact lookup and bounded neighbour traversal remain within page-read ceilings calculated from the built tree height and requested result bound. +- Neither lookup nor traversal decrypts or validates the complete collection. +- Ciphertext tampering, wrong associated data, wrong page identity and digest mismatch are rejected. +- Equal plaintext placed under distinct page IDs does not produce a plaintext equality address. +- Concurrent writers prove expected-head conflict behaviour; crash-before and crash-after-CAS fixtures preserve the last committed root. +- Existing v1 build, unit, package-smoke and benchmark gates remain green. + +Stop if a root or individual page grows with total graph size, a lookup follows content-provided URLs, an automatic sibling winner replaces compare-and-swap, or scale is achieved by weakening isolation or response bounds. + +## Non-goals for the first slice + +- v1 migration; +- `ContextVault` or file-store integration; +- hosted storage or synchronisation; +- repository watch or incremental refresh; +- garbage-collection implementation; and +- lifting any public whole-codebase scale claim. diff --git a/docs/BOUNDED-SEARCH-DESIGN.md b/docs/BOUNDED-SEARCH-DESIGN.md new file mode 100644 index 0000000..c6e6626 --- /dev/null +++ b/docs/BOUNDED-SEARCH-DESIGN.md @@ -0,0 +1,27 @@ +# Bounded search experiment + +21 September 2026. Design for an isolated public-fixture experiment, not an API commitment. + +## Decision + +Evaluate a composite posting index keyed by `(scope, generation, term, recordId)` alongside the existing FTS baseline. Exact-token pages use an indexed keyset range, not an FTS match followed by sorting. Keep payload fetching, graph expansion and relevance ranking out of this first experiment so their costs cannot hide inside a posting-row counter. + +Each request names a scope, immutable generation, one normalised token, output limit and maximum posting rows. Fetch at most the smaller of the two limits. Do not fetch an uncounted lookahead row. A full page conservatively reports incomplete, even when it happens to be the final page; a subsequent empty page proves exhaustion. Continuation binds scope, generation, token and last record ID. + +This bounds posting rows materialised by the application. It does **not** bound SQLite page reads, CPU, lock waits or wall time. Inspect the query plan for an indexed range without a temporary sort, and measure common-term performance as the corpus grows. Keep `internalWorkMeasured: false` until internal work is genuinely measured. Do not call this a hard execution budget. + +## Boundaries + +- IDs only: fetching text later requires its own byte and record budgets. +- Single ASCII token only: no equivalence claim with phrase search, Unicode tokenisation, multi-term retrieval or ranking. +- Deterministic ID order is not relevance order or exact global top-k. +- A plain cursor is experiment state, not authorisation or a secure service token. A future service must validate current access on every page. +- Scope keys prevent accidental result mixing in this fixture, not timing leakage or tenant resource interference. +- Full transactional generations first. Incremental refresh, reader leases and garbage collection remain separate work. +- Public generated data only. Even token-only indexes leak private repository information. + +## Acceptance before integration + +Prove capped rows, complete pagination, scope/generation/query cursor binding, immutable old-generation reads, empty-generation identity, rollback and persistent reopening. Record query plans and reproducible common/unique-token measurements. Preserve existing package and benchmark gates. + +Next design gate: combine multi-term candidate selection with explicit ranking/incompleteness semantics, bounded payload retrieval and cancellable execution. Do not silently promote this simplified experiment to production search. diff --git a/docs/ENTERPRISE-SCALE-REVIEW.md b/docs/ENTERPRISE-SCALE-REVIEW.md new file mode 100644 index 0000000..6f640ab --- /dev/null +++ b/docs/ENTERPRISE-SCALE-REVIEW.md @@ -0,0 +1,103 @@ +# Enterprise scale: architecture review + +Date: 21 September 2026. Status: proposed direction; implementation gates remain open. + +## Decision + +Separate the repository index, bounded evidence retrieval and signed portable storage. Prove repository scale with a local indexed engine before designing a new encrypted database format. Preserve the browser-safe protocol and verification core; provide persistent indexing through the Node tools adapter. + +ADR-001 contains useful paging and compare-and-swap ideas, but it is not yet a sufficient design for enterprise queries. Its cryptographic format must remain provisional. Ten thousand records is an initial fixture, not an enterprise capacity claim. + +## What the current design misses + +1. **Search before lookup.** A record-ID tree answers where a known record lives. It does not find records relevant to an agent's question. Current retrieval scores every visible record. Paging those records alone would still require a full scan. +2. **Work budgets.** A 32 KiB answer can require millions of candidate reads. Candidate evaluation, posting-list reads, edge expansion, decrypted bytes and elapsed work need limits independent of output size. +3. **Publication and rollback.** A root signature authenticates a root, not that it is current. A parent pointer alone does not replace v1's ancestry checks when a reader skips several revisions. CAS in memory supplies neither disk durability nor rollback protection. +4. **Author and index trust.** A signed root commits to its contents but does not establish that its index is complete or correctly derived. Record author signatures, grant validity at creation and query authorisation need explicit rules. Merkle membership proves inclusion, not search relevance or completeness. +5. **Refresh.** Repository updates must invalidate derived symbols, outgoing edges, reverse references and search entries together. Appending another scan eventually fills any finite store and leaves stale results. +6. **Operational boundaries.** Access domains, concurrent readers, crash recovery, deletion, quotas and retained snapshots determine enterprise behaviour as much as graph size does. + +## Three layers + +| Layer | Responsibility | First implementation | +| --- | --- | --- | +| Repository index | Files, symbols, relations, inverted search index, source revisions and active generation | Local Node adapter using a mature transactional embedded database; evaluate SQLite as the initial candidate | +| Evidence retrieval | Authorised scope, ranking, bounded expansion, source excerpts and explicit incompleteness | Storage-independent asynchronous query API with a budget shared across all work | +| Signed portable storage | Authenticity, encrypted export/import, provenance and snapshot commitments | Keep v1 compatibility; specify v2 only after the query workload and update patterns are measured | + +SQLite is a candidate implementation choice, not a dependency selected by this document. Qualify the maintained driver against the pinned Node runtime, packaging, concurrent access and encryption requirements before adopting it. The adapter must not enter the browser-safe package import graph. + +The repository index is disposable derived state. A local source revision or verified imported record is its input. Its rows must distinguish local extraction from cryptographically verified imported evidence. Unsigned index rows must never be returned as signed records merely because they came from a trusted process. + +An index containing source names, tokens or edges is sensitive even without full source text. V1 currently promises no persistent plaintext index. A persistent index therefore requires an explicit new storage mode with a specified encryption/key-custody implementation. Prototype with public fixtures until that decision passes review; never silently create a plaintext private-code index. + +## Query contract + +Every query specifies authorised collection/project scope, source generation and resource budgets. The engine resolves the active generation once and pins it for the operation. An explicit bounded set of repositories may form an ecosystem; repository membership alone does not widen query access. + +1. Check authorisation before reading project metadata or searching. +2. Select candidates through an inverted lexical index. Use bounded posting-list work and deterministic ranking/tie-breaking. Existing lexical relevance is an adequate first baseline; semantic models are optional later experiments. +3. Expand outgoing and incoming adjacency indexes within a shared edge/node/page budget. High-degree nodes cannot consume unlimited work. +4. Fetch source spans or verified records only for selected evidence. Count those bytes and any verification work against the same query budget. +5. Return the pinned generation, source revision, provenance, consumed budget and explicit truncation reason. Use an opaque generation-bound cursor where continuation is supported. + +Proposed API shape, not an existing export: + +```ts +queryEvidence({ scope, generation, query, budget, signal }): Promise +``` + +The budget covers candidate count, postings visited, edges visited, page reads, decoded bytes, output bytes and deadline. Cancellation must propagate to the storage adapter. Budgets are enforced during work, not checked only after a complete scan. + +Do not promise exact global top-k under arbitrary strict work limits. Report whether the search exhausted its candidate domain or stopped early. Pagination must not silently switch generations. A bounded path search can report `budget-exhausted`; it must not mislabel that result as proof that no path exists. + +## Refresh and storage + +Use one active generation per repository snapshot. A manifest records repository identity, source revision, extractor version, configuration and per-file hashes. Parse changed files into staging; replace their derived rows and outgoing edges, then reconcile affected incoming references using reverse indexes. Configuration or extractor changes can trigger a documented full rebuild. + +Deletion removes a file's active derived records. Rename is initially a delete plus add, with optional explicit lineage. Preserve stable IDs for unchanged source locations; do not promise that every refactor preserves symbol identity. Historical signed assertions remain separate from the active source view. + +Publish the new generation atomically after validation. Readers already pinned to the old generation finish against it. Failed builds leave the last committed generation available with visible freshness metadata. Start with one writer per repository and concurrent snapshot readers; independent repositories can build concurrently within quotas. + +Durability requires writing and syncing referenced data before publishing the head, and persisting the head through a transaction or correctly synced rename protocol. Test process termination around each publication boundary. MemoryRootStore is only a fixture and proves none of these disk properties. + +Garbage collection requires both a reachability check and protection for active readers/builds. Elapsed grace time alone is insufficient. Pin retained roots explicitly and document backup retention separately from active-data deletion. + +## Signed storage and access + +Keep v1 readable and its existing signed domains unchanged. Do not enlarge v1 constants or reuse v1 signatures with new meanings. A later v2 can use bounded encrypted index/record pages and a signed commitment root, with individually attributable evidence retained where required. + +Before implementing v2 crypto, resolve: + +- the exact canonical encoding, signature domain and audited encryption construction; +- how authorship and historical grant validation survive paging; +- how clients establish descent from their last trusted head without an unbounded inline ancestor array; +- bounded epoch-key history and whether newcomers receive historical access; +- fresh nonce/key derivation for every immutable page version; +- metadata leakage from clear page headers and size/access patterns; and +- what a reader can verify about index derivation versus merely trusting the authorised index builder. + +An opaque page ID must be fresh for every sealed version or accompanied by a proven nonce strategy. Reusing a logical page ID on updates must not accidentally reuse an AEAD key/nonce pair. Ciphertext hashes can authenticate stored bytes but do not resolve this issue. + +Offline readers can verify their cached policy and expiry; they cannot discover a newer revocation without synchronisation. Results must expose that freshness boundary. Hosted queries must check current service authorisation and bind cursors/caches to it. Previously disclosed keys and plaintext remain outside retrospective revocation. + +For shared deployments, partition storage, indexes and caches by access domain. Authorise before candidate selection, including counts and snippets. Cross-domain querying requires explicit grants for every participant and a global budget. Do not add one organisation-wide index and rely solely on filtering final results. + +## Ordered implementation packets + +1. **Scale contract and harness.** Add deterministic public corpora and query expectations at 10k/50k, 100k/500k and 1m/5m records/edges. Include skewed hubs, duplicate identifiers and source edits/deletes. These are proposed tiers, not capacities already achieved. +2. **Local index proof.** Qualify an embedded database adapter against those workloads. Implement generation transactions, lexical candidate lookup, adjacency lookup and measured query budgets. Use public fixtures while encryption is unresolved. +3. **Refresh and agent workflow.** Implement staged source refresh, invalidation and cancellation, then exercise a real MCP client. Keep the existing benchmark unchanged and repair its current recall regression as a separate bounded scanner task. +4. **Private-data storage gate.** Select and test encryption, key handling, recovery and explicit opt-in before indexing private repositories persistently. +5. **Portable signed v2.** Complete a reviewed format specification and compatibility fixtures, then implement only the pieces justified by the measured index workload. +6. **Shared operation.** Add managed repository connections, scoped service access, scheduling, audit and service quotas after the local engine proves useful on the pilot team's tasks. + +The scalable local engine, local refresh and portable verification should remain open source. Managed operation, team administration and support provide the private commercial layer. + +## Acceptance evidence + +For each scale tier, record hardware, corpus/revision, engine version, cold and warm query latency, p50/p95, peak memory, disk footprint, initial-build time, refresh work and query resource counters. Set numerical latency/memory targets before comparing implementations. Do not invent them from an unmeasured design. + +Require declared-source recall and accepted-task quality alongside cost. Prove that single-file refresh avoids a whole-repository rebuild where dependency semantics permit it. Include cancellation, a high-degree hub, concurrent query/build, crash recovery, denied access and stale snapshots. Query work should follow its enforced budgets as the corpus grows. + +At the initial review, the memory stores were foundations only and the expanded source-navigation corpus failed recall. Subsequent worker changes restored both benchmark gates. The synthetic SQLite baseline and a separate bounded posting-page experiment now exist; see [scale evidence](../benchmarks/scale/README.md) and the [bounded search design](BOUNDED-SEARCH-DESIGN.md). These do not close the enterprise acceptance gate or select a production adapter. The next useful work remains query qualification and refresh, not another cryptographic page codec. diff --git a/docs/OVERNIGHT-HANDOFF.md b/docs/OVERNIGHT-HANDOFF.md new file mode 100644 index 0000000..264736a --- /dev/null +++ b/docs/OVERNIGHT-HANDOFF.md @@ -0,0 +1,31 @@ +# Overnight scale work: handoff + +21 September 2026. Historical overnight checkpoint, recorded before the subsequent source-shipment PR. At this checkpoint the changes were local only. Package publication and deployment are separate from a source merge. + +## Completed + +- Saved the bounded posting-page design in `BOUNDED-SEARCH-DESIGN.md`. +- Integrated worker-written `benchmarks/scale/postings.mjs`, its 14 tests and a reproducible measurement runner. No package/public API changes. +- Qualified generated corpora at 10k, 100k and 1m records, with zero edges in this separate experiment. See the scale README for exact measurements and limitations. +- Fresh baseline: `npm run check` passed 59 tests and independent package smoke; both token benchmark gates passed. `node --test benchmarks/scale/postings.test.mjs benchmarks/scale/sqlite.test.mjs` passed 22 tests. `git diff --check` passed before this documentation update. +- Corrected the architecture review's historical recall-failure status. + +## What this proves + +An exact-token composite index can serve stable bounded ID pages without the full-match sorting behaviour of the original common-term FTS probe. Application-materialised posting rows are capped. SQLite internal work and hard deadlines are not measured/enforced; ranked multi-term evidence retrieval is not implemented. + +## Worker routing and cost evidence + +Used `ollama-workers`: Qwen first, then Flash after two inadequate drafts per packet. Qwen drafts were rejected; accepted implementation came from DeepSeek V4.1 Flash, with reviewed mechanical integration. No Pro or frontier coding fallback. All calls used thinking=false. No 402 occurred. + +Helper receipts: `/private/tmp/z1p-overnight-workers`. Reported token totals: Qwen 18,612; Flash 17,087; combined 35,699. Three busy dispatch receipts have unknown counts; they were blocked by the endpoint guard. Four Qwen inference drafts were rejected, one Flash response accepted, two Flash responses partially retained. These counts are not billing or cash-savings evidence. The shared endpoint was frequently occupied by another live session; its requests were not interrupted or replayed. + +## Next bounded packets + +1. Strengthen posting-prototype lifecycle and input tests: constructor failure, malformed cursor shapes, invalid source records, iterator cleanup on consumer-side validation failure, exact expected pagination IDs and cross-generation content differences. Repair with workers, keeping the public API untouched. +2. Resolve multi-term search semantics before implementation: deterministic candidate order, ranking over examined candidates only, global query budget across continuations, explicit exhaustion/truncation reasons and independently budgeted payload bytes. Do not promise exact global top-k under arbitrary budgets. +3. Prototype cancellable execution and test slow/blocked queries. SQLite keyset LIMIT is not a deadline or CPU cap. +4. Add skewed graph hubs and staged edits/deletes, then incremental refresh and interrupted-publication tests. Keep immutable generation pinning. +5. Complete real-repository correctness and paired accepted-task cost evaluation before private operational features or public savings claims. + +No private repositories may be indexed persistently by these plaintext prototypes. Encryption/key custody, access control, service cursor authentication, crash recovery and schema lifecycle remain open gates. If cloud returns 402, stop that lane and leave a handoff; do not reroute its coding into Codex. diff --git a/packages/context-tools/src/source-scan.test.ts b/packages/context-tools/src/source-scan.test.ts index e590d1f..c6ec9d8 100644 --- a/packages/context-tools/src/source-scan.test.ts +++ b/packages/context-tools/src/source-scan.test.ts @@ -150,4 +150,81 @@ describe('bounded TypeScript and JavaScript source graph scan', () => { 'repo://a.ts', 'repo://b.ts', 'repo://a.ts#publicA', 'repo://b.ts#publicB', ]) }) + + it('early file with many exports does not starve later file anchors under maxRecords 4', async () => { + const root = await fixture() + await source(root, 'early.ts', `export function e1() {}\nexport function e2() {}\nexport function e3() {}\nexport function e4() {}`) + await source(root, 'later.ts', `export function onlyExport() {}`) + const result = await scanSourceGraph(root, { observedAt: 123, maxRecords: 4 }) + expect(result.records.length).toBe(4) + const sources = result.records.map(record => record.source) + expect(sources).toContain('repo://early.ts') + expect(sources).toContain('repo://later.ts') + expect(sources).toContain('repo://early.ts#e1') + expect(sources).toContain('repo://later.ts#onlyExport') + const laterExport = bySource(result.records, 'repo://later.ts#onlyExport')! + expect(laterExport).toBeDefined() + }) + + it('file summary includes exported names with exact omitted count under name and text bounds', async () => { + const root = await fixture() + const longName = `long${'a'.repeat(650)}` + await source(root, 'mixed.ts', [ + `export function shortOne() {}`, + `export function shortTwo() {}`, + `export function ${longName}() {}`, + `export function shortThree() {}`, + ].join('\n')) + const first = await scanSourceGraph(root, { observedAt: 123 }) + const second = await scanSourceGraph(root, { observedAt: 123 }) + expect(first).toEqual(second) + const fileRecord = bySource(first.records, 'repo://mixed.ts')! + expect(fileRecord.text).toContain('exported: shortOne, shortTwo, shortThree') + expect(fileRecord.text).toContain('(+1 omitted)') + expect(fileRecord.text).not.toContain(longName) + expect(fileRecord.text).toContain('shortThree') + const exportedSegment = fileRecord.text.slice(fileRecord.text.indexOf('exported:')) + const namesSegment = exportedSegment.slice('exported: '.length, exportedSegment.indexOf('.')) + expect(namesSegment.length).toBeLessThan(600) + expect(fileRecord.text.length).toBeLessThan(4000) + const identity = createNostrIdentity(new Uint8Array(32).fill(29)) + const vault = new ContextVault({ identity, now: () => 123 }) + const view = await vault.create({ title: 'Long name source', scope: 'personal' }) + await expect(vault.appendBatch(view.id, view.head, first.records)).resolves.toBeDefined() + }) + + it('reports omitted exports when no anchor name fits within the budget', async () => { + const root = await fixture() + const firstName = `first${'a'.repeat(650)}` + const secondName = `second${'b'.repeat(650)}` + await source(root, 'alllong.ts', [ + `export function ${firstName}() {}`, + `export function ${secondName}() {}`, + ].join('\n')) + const result = await scanSourceGraph(root, { observedAt: 123 }) + const fileRecord = bySource(result.records, 'repo://alllong.ts')! + expect(fileRecord.text).toContain('(+2 omitted)') + expect(fileRecord.text).not.toContain(firstName) + expect(fileRecord.text).not.toContain(secondName) + const identity = createNostrIdentity(new Uint8Array(32).fill(28)) + const vault = new ContextVault({ identity, now: () => 123 }) + const view = await vault.create({ title: 'All long names', scope: 'personal' }) + await expect(vault.appendBatch(view.id, view.head, result.records)).resolves.toBeDefined() + }) + + it('file anchors include every exported name and exclude local ones', async () => { + const root = await fixture() + await source(root, 'anchors.ts', [ + `export function exportedOne() {}`, + `function localOne() {}`, + `export function exportedTwo() {}`, + `function localTwo() {}`, + ].join('\n')) + const result = await scanSourceGraph(root, { observedAt: 123 }) + const fileRecord = bySource(result.records, 'repo://anchors.ts')! + expect(fileRecord.text).toContain('exportedOne') + expect(fileRecord.text).toContain('exportedTwo') + expect(fileRecord.text).not.toContain('localOne') + expect(fileRecord.text).not.toContain('localTwo') + }) }) diff --git a/packages/context-tools/src/source-scan.ts b/packages/context-tools/src/source-scan.ts index 5c62a8e..7e2e35b 100644 --- a/packages/context-tools/src/source-scan.ts +++ b/packages/context-tools/src/source-scan.ts @@ -173,12 +173,44 @@ export async function scanSourceGraph(root: string, options: SourceGraphScanOpti // Preserve repository coverage before declaration detail. Exported symbols // are the most useful navigation entry points when the collection's bounded // record capacity cannot retain every local declaration. - const files = parsed.map(file => ({ key: file.path, type: 'file' as const, file })) - const symbols = parsed.flatMap(file => file.symbols.map(symbol => ({ key: symbol.key, type: 'symbol' as const, file, symbol }))) - const candidates = [...files, ...symbols.filter(candidate => candidate.symbol.exported), - ...symbols.filter(candidate => !candidate.symbol.exported)] - const retained = candidates.slice(0, maxRecords) + type FileCandidate = { key: string; type: 'file'; file: ParsedFile } + type SymbolCandidate = { key: string; type: 'symbol'; file: ParsedFile; symbol: ParsedSymbol } + type Candidate = FileCandidate | SymbolCandidate + const files: FileCandidate[] = parsed.map(file => ({ key: file.path, type: 'file', file })) + const symbols: SymbolCandidate[] = parsed.flatMap(file => file.symbols.map(symbol => ({ key: symbol.key, type: 'symbol', file, symbol }))) + const exportedSymbols = symbols.filter(candidate => candidate.symbol.exported) + const localSymbols = symbols.filter(candidate => !candidate.symbol.exported) + const exportedByFile = new Map(parsed.map(file => [file.path, []])) + const localByFile = new Map(parsed.map(file => [file.path, []])) + for (const candidate of exportedSymbols) exportedByFile.get(candidate.file.path)!.push(candidate) + for (const candidate of localSymbols) localByFile.get(candidate.file.path)!.push(candidate) + const retained: Candidate[] = files.slice(0, maxRecords) const retainedKeys = new Set(retained.map(candidate => candidate.key)) + // Round-robin across per-file buckets so an early file with many exports + // cannot starve later files. Exports are visited before locals and files + // are retained first; each bucket preserves source order and the record + // cap still terminates every loop. + const rounds = Math.max(1, ...parsed.map(file => Math.max(exportedByFile.get(file.path)!.length, localByFile.get(file.path)!.length))) + for (let round = 0; round < rounds && retained.length < maxRecords; round++) { + for (const file of parsed) { + if (retained.length >= maxRecords) break + const bucket = exportedByFile.get(file.path)! + if (round < bucket.length) { + const candidate = bucket[round] + if (!retainedKeys.has(candidate.key)) { retained.push(candidate); retainedKeys.add(candidate.key) } + } + } + } + for (let round = 0; round < rounds && retained.length < maxRecords; round++) { + for (const file of parsed) { + if (retained.length >= maxRecords) break + const bucket = localByFile.get(file.path)! + if (round < bucket.length) { + const candidate = bucket[round] + if (!retainedKeys.has(candidate.key)) { retained.push(candidate); retainedKeys.add(candidate.key) } + } + } + } const ids = new Map(retained.map(candidate => [candidate.key, id(candidate.type, candidate.key)])) const symbolsByFile = new Map(parsed.map(file => [file.path, new Map(file.symbols.map(symbol => [symbol.name, symbol]))])) let callsFound = 0 @@ -189,7 +221,21 @@ export async function scanSourceGraph(root: string, options: SourceGraphScanOpti const kinds = new Map() for (const symbol of candidate.file.symbols) kinds.set(symbol.kind, (kinds.get(symbol.kind) ?? 0) + 1) const summary = [...kinds].map(([kind, count]) => `${count} ${kind}${count === 1 ? '' : 's'}`).join(', ') || 'no named declarations' - return { id: ids.get(candidate.key)!, kind: 'evidence', text: `Source file ${candidate.file.path}: ${summary}; ${candidate.file.imports.length} resolved internal import${candidate.file.imports.length === 1 ? '' : 's'}.`, source: `repo://${candidate.file.path}`, observedAt, provenance, ...(relations.length ? { relations: relations.sort(relationSort).slice(0, 16) } : {}) } + const nameList: string[] = [] + let namesLength = 0 + let omitted = 0 + for (const symbol of candidate.file.symbols) { + if (!symbol.exported) continue + const separator = nameList.length ? ', ' : '' + if (namesLength + separator.length + symbol.name.length > 600) { omitted++; continue } + nameList.push(symbol.name) + namesLength += separator.length + symbol.name.length + } + const omittedText = omitted ? ` (+${omitted} omitted)` : '' + const names = nameList.length || omitted + ? `; exported: ${nameList.join(', ')}${omittedText}` + : '' + return { id: ids.get(candidate.key)!, kind: 'evidence', text: `Source file ${candidate.file.path}: ${summary}; ${candidate.file.imports.length} resolved internal import${candidate.file.imports.length === 1 ? '' : 's'}${names}.`, source: `repo://${candidate.file.path}`, observedAt, provenance, ...(relations.length ? { relations: relations.sort(relationSort).slice(0, 16) } : {}) } } const symbol = candidate.symbol if (retainedKeys.has(symbol.file)) relations.push(relation(ids.get(symbol.file)!, 'relates-to')) diff --git a/packages/context/src/paging/store.test.ts b/packages/context/src/paging/store.test.ts new file mode 100644 index 0000000..6696e06 --- /dev/null +++ b/packages/context/src/paging/store.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest' +import { sha256 } from '@noble/hashes/sha2' +import { bytesToHex } from '@noble/hashes/utils' +import { Buffer } from 'node:buffer' +import { MemoryBlobStore, MemoryRootStore } from './store.js' + +describe('MemoryBlobStore', () => { + it('stores immutable bytes at their SHA-256 digest and is idempotent', async () => { + const store = new MemoryBlobStore() + const input = new Uint8Array([1, 2, 3, 4]) + const expected = bytesToHex(sha256(input)) + const digest = await store.put(input) + expect(digest).toBe(expected) + expect(await store.put(new Uint8Array([1, 2, 3, 4]))).toBe(expected) + + input[0] = 99 + const firstRead = await store.get(digest) + expect(firstRead).toEqual(new Uint8Array([1, 2, 3, 4])) + firstRead![1] = 88 + expect(await store.get(digest)).toEqual(new Uint8Array([1, 2, 3, 4])) + }) + + it('returns undefined only for a valid missing digest', async () => { + const store = new MemoryBlobStore() + expect(await store.get('00'.repeat(32))).toBeUndefined() + await expect(store.get('abc')).rejects.toThrow('lowercase SHA-256') + await expect(store.get('AA'.repeat(32))).rejects.toThrow('lowercase SHA-256') + }) + + it('rejects empty and non-byte input', async () => { + const store = new MemoryBlobStore() + await expect(store.put(new Uint8Array())).rejects.toThrow('non-empty Uint8Array') + await expect(store.put('bytes' as unknown as Uint8Array)).rejects.toThrow('non-empty Uint8Array') + }) + + it('isolates the stored bytes from a Buffer that is mutated after put', async () => { + const store = new MemoryBlobStore() + const buf = Buffer.from([1, 2, 3, 4]) + const digest = await store.put(buf) + buf[0] = 99 + expect(await store.get(digest)).toEqual(new Uint8Array([1, 2, 3, 4])) + }) + + it('rejects a coercible object passed as a digest', async () => { + const store = new MemoryBlobStore() + await expect(store.get({ toString: () => '00'.repeat(32) } as unknown as string)) + .rejects.toThrow('lowercase SHA-256') + }) +}) + +describe('MemoryRootStore', () => { + it('commits the first root and rejects a stale expected head', async () => { + const store = new MemoryRootStore() + const first = '11'.repeat(32), second = '22'.repeat(32) + expect(await store.readHead()).toBeNull() + expect(await store.compareAndSwap(null, first)).toBe(true) + expect(await store.compareAndSwap(null, second)).toBe(false) + expect(await store.readHead()).toBe(first) + expect(await store.compareAndSwap(first, second)).toBe(true) + expect(await store.readHead()).toBe(second) + }) + + it('allows exactly one concurrent writer for the same expected head', async () => { + const store = new MemoryRootStore() + const left = '33'.repeat(32), right = '44'.repeat(32) + const results = await Promise.all([ + store.compareAndSwap(null, left), + store.compareAndSwap(null, right), + ]) + expect(results.filter(Boolean)).toHaveLength(1) + expect(await store.readHead()).toBe(results[0] ? left : right) + }) + + it('rejects malformed expected and next digests without changing the head', async () => { + const store = new MemoryRootStore() + const valid = '55'.repeat(32) + await expect(store.compareAndSwap(null, 'bad')).rejects.toThrow('lowercase SHA-256') + await expect(store.compareAndSwap('GG'.repeat(32), valid)).rejects.toThrow('lowercase SHA-256') + expect(await store.readHead()).toBeNull() + }) +}) diff --git a/packages/context/src/paging/store.ts b/packages/context/src/paging/store.ts new file mode 100644 index 0000000..a687741 --- /dev/null +++ b/packages/context/src/paging/store.ts @@ -0,0 +1,54 @@ +import { sha256 } from '@noble/hashes/sha2' +import { bytesToHex } from '@noble/hashes/utils' + +export interface BlobStore { + put(bytes: Uint8Array): Promise + get(digest: string): Promise +} + +export interface RootStore { + readHead(): Promise + compareAndSwap(expectedHead: string | null, nextHead: string): Promise +} + +const digestPattern = /^[0-9a-f]{64}$/ + +function checkDigest(value: string, name: string): void { + if (typeof value !== 'string') throw new Error(`${name} must be a lowercase SHA-256 digest.`) + if (!digestPattern.test(value)) throw new Error(`${name} must be a lowercase SHA-256 digest.`) +} + +export class MemoryBlobStore implements BlobStore { + readonly #blobs = new Map() + + async put(bytes: Uint8Array): Promise { + if (!(bytes instanceof Uint8Array) || bytes.length === 0) throw new Error('Blob bytes must be a non-empty Uint8Array.') + const copy = new Uint8Array(bytes) + const digest = bytesToHex(sha256(copy)) + if (!this.#blobs.has(digest)) this.#blobs.set(digest, copy) + return digest + } + + async get(digest: string): Promise { + checkDigest(digest, 'Blob digest') + return this.#blobs.get(digest)?.slice() + } +} + +/** Test foundation for the future persistent head store. Comparison and update + * are synchronous within one JavaScript turn; no sequence chooses a sibling. */ +export class MemoryRootStore implements RootStore { + #head: string | null = null + + async readHead(): Promise { + return this.#head + } + + async compareAndSwap(expectedHead: string | null, nextHead: string): Promise { + if (expectedHead !== null) checkDigest(expectedHead, 'Expected head') + checkDigest(nextHead, 'Next head') + if (expectedHead !== this.#head) return false + this.#head = nextHead + return true + } +}