From e704b3751f732afdeb05b4d41ad3cd84628d0512 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 07:21:59 +0700 Subject: [PATCH 1/9] docs(release): prepare 1.6.10-electric.11 --- .agents/plugins/marketplace.json | 2 +- .claude-plugin/marketplace.json | 2 +- ARCHITECTURE.md | 43 ++++++++++++----- Documentation/releases/1.6.10-electric.11.md | 46 +++++++++++++++++++ GUARDRAILS.md | 8 ++++ MIGRATION.md | 18 ++++++++ RUNBOOK.md | 45 +++++++++++++++--- .../.claude-plugin/plugin.json | 2 +- .../.codex-plugin/plugin.json | 2 +- gitnexus/CHANGELOG.md | 24 ++++++++++ gitnexus/package-lock.json | 4 +- gitnexus/package.json | 2 +- 12 files changed, 173 insertions(+), 25 deletions(-) create mode 100644 Documentation/releases/1.6.10-electric.11.md diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index a2ac6fa595..39e1944b4a 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "source": { "source": "local", "path": "./gitnexus-claude-plugin" diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 0592aee5bf..4dff9ff269 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "source": "./gitnexus-claude-plugin", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase." } diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4390cae8ce..1586573f97 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -362,19 +362,38 @@ Inheritance is captured by the `@reference.inherits` tag and emitted by the scop ``` CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) - 1. Early exit if lastCommit == HEAD (unless --force) [0%] - 2. Cache existing embeddings from prior index [0%] - 3. runPipelineFromRepo() → KnowledgeGraph [0-60%] - 4. Clean up legacy KuzuDB files [60%] - 5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%] - 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] - 7. Restore cached embeddings (batch insert) [88%] - 8. Generate new embeddings if --embeddings [90-98%] - 9. Save metadata + register repo + update .gitignore [98-100%] - 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] + 1. Early exit if lastCommit == HEAD (unless --force) [0%] + 2. Read metadata and select surgical incremental, full, or staged [0%] + 3. Capture a bounded embedding snapshot when a rebuild needs restore [0%] + 4. runPipelineFromRepo() → KnowledgeGraph [0-60%] + 5. Write changed rows surgically, or wipe + bulk-COPY a full graph [60-85%] + 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] + 7. Restore cached embeddings in bounded batches [88%] + 8. Generate new embeddings within the requested cap [90-98%] + 9. Validate, save metadata, register repo, and update .gitignore [98-100%] + 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] ``` -**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`. +**Options:** `--force` (rebuild regardless), `--embeddings [n]` (opt-in; a positive `n` is the +embedding node cap, `0` disables it), `--staged` (isolated generation plus validated promotion), +`--skipGit`, `--noStats`. + +### Bounded embedding preservation and staged recovery + +- For a healthy index that is stale against its source, a positive-cap incremental run (`--embeddings `) + rewrites only the changed/effective file set and regenerates changed or new embedding owners. + Unaffected owners retain their complete logical embedding-row payload. If the effective write set is + large, the existing escalation valve selects the full-write plan; the bounded snapshot and restore + contract remains the same. +- For malformed or provenance-unproven embedding state, use an explicit positive-cap staged clean + rebuild (`--staged --embeddings --drop-embeddings`). The staged database is disposable and the + canonical generation stays readable until validation and journaled promotion complete. A promotion + failure restores the canonical logical embedding-row and metadata preimage. +- The deterministic M8b proof compares every unaffected embedding-row payload field (identity, owner, + chunk, line range, content hash, and vector). It proves logical payload-byte preservation on that + supported path, not unchanged raw LadybugDB container bytes. +- Legacy rows whose metadata lacks durable provider or identity proof remain unproven and are not + described as verified preservation. A clean staged rebuild replaces that unclassified state. ## Storage @@ -385,6 +404,8 @@ CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) ├── lbug.shadow # Shadow sidecar (checkpoint staging) ├── lbug.lock # Single-writer lock ├── lbug.{wal,shadow}.dirty-recovery # parked sidecars from a crashed run; safe to delete + ├── embedding-preservation.jsonl # transient checksummed restore snapshot + ├── embedding-table-rebuild.json # transient staged-table recovery marker ├── gitnexus.json # lastCommit, indexedAt, stats (primary metadata file) └── meta.json # legacy mirror of gitnexus.json, kept in sync (see MIGRATION.md) diff --git a/Documentation/releases/1.6.10-electric.11.md b/Documentation/releases/1.6.10-electric.11.md new file mode 100644 index 0000000000..dc4e04560d --- /dev/null +++ b/Documentation/releases/1.6.10-electric.11.md @@ -0,0 +1,46 @@ +# GitNexus 1.6.10-electric.11 + +`1.6.10-electric.11` carries the post-M8b release-preparation contract for bounded embedding +preservation and staged recovery. It packages the already-supported paths; it does not add a +new preservation schema, lock, service, or public preservation command surface. + +## Behavior contract + +- **Healthy/source-stale copy:** run incremental analysis with a positive cap, for example + `gitnexus analyze --embeddings `. Unaffected embedding owners retain their complete + logical row payload bytes while changed or new owners are regenerated within the cap. +- **Malformed/provenance-unproven copy:** run an explicit positive-cap staged clean rebuild: + `gitnexus analyze --staged --embeddings --drop-embeddings`. The disposable staged + generation is validated before promotion; the canonical generation remains readable and + unchanged until that boundary. +- The M8b disposable canary deterministically compared the complete unaffected embedding-row + payload (identity, owner, chunk, line range, content hash, and vector). This proves logical + row-payload preservation for that supported path; it does not claim raw LadybugDB container + bytes are unchanged. +- Rows inherited from older metadata without durable provider or identity proof remain + unproven and are not called verified preservation. The clean staged rebuild replaces that + unclassified state rather than upgrading its claim. + +## Safety and rollback boundary + +- Staged recovery retains a canonical preimage and uses journaled promotion/rollback. The + M8b receipt proves exact logical embedding-row and metadata preimage restoration after an + injected promotion failure, with no staged, backup, or journal residue. +- `1.6.10-electric.10` remains available as the immediate non-writing rollback artifact. +- This source gate is local and disposable-copy evidence only. It does not prove the canonical + 65-index estate, fleet, production, customer, browser, or runtime readiness. + +## Distribution boundary + +- Electric releases are GitHub Release assets only. `.11` is not published to npm, a container + registry, or any other package registry, and no floating tag is used. +- This release-prep change does not install, dispatch the release workflow, mutate a registry or + live index, or perform a runtime/customer rollout. + +## Source-gate evidence + +- Exact post-M8b main: `50a0553c01fea92edff560f9e43df838235b3cf0`. +- Milestone 10 closeout receipt: SHA-256 + `0f6193fa5066745108a79acbeeccf485e4a52c7d08e0a5c67a6fa334ba0c94f8`. +- The receipt is the source/disposable-copy gate carried into this release-prep PR; release, + install, and runtime claims require their separate exact-head workflow evidence. diff --git a/GUARDRAILS.md b/GUARDRAILS.md index a6297f4f43..993b29c5ca 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -20,6 +20,7 @@ Maintainer may widen scope per task. 3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. 4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. 5. **Preserve embeddings** — plain `npx gitnexus analyze` now preserves any embeddings recorded in the index metadata (`.gitnexus/gitnexus.json`, mirrored to the legacy `meta.json`) — the previous behavior wiped them. Use `--embeddings` to also generate vectors for new/changed nodes; use `--drop-embeddings` only when an explicit wipe is intended (e.g., model swap). +6. **Use the bounded recovery contract** — for healthy indexes that are stale against source, use a positive embedding cap (`--embeddings `) so the incremental path retains unaffected logical embedding-row payloads and regenerates only changed/new owners. For malformed or provenance-unproven state, use an explicit positive-cap staged clean rebuild (`--staged --embeddings --drop-embeddings`) and keep the canonical generation as the rollback preimage until validated promotion completes. Older rows without durable provider or identity proof remain unproven; never call them verified preservation. --- @@ -45,6 +46,12 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Do:** Re-run `npx gitnexus analyze --embeddings` to regenerate. Check the analyze log for a `Warning: could not load cached embeddings` line — if present, the cache restore failed (corrupt DB / schema mismatch) and the rebuild had nothing to preserve. If you intentionally passed `--drop-embeddings`, this is expected. - **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; the only ways to end up at zero are an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. +### Embedding state is malformed or provenance is unproven + +- **Trigger:** Doctor or read-only integrity inspection reports malformed, duplicate, orphaned, wrong-dimension, or otherwise unproven embedding state, including legacy rows without durable provider or identity proof. +- **Do:** Preserve the canonical logical embedding-row and metadata preimage, then run an explicit positive-cap staged clean rebuild: `npx gitnexus analyze --staged --embeddings --drop-embeddings`. Validate the isolated generation before journaled promotion; on a promotion failure, use the staged rollback path and verify that the logical preimage is restored with no stage, backup, or journal residue. +- **Why:** A staged clean rebuild gives malformed or unclassified derived state a bounded replacement path without treating old rows as verified. The deterministic proof covers logical row payloads, not raw LadybugDB container bytes. + ### MCP lists no repos - **Trigger:** MCP stderr says no indexed repos. @@ -74,6 +81,7 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ## Publishing & supply chain - **npm:** Do not publish from unreviewed automation. Bump version intentionally; tag releases to match `package.json`. +- **Electric releases:** Distribution is GitHub Release assets only. Do not publish an Electric version to npm, a container registry, or another package registry, and do not use a floating tag. Keep `1.6.10-electric.10` available as the immediate non-writing rollback artifact while `.11` release/install/runtime work is proved separately. - **Dependencies:** Minimal, auditable `package.json` changes; run tests and CI after lockfile updates. - **License:** PolyForm Noncommercial 1.0.0 — do not relicense without maintainer approval. diff --git a/MIGRATION.md b/MIGRATION.md index f6af6c6a72..7f1789a45f 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -110,3 +110,21 @@ repo as never analyzed. The `meta.json` mirror will remain until a future major version. Removal will be announced in this file and in the changelog before it happens. + +## Bounded embedding preservation and staged recovery (`1.6.10-electric.11`) + +There is no embedding-schema migration. The release documents two existing paths: + +- A healthy index that is stale against its source can use a positive-cap incremental run, + `gitnexus analyze --embeddings `. Unaffected logical embedding-row payloads remain intact; + changed or new owners are regenerated within the cap. +- Malformed or provenance-unproven embedding state uses an explicit positive-cap staged clean + rebuild, `gitnexus analyze --staged --embeddings --drop-embeddings`. The staged generation is + validated before promotion, and the canonical logical embedding-row and metadata preimage stays + available for journaled rollback. + +Rows inherited from metadata without durable provider or identity proof remain unproven. They are +not called verified preservation; the clean staged rebuild is the replacement path for that state. +The deterministic proof covers logical row payloads, not raw LadybugDB container bytes. Existing +`1.6.10-electric.10` installation bytes remain suitable for non-writing rollback while `.11` +release and runtime steps are handled separately. diff --git a/RUNBOOK.md b/RUNBOOK.md index f5ecd0d91c..90c7dc8e20 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -56,7 +56,11 @@ npx gitnexus list npx gitnexus analyze --embeddings ``` -**Important:** If you already had embeddings, **always** pass `--embeddings` on later analyzes, or they can be dropped. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none). +**Important:** A plain `analyze` preserves existing embeddings. Pass `--embeddings ` when +you also want changed or new owners regenerated under a positive node cap; use +`--drop-embeddings` only when intentionally replacing the existing vectors. See +`stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means +none). **Large repos:** Analyze may skip or limit embedding work when node counts are very high; watch CLI output. @@ -164,6 +168,32 @@ If the error text is `"Only one write transaction at a time is allowed in the sy - Agent safety rules: [GUARDRAILS.md](GUARDRAILS.md) - Tests: [TESTING.md](TESTING.md) +## Bounded embedding preservation and staged recovery + +For a healthy index that is stale against its source, run incremental analysis with a positive +embedding cap: + +```bash +npx gitnexus analyze --embeddings +``` + +The incremental path rewrites the changed/effective file set, regenerates changed or new +embedding owners within the cap, and retains unaffected logical embedding-row payloads. If the +effective write set is large, the existing escalation valve may select a full write while +retaining the same bounded snapshot and restore contract. + +For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild: + +```bash +npx gitnexus analyze --staged --embeddings --drop-embeddings +``` + +The staged generation is disposable and the canonical generation remains readable until +validation and journaled promotion complete. Keep the canonical logical embedding-row and +metadata preimage available for rollback. The deterministic proof covers logical row payloads, +not raw LadybugDB container bytes; legacy rows without durable provider or identity proof remain +unproven and are not called verified preservation. + --- ## Electric fork releases @@ -173,8 +203,8 @@ to npm, GitHub Container Registry, Docker Hub, or another package registry. Prepare each version through a reviewed release PR: -1. Set `gitnexus/package.json` and `gitnexus/package-lock.json` to a version such - as `1.6.10-electric.1`. +1. Set `gitnexus/package.json` and `gitnexus/package-lock.json` to the reviewed + fork version, such as `1.6.10-electric.11`. 2. Run `node gitnexus/scripts/sync-plugin-versions.mjs` from the repository root. 3. Add `Documentation/releases/.md` and update `gitnexus/CHANGELOG.md`. 4. Run `npm run check:electric-release-policy`, the plugin-version test, build, @@ -187,7 +217,7 @@ Create the release from the current `main` branch with the protected workflow: gh workflow run electric-release.yml \ --repo electricsheephq/evaOS-gitnexus \ --ref main \ - -f expected_version=1.6.10-electric.1 \ + -f expected_version=1.6.10-electric.11 \ -f prerelease=false ``` @@ -219,10 +249,11 @@ After downloading both assets, verify and install locally: ```bash shasum -a 256 --check SHA256SUMS -npm install --global ./gitnexus-1.6.10-electric.1.tgz +npm install --global ./gitnexus-1.6.10-electric.11.tgz gitnexus --version ``` A GitHub Release does not authorize an OpenClaw/evaOS rollout. Keep the prior -installation available for rollback, switch runtime source separately, and do -not rebuild or migrate a live index unless that rollout explicitly requires it. +`1.6.10-electric.10` installation available as non-writing rollback bytes, switch +runtime source separately, and do not rebuild or migrate a live index unless that +rollout explicitly requires it. diff --git a/gitnexus-claude-plugin/.claude-plugin/plugin.json b/gitnexus-claude-plugin/.claude-plugin/plugin.json index caac21b88f..2d6b067dda 100644 --- a/gitnexus-claude-plugin/.claude-plugin/plugin.json +++ b/gitnexus-claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "author": { "name": "GitNexus" }, diff --git a/gitnexus-claude-plugin/.codex-plugin/plugin.json b/gitnexus-claude-plugin/.codex-plugin/plugin.json index d57f9f52ec..d7969761bd 100644 --- a/gitnexus-claude-plugin/.codex-plugin/plugin.json +++ b/gitnexus-claude-plugin/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "skills": "./skills", "mcpServers": "./.mcp.json", "hooks": "./hooks/hooks.json", diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index b046c92bf6..5ce4d493d7 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +## [1.6.10-electric.11] - 2026-08-31 + +### Changed + +- **Healthy source-stale indexes keep their unchanged embedding payloads on the capped + incremental path**: run `analyze` with a positive embedding cap (`--embeddings `) to + regenerate changed or new owners while retaining unaffected logical embedding-row payload + bytes. Large effective write sets may still use the existing full-write escalation while + preserving the same bounded snapshot contract. +- **Malformed or provenance-unproven indexes use an explicit capped staged clean rebuild**: + `--staged --embeddings --drop-embeddings` builds an isolated generation, keeps the + canonical generation untouched until validation and promotion, and retains the canonical + logical embedding-row and metadata preimage for journaled rollback on a promotion failure. +- Rows from older metadata that lack durable provider or identity proof remain unclassified; + they are not described as verified preservation. + +### Release boundary + +- Electric distribution remains GitHub Release assets only. This version does not publish to + npm or a container registry, follow a floating tag, or perform runtime, index, fleet, or + customer mutation. +- The installed `1.6.10-electric.10` artifact remains the immediate non-writing rollback + runtime while `.11` release and install proof are completed separately. + ## [1.6.10-electric.10] - 2026-08-20 ### Fixed diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index ad6e7701f4..99603a85dc 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,12 +1,12 @@ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { diff --git a/gitnexus/package.json b/gitnexus/package.json index a0b7eb9884..6a73c1c979 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", From 22b6cbe939e39ea9250ac5ce799085a83ec12dab Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 07:35:09 +0700 Subject: [PATCH 2/9] docs: clarify embedding cap admission --- ARCHITECTURE.md | 7 ++++--- Documentation/releases/1.6.10-electric.11.md | 10 ++++++---- GUARDRAILS.md | 4 ++-- MIGRATION.md | 12 +++++++----- RUNBOOK.md | 11 ++++++----- gitnexus/CHANGELOG.md | 9 +++++---- 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1586573f97..10e1f8a0c4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -381,12 +381,13 @@ embedding node cap, `0` disables it), `--staged` (isolated generation plus valid ### Bounded embedding preservation and staged recovery - For a healthy index that is stale against its source, a positive-cap incremental run (`--embeddings `) - rewrites only the changed/effective file set and regenerates changed or new embedding owners. + first admits or refuses the entire embedding pass against the total graph node count. When admitted, + it rewrites only the changed/effective file set and regenerates changed or new embedding owners. Unaffected owners retain their complete logical embedding-row payload. If the effective write set is large, the existing escalation valve selects the full-write plan; the bounded snapshot and restore contract remains the same. -- For malformed or provenance-unproven embedding state, use an explicit positive-cap staged clean - rebuild (`--staged --embeddings --drop-embeddings`). The staged database is disposable and the +- For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild with a + positive cap on total graph nodes (`--staged --embeddings --drop-embeddings`). The staged database is disposable and the canonical generation stays readable until validation and journaled promotion complete. A promotion failure restores the canonical logical embedding-row and metadata preimage. - The deterministic M8b proof compares every unaffected embedding-row payload field (identity, owner, diff --git a/Documentation/releases/1.6.10-electric.11.md b/Documentation/releases/1.6.10-electric.11.md index dc4e04560d..b1cfeb6e9c 100644 --- a/Documentation/releases/1.6.10-electric.11.md +++ b/Documentation/releases/1.6.10-electric.11.md @@ -6,10 +6,12 @@ new preservation schema, lock, service, or public preservation command surface. ## Behavior contract -- **Healthy/source-stale copy:** run incremental analysis with a positive cap, for example - `gitnexus analyze --embeddings `. Unaffected embedding owners retain their complete - logical row payload bytes while changed or new owners are regenerated within the cap. -- **Malformed/provenance-unproven copy:** run an explicit positive-cap staged clean rebuild: +- **Healthy/source-stale copy:** run incremental analysis with a positive total-graph node cap, + for example `gitnexus analyze --embeddings `. GitNexus refuses the whole embedding pass + when the graph has more than `` nodes. When admitted, unaffected embedding owners retain + their complete logical row payload bytes while changed or new owners are regenerated. +- **Malformed/provenance-unproven copy:** run an explicit staged clean rebuild with a positive cap + on total graph nodes: `gitnexus analyze --staged --embeddings --drop-embeddings`. The disposable staged generation is validated before promotion; the canonical generation remains readable and unchanged until that boundary. diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 993b29c5ca..50b60a7f53 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -20,7 +20,7 @@ Maintainer may widen scope per task. 3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. 4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. 5. **Preserve embeddings** — plain `npx gitnexus analyze` now preserves any embeddings recorded in the index metadata (`.gitnexus/gitnexus.json`, mirrored to the legacy `meta.json`) — the previous behavior wiped them. Use `--embeddings` to also generate vectors for new/changed nodes; use `--drop-embeddings` only when an explicit wipe is intended (e.g., model swap). -6. **Use the bounded recovery contract** — for healthy indexes that are stale against source, use a positive embedding cap (`--embeddings `) so the incremental path retains unaffected logical embedding-row payloads and regenerates only changed/new owners. For malformed or provenance-unproven state, use an explicit positive-cap staged clean rebuild (`--staged --embeddings --drop-embeddings`) and keep the canonical generation as the rollback preimage until validated promotion completes. Older rows without durable provider or identity proof remain unproven; never call them verified preservation. +6. **Use the bounded recovery contract** — for healthy indexes that are stale against source, use a positive cap on total graph nodes (`--embeddings `). The cap admits or refuses the whole embedding pass before generation; when admitted, the incremental path retains unaffected logical embedding-row payloads and regenerates only changed/new owners. For malformed or provenance-unproven state, use an explicit staged clean rebuild with the same positive total-graph cap (`--staged --embeddings --drop-embeddings`) and keep the canonical generation as the rollback preimage until validated promotion completes. Older rows without durable provider or identity proof remain unproven; never call them verified preservation. --- @@ -49,7 +49,7 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Embedding state is malformed or provenance is unproven - **Trigger:** Doctor or read-only integrity inspection reports malformed, duplicate, orphaned, wrong-dimension, or otherwise unproven embedding state, including legacy rows without durable provider or identity proof. -- **Do:** Preserve the canonical logical embedding-row and metadata preimage, then run an explicit positive-cap staged clean rebuild: `npx gitnexus analyze --staged --embeddings --drop-embeddings`. Validate the isolated generation before journaled promotion; on a promotion failure, use the staged rollback path and verify that the logical preimage is restored with no stage, backup, or journal residue. +- **Do:** Preserve the canonical logical embedding-row and metadata preimage, then run an explicit staged clean rebuild with a positive cap on total graph nodes: `npx gitnexus analyze --staged --embeddings --drop-embeddings`. Validate the isolated generation before journaled promotion; on a promotion failure, use the staged rollback path and verify that the logical preimage is restored with no stage, backup, or journal residue. - **Why:** A staged clean rebuild gives malformed or unclassified derived state a bounded replacement path without treating old rows as verified. The deterministic proof covers logical row payloads, not raw LadybugDB container bytes. ### MCP lists no repos diff --git a/MIGRATION.md b/MIGRATION.md index 7f1789a45f..699724fc5a 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -115,11 +115,13 @@ will be announced in this file and in the changelog before it happens. There is no embedding-schema migration. The release documents two existing paths: -- A healthy index that is stale against its source can use a positive-cap incremental run, - `gitnexus analyze --embeddings `. Unaffected logical embedding-row payloads remain intact; - changed or new owners are regenerated within the cap. -- Malformed or provenance-unproven embedding state uses an explicit positive-cap staged clean - rebuild, `gitnexus analyze --staged --embeddings --drop-embeddings`. The staged generation is +- A healthy index that is stale against its source can use an incremental run with a positive cap + on total graph nodes, + run, `gitnexus analyze --embeddings `. The pass is refused when the graph has more than + `` nodes; when admitted, unaffected logical embedding-row payloads remain intact and changed + or new owners are regenerated. +- Malformed or provenance-unproven embedding state uses an explicit staged clean rebuild with the + same positive total-graph cap, `gitnexus analyze --staged --embeddings --drop-embeddings`. The staged generation is validated before promotion, and the canonical logical embedding-row and metadata preimage stays available for journaled rollback. diff --git a/RUNBOOK.md b/RUNBOOK.md index 90c7dc8e20..b51e16705c 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -171,16 +171,17 @@ If the error text is `"Only one write transaction at a time is allowed in the sy ## Bounded embedding preservation and staged recovery For a healthy index that is stale against its source, run incremental analysis with a positive -embedding cap: +total-graph node cap: ```bash npx gitnexus analyze --embeddings ``` -The incremental path rewrites the changed/effective file set, regenerates changed or new -embedding owners within the cap, and retains unaffected logical embedding-row payloads. If the -effective write set is large, the existing escalation valve may select a full write while -retaining the same bounded snapshot and restore contract. +GitNexus refuses the whole embedding pass when the graph has more than `` nodes. When admitted, +the incremental path rewrites the changed/effective file set, regenerates changed or new embedding +owners, and retains unaffected logical embedding-row payloads. If the effective write set is large, +the existing escalation valve may select a full write while retaining the same bounded snapshot and +restore contract. For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild: diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index 5ce4d493d7..0932ff261c 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -9,10 +9,11 @@ All notable changes to GitNexus will be documented in this file. ### Changed - **Healthy source-stale indexes keep their unchanged embedding payloads on the capped - incremental path**: run `analyze` with a positive embedding cap (`--embeddings `) to - regenerate changed or new owners while retaining unaffected logical embedding-row payload - bytes. Large effective write sets may still use the existing full-write escalation while - preserving the same bounded snapshot contract. + incremental path**: run `analyze` with a positive total-graph node cap (`--embeddings `). + GitNexus refuses the whole embedding pass when the graph exceeds the cap; when admitted, it + regenerates changed or new owners while retaining unaffected logical embedding-row payload bytes. + Large effective write sets may still use the existing full-write escalation while preserving the + same bounded snapshot contract. - **Malformed or provenance-unproven indexes use an explicit capped staged clean rebuild**: `--staged --embeddings --drop-embeddings` builds an isolated generation, keeps the canonical generation untouched until validation and promotion, and retains the canonical From 0b73af127d03af1eaf1c0e6bf224315f4247025e Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 08:21:04 +0700 Subject: [PATCH 3/9] fix(embeddings): refuse capped runs before persistence --- ARCHITECTURE.md | 4 +- Documentation/releases/1.6.10-electric.11.md | 6 +- GUARDRAILS.md | 2 +- MIGRATION.md | 6 +- RUNBOOK.md | 12 +- gitnexus/CHANGELOG.md | 4 +- gitnexus/src/core/run-analyze.ts | 114 ++++++++++-------- .../m8b-incremental-embedding-restore.test.ts | 71 +++++++++++ .../test/unit/run-analyze-fts-repair.test.ts | 56 +++++---- gitnexus/test/unit/run-analyze-staged.test.ts | 53 ++++++++ 10 files changed, 242 insertions(+), 86 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 10e1f8a0c4..2587175a03 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -381,7 +381,9 @@ embedding node cap, `0` disables it), `--staged` (isolated generation plus valid ### Bounded embedding preservation and staged recovery - For a healthy index that is stale against its source, a positive-cap incremental run (`--embeddings `) - first admits or refuses the entire embedding pass against the total graph node count. When admitted, + first admits or refuses the entire embedding pass against the provider-free pipeline's total graph + node count, before graph persistence, final metadata, registry update, or staged promotion. Transient + pipeline caches and prepared stage recovery files may already exist. When admitted, it rewrites only the changed/effective file set and regenerates changed or new embedding owners. Unaffected owners retain their complete logical embedding-row payload. If the effective write set is large, the existing escalation valve selects the full-write plan; the bounded snapshot and restore diff --git a/Documentation/releases/1.6.10-electric.11.md b/Documentation/releases/1.6.10-electric.11.md index b1cfeb6e9c..e83f0f1e28 100644 --- a/Documentation/releases/1.6.10-electric.11.md +++ b/Documentation/releases/1.6.10-electric.11.md @@ -8,8 +8,10 @@ new preservation schema, lock, service, or public preservation command surface. - **Healthy/source-stale copy:** run incremental analysis with a positive total-graph node cap, for example `gitnexus analyze --embeddings `. GitNexus refuses the whole embedding pass - when the graph has more than `` nodes. When admitted, unaffected embedding owners retain - their complete logical row payload bytes while changed or new owners are regenerated. + when the graph has more than `` nodes, before graph persistence, final metadata, registry + update, or staged promotion. Provider-free pipeline caches and prepared stage recovery files may + already exist. When admitted, unaffected embedding owners retain their complete logical row + payload bytes while changed or new owners are regenerated. - **Malformed/provenance-unproven copy:** run an explicit staged clean rebuild with a positive cap on total graph nodes: `gitnexus analyze --staged --embeddings --drop-embeddings`. The disposable staged diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 50b60a7f53..5f70d8aa64 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -20,7 +20,7 @@ Maintainer may widen scope per task. 3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. 4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. 5. **Preserve embeddings** — plain `npx gitnexus analyze` now preserves any embeddings recorded in the index metadata (`.gitnexus/gitnexus.json`, mirrored to the legacy `meta.json`) — the previous behavior wiped them. Use `--embeddings` to also generate vectors for new/changed nodes; use `--drop-embeddings` only when an explicit wipe is intended (e.g., model swap). -6. **Use the bounded recovery contract** — for healthy indexes that are stale against source, use a positive cap on total graph nodes (`--embeddings `). The cap admits or refuses the whole embedding pass before generation; when admitted, the incremental path retains unaffected logical embedding-row payloads and regenerates only changed/new owners. For malformed or provenance-unproven state, use an explicit staged clean rebuild with the same positive total-graph cap (`--staged --embeddings --drop-embeddings`) and keep the canonical generation as the rollback preimage until validated promotion completes. Older rows without durable provider or identity proof remain unproven; never call them verified preservation. +6. **Use the bounded recovery contract** — for healthy indexes that are stale against source, use a positive cap on total graph nodes (`--embeddings `). The cap admits or refuses the whole embedding pass after the provider-free pipeline but before graph persistence, final metadata, registry update, or staged promotion; transient caches or prepared stage recovery files may already exist. When admitted, the incremental path retains unaffected logical embedding-row payloads and regenerates only changed/new owners. For malformed or provenance-unproven state, use an explicit staged clean rebuild with the same positive total-graph cap (`--staged --embeddings --drop-embeddings`) and keep the canonical generation as the rollback preimage until validated promotion completes. Older rows without durable provider or identity proof remain unproven; never call them verified preservation. --- diff --git a/MIGRATION.md b/MIGRATION.md index 699724fc5a..4141d7c44c 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -118,8 +118,10 @@ There is no embedding-schema migration. The release documents two existing paths - A healthy index that is stale against its source can use an incremental run with a positive cap on total graph nodes, run, `gitnexus analyze --embeddings `. The pass is refused when the graph has more than - `` nodes; when admitted, unaffected logical embedding-row payloads remain intact and changed - or new owners are regenerated. + `` nodes. Refusal happens before graph persistence, final metadata, registry update, or staged + promotion, although transient pipeline caches or prepared stage recovery files may already exist. + When admitted, unaffected logical embedding-row payloads remain intact and changed or new owners + are regenerated. - Malformed or provenance-unproven embedding state uses an explicit staged clean rebuild with the same positive total-graph cap, `gitnexus analyze --staged --embeddings --drop-embeddings`. The staged generation is validated before promotion, and the canonical logical embedding-row and metadata preimage stays diff --git a/RUNBOOK.md b/RUNBOOK.md index b51e16705c..0e157e8fd7 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -177,11 +177,13 @@ total-graph node cap: npx gitnexus analyze --embeddings ``` -GitNexus refuses the whole embedding pass when the graph has more than `` nodes. When admitted, -the incremental path rewrites the changed/effective file set, regenerates changed or new embedding -owners, and retains unaffected logical embedding-row payloads. If the effective write set is large, -the existing escalation valve may select a full write while retaining the same bounded snapshot and -restore contract. +GitNexus refuses the whole embedding pass when the provider-free pipeline reports more than `` +nodes. Refusal happens before graph persistence, final metadata, registry update, or staged +promotion; transient pipeline caches or prepared stage recovery files may already exist. When +admitted, the incremental path rewrites the changed/effective file set, regenerates changed or new +embedding owners, and retains unaffected logical embedding-row payloads. If the effective write set +is large, the existing escalation valve may select a full write while retaining the same bounded +snapshot and restore contract. For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild: diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index 0932ff261c..bf66cfabee 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -10,7 +10,9 @@ All notable changes to GitNexus will be documented in this file. - **Healthy source-stale indexes keep their unchanged embedding payloads on the capped incremental path**: run `analyze` with a positive total-graph node cap (`--embeddings `). - GitNexus refuses the whole embedding pass when the graph exceeds the cap; when admitted, it + GitNexus refuses the whole embedding pass after the provider-free pipeline but before graph + persistence, final metadata, registry update, or staged promotion; transient caches or prepared + stage recovery files may already exist. When admitted, it regenerates changed or new owners while retaining unaffected logical embedding-row payload bytes. Large effective write sets may still use the existing full-write escalation while preserving the same bounded snapshot contract. diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 3da3e6e4ad..fb685c6ad9 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -2368,6 +2368,68 @@ const runFullAnalysisImpl = async ( }, ); + const streamedPdgNodeRows = pipelineResult.pdgEmitManifest + ? [...pipelineResult.pdgEmitManifest.nodeFiles.values()].reduce( + (total, entry) => total + entry.rows, + 0, + ) + : 0; + const streamedPdgRelationshipRows = pipelineResult.pdgEmitManifest + ? [...pipelineResult.pdgEmitManifest.relsByPair.values()].reduce( + (total, entry) => total + entry.rows, + 0, + ) + : 0; + const streamedPdgRows = streamedPdgNodeRows + streamedPdgRelationshipRows; + + // An explicit positive embedding cap is an all-or-nothing admission gate, + // not a request to finish the graph write without semantic rows. Settle it + // immediately after the provider-free pipeline has produced the complete + // prospective node count and before any LadybugDB, metadata, registry, or + // staged-promotion write begins. Pipeline cache reconciliation and prepared + // staging files may already exist; the refusal deliberately preserves them + // for the existing recovery path. + let embeddingSkipped = true; + let semanticMode: 'vector-index' | 'exact-scan' | undefined; + let httpMode = false; + if (shouldGenerateEmbeddings) { + const { isHttpMode } = await import('./embeddings/http-client.js'); + httpMode = isHttpMode(); + const embeddingAdmissionNodeCount = pipelineResult.graph.nodeCount + streamedPdgNodeRows; + const { skipForCap, capDisabled, nodeLimit } = deriveEmbeddingCap( + embeddingAdmissionNodeCount, + resolveEmbeddingNodeLimit(options.embeddingsNodeLimit, resumeEmbeddingCheckpoint), + httpMode, + ); + if (skipForCap) { + throw new Error( + `Embedding generation refused: ${embeddingAdmissionNodeCount.toLocaleString()} nodes exceeds ` + + `the ${nodeLimit.toLocaleString()}-node safety cap. ` + + 'No analyze graph write, final metadata commit, or staged promotion was started. ' + + 'Retry with `--embeddings 0` or a limit covering the full graph.', + ); + } + + embeddingSkipped = false; + if (capDisabled && embeddingAdmissionNodeCount > DEFAULT_EMBEDDING_NODE_LIMIT) { + if (httpMode) { + log( + `Remote embedding endpoint selected — generating embeddings for ` + + `${embeddingAdmissionNodeCount.toLocaleString()} nodes; the ` + + `${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model cap ` + + `does not apply.`, + ); + } else { + log( + `Embedding node-count cap disabled — generating embeddings for ` + + `${embeddingAdmissionNodeCount.toLocaleString()} nodes. Ensure sufficient memory; the ` + + `default ${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model ` + + `cap exists to prevent OOM.`, + ); + } + } + } + // ── Phase 2: LadybugDB (60–85%) ────────────────────────────────── progress('lbug', 60, 'Loading into LadybugDB...'); @@ -2464,12 +2526,6 @@ const runFullAnalysisImpl = async ( await wipeLbugDbFiles(lbugPath); } - const streamedPdgRows = pipelineResult.pdgEmitManifest - ? [ - ...pipelineResult.pdgEmitManifest.nodeFiles.values(), - ...pipelineResult.pdgEmitManifest.relsByPair.values(), - ].reduce((total, entry) => total + entry.rows, 0) - : 0; setBufferPoolSizeHint( estimateBufferPool( pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount + streamedPdgRows, @@ -2937,50 +2993,10 @@ const runFullAnalysisImpl = async ( progress('fts', 90, 'Search indexes skipped (FTS unavailable)'); } - // Settle embedding admission before streaming the snapshot. Exact row-ID - // sidecars are needed only when Phase 4 will run; preserve-only and - // cap-refused analyses must keep the snapshot pass bounded to its batches. + // Exact row-ID sidecars are needed only when Phase 4 will run; preserve-only + // analyses keep the snapshot pass bounded to its batches. Explicit positive + // cap refusal already terminated before Phase 2. const stats = await getLbugStats(); - let embeddingSkipped = true; - let semanticMode: 'vector-index' | 'exact-scan' | undefined; - let httpMode = false; - - if (shouldGenerateEmbeddings) { - const { isHttpMode } = await import('./embeddings/http-client.js'); - httpMode = isHttpMode(); - const { skipForCap, capDisabled, nodeLimit } = deriveEmbeddingCap( - stats.nodes, - resolveEmbeddingNodeLimit(options.embeddingsNodeLimit, resumeEmbeddingCheckpoint), - httpMode, - ); - if (!skipForCap) { - embeddingSkipped = false; - if (capDisabled && stats.nodes > DEFAULT_EMBEDDING_NODE_LIMIT) { - if (httpMode) { - log( - `Remote embedding endpoint selected — generating embeddings for ` + - `${stats.nodes.toLocaleString()} nodes; the ` + - `${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model cap ` + - `does not apply.`, - ); - } else { - log( - `Embedding node-count cap disabled — generating embeddings for ` + - `${stats.nodes.toLocaleString()} nodes. Ensure sufficient memory; the ` + - `default ${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model ` + - `cap exists to prevent OOM.`, - ); - } - } - } else { - log( - `Embeddings skipped: ${stats.nodes.toLocaleString()} nodes exceeds ` + - `the ${nodeLimit.toLocaleString()}-node safety cap. ` + - `Override with \`--embeddings 0\` to disable the cap, or ` + - `\`--embeddings \` to set a custom cap.`, - ); - } - } // ── Phase 3.5: Re-insert cached embeddings ──────────────────────── // Runs on BOTH the full-rebuild path and the incremental path: diff --git a/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts b/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts index 873ff6a313..8f5a6a9a66 100644 --- a/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts +++ b/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts @@ -194,4 +194,75 @@ describe('M8b incremental embedding restore', () => { vi.unstubAllGlobals(); } }, 180_000); + + it('refuses an over-cap incremental run before canonical graph or metadata mutation', async () => { + const previousEnv = new Map(EMBEDDING_ENV_KEYS.map((key) => [key, process.env[key]] as const)); + const home = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-cap-refusal-home-')); + const repo = await setupMiniRepo('gitnexus-cap-refusal-'); + try { + process.env.GITNEXUS_HOME = home; + process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'never'; + process.env.GITNEXUS_EMBEDDING_URL = 'http://in-process.invalid/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'cap-refusal-test'; + process.env.GITNEXUS_EMBEDDING_DIMS = String(EMBEDDING_DIMS); + process.env.GITNEXUS_EMBEDDING_MAX_ATTEMPTS = '1'; + process.env.GITNEXUS_EMBEDDING_RETRY_CAP_MS = '1'; + process.env.GITNEXUS_EMBEDDING_MIN_INTERVAL_MS = '0'; + delete process.env.GITNEXUS_EMBEDDING_API_KEY; + const fetchMock = vi.fn(async () => { + throw new Error('provider must not be called on cap refusal'); + }); + vi.stubGlobal('fetch', fetchMock); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis( + repo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ); + + const { lbugPath, storagePath } = getStoragePaths(repo.dbPath); + const graphBefore = await readFile(lbugPath); + const metaPath = path.join(storagePath, 'meta.json'); + const metaBefore = await readFile(metaPath); + + const handlerPath = path.join(repo.dbPath, 'src', 'handler.ts'); + const handlerSource = await readFile(handlerPath, 'utf8'); + await writeFile( + handlerPath, + handlerSource.replace( + 'return formatResponse(saved);', + "return formatResponse(saved) + ' cap-refusal';", + ), + 'utf8', + ); + gitCommitAll(repo.dbPath, 'cap refusal source delta'); + + await expect( + runFullAnalysis( + repo.dbPath, + { + incrementalOnly: true, + skipAgentsMd: true, + skipSkills: true, + embeddings: true, + embeddingsNodeLimit: 1, + }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/Embedding generation refused: .* exceeds the 1-node safety cap/i); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(await readFile(lbugPath)).toEqual(graphBefore); + expect(await readFile(metaPath)).toEqual(metaBefore); + } finally { + await repo.cleanup(); + await rm(home, { recursive: true, force: true }); + for (const [key, value] of previousEnv) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + vi.unstubAllGlobals(); + } + }, 120_000); }); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 3eae363218..83f1704dec 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -718,7 +718,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 vi.unstubAllEnvs(); }); - it('stamps capabilities.vectorSearch.status = exact-scan when post-restore index recreation reports failure', async () => { + it('refuses an over-cap rebuild before graph writes or cached-row restoration', async () => { const RESTORED_NODE_ID = 'Function:src/app.ts:handler:1'; const stubNode = { id: RESTORED_NODE_ID, @@ -726,11 +726,14 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 name: 'handler', properties: { filePath: 'src/app.ts' }, }; + const initLbug = vi.fn(async () => undefined); + const loadGraphToLbug = vi.fn(async () => undefined); + const wipeLbugDbFiles = vi.fn(async () => undefined); const buildVectorIndex = vi.fn(async () => false); const executeWithReusedStatement = vi.fn(async () => []); vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ - initLbug: vi.fn(async () => undefined), - loadGraphToLbug: vi.fn(async () => undefined), + initLbug, + loadGraphToLbug, getLbugStats: vi.fn(async () => ({ nodes: 2, edges: 0, communities: 0, processes: 0 })), // The finalize embedding count answers 1 (the restored row) — a zero // count would stamp 'unavailable' and the exact-scan assertion below @@ -747,7 +750,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 embeddingIntegrityFailures: vi.fn(() => 0), // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. - wipeLbugDbFiles: vi.fn(async () => undefined), + wipeLbugDbFiles, // ≥1 cached row with a real-dims embedding: the harness default (empty // cache) would leave restoredEmbeddingCount at 0 and the recreation // gate shut — this test would then assert nothing. @@ -787,7 +790,13 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, totalFileCount: 1, + pdgEmitManifest: { + nodeFiles: new Map([['chunk-0', { rows: 1 }]]), + relsByPair: new Map(), + }, graph: { + nodeCount: 1, + relationshipCount: 0, forEachNode: (fn: (node: typeof stubNode) => void) => fn(stubNode), getNode: (id: string) => (id === RESTORED_NODE_ID ? stubNode : undefined), }, @@ -811,35 +820,32 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 try { const { storagePath } = getStoragePaths(tmpRepo.dbPath); await fs.mkdir(storagePath, { recursive: true }); - // stats.embeddings > 0 → deriveEmbeddingMode loads the cache; force + - // embeddingsNodeLimit(1) < getLbugStats().nodes(2) → generation is - // cap-skipped. That makes this a wiped PRESERVE-shaped run — exactly - // the KTD1 case where a naive `!shouldGenerateEmbeddings` gate would - // wrongly stay shut (shouldGenerate is TRUE here, yet the Phase 4 - // pipeline never runs). - await saveMeta(storagePath, { + const originalMeta: RepoMeta = { repoPath: tmpRepo.dbPath, lastCommit: '', indexedAt: new Date().toISOString(), stats: { embeddings: 1 }, - }); + }; + await saveMeta(storagePath, originalMeta); const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); - await runFullAnalysis( - tmpRepo.dbPath, - { force: true, embeddingsNodeLimit: 1 }, - { onProgress: () => {} }, - ); + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { force: true, embeddingsNodeLimit: 1 }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/Embedding generation refused: 2 nodes exceeds the 1-node safety cap/i); - // The recreation seam fired exactly once… - expect(buildVectorIndex).toHaveBeenCalledTimes(1); - // …the restore actually submitted the cached row (one 200-row batch)… - expect(executeWithReusedStatement).toHaveBeenCalledTimes(1); - // …and the persisted stamp reflects the DB's ACTUAL state, not the - // platform capability fallback. + // The bounded pre-pipeline embedding snapshot may open the existing DB. + // Refusal must still happen before any graph load, wipe, or restore write. + expect(initLbug).toHaveBeenCalledTimes(1); + expect(loadGraphToLbug).not.toHaveBeenCalled(); + expect(wipeLbugDbFiles).not.toHaveBeenCalled(); + expect(buildVectorIndex).not.toHaveBeenCalled(); + expect(executeWithReusedStatement).not.toHaveBeenCalled(); const meta = JSON.parse(await fs.readFile(`${storagePath}/meta.json`, 'utf-8')) as RepoMeta; - expect(meta.capabilities?.vectorSearch.status).toBe('exact-scan'); - expect(meta.stats?.embeddings).toBe(1); + expect(meta).toEqual(originalMeta); } finally { await tmpRepo.cleanup(); } diff --git a/gitnexus/test/unit/run-analyze-staged.test.ts b/gitnexus/test/unit/run-analyze-staged.test.ts index e0b1dbff4c..818856b672 100644 --- a/gitnexus/test/unit/run-analyze-staged.test.ts +++ b/gitnexus/test/unit/run-analyze-staged.test.ts @@ -218,6 +218,59 @@ describe('runFullAnalysis --staged', () => { await expect(fs.access(staged.stageRoot)).rejects.toMatchObject({ code: 'ENOENT' }); }, 120_000); + it('refuses an over-cap staged rebuild before canonical mutation or promotion', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-staged-cap-refusal-')); + tempDirs.push(repo); + process.env.GITNEXUS_HOME = path.join(repo, '.registry-home'); + process.env.GITNEXUS_EMBEDDING_URL = 'http://test.invalid/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'cap-refusal-test'; + process.env.GITNEXUS_EMBEDDING_DIMS = String(EMBEDDING_DIMS); + const fetchMock = vi.fn(async () => { + throw new Error('provider must not be called on cap refusal'); + }); + vi.stubGlobal('fetch', fetchMock); + await fs.writeFile( + path.join(repo, 'index.ts'), + 'export function stagedCapRefusal() { return 1; }\n', + ); + execFileSync('git', ['init'], { cwd: repo }); + execFileSync('git', ['add', 'index.ts'], { cwd: repo }); + execFileSync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@test', 'commit', '-m', 'initial'], + { cwd: repo }, + ); + + await runFullAnalysis(repo, { skipAgentsMd: true, skipSkills: true }, { onProgress: () => {} }); + const canonical = getStoragePaths(repo); + const canonicalDbBefore = await fs.readFile(canonical.lbugPath); + const canonicalMetaBefore = await fs.readFile(path.join(canonical.storagePath, 'meta.json')); + const staged = getStagedAnalyzePaths(canonical.lbugPath, canonical.storagePath); + + await expect( + runFullAnalysis( + repo, + { + staged: true, + embeddings: true, + embeddingsNodeLimit: 1, + dropEmbeddings: true, + skipAgentsMd: true, + skipSkills: true, + }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/Embedding generation refused: .* exceeds the 1-node safety cap/i); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(await fs.readFile(canonical.lbugPath)).toEqual(canonicalDbBefore); + expect(await fs.readFile(path.join(canonical.storagePath, 'meta.json'))).toEqual( + canonicalMetaBefore, + ); + await expect(fs.access(staged.backupLbugPath)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(fs.access(staged.journalPath)).rejects.toMatchObject({ code: 'ENOENT' }); + }, 120_000); + it('never resumes a staged embedding checkpoint without an explicit clean rebuild', async () => { const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-staged-checkpoint-refusal-')); tempDirs.push(repo); From bf104797d854f806234585247627d6b64410ce32 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 08:32:32 +0700 Subject: [PATCH 4/9] test(analysis): exercise dirty recovery with admitted provider --- .../unit/incremental-dirty-recovery.test.ts | 70 +++++++++++++++---- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/gitnexus/test/unit/incremental-dirty-recovery.test.ts b/gitnexus/test/unit/incremental-dirty-recovery.test.ts index eafb933ce6..521b26813a 100644 --- a/gitnexus/test/unit/incremental-dirty-recovery.test.ts +++ b/gitnexus/test/unit/incremental-dirty-recovery.test.ts @@ -18,8 +18,9 @@ * Windows. */ +import { createHash } from 'node:crypto'; import { writeFile, readFile } from 'fs/promises'; -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { getStoragePaths, saveMeta, @@ -31,12 +32,32 @@ import { setupMiniRepo as setupSharedMiniRepo } from '../helpers/mini-repo.js'; // zero-vector seeding pattern previously lived here as a divergent copy of // incremental-orchestration.test.ts's (a helper module has no // describe-registration problem, unlike importing a sibling test file). -import { seedEmbeddingsForFiles } from '../helpers/embedding-seed.js'; +import { readEmbeddingNodeIds, seedEmbeddingsForFiles } from '../helpers/embedding-seed.js'; const setupMiniRepo = () => setupSharedMiniRepo('gitnexus-incr-dirty-rec-'); +const EMBEDDING_DIMS = 384; +const EMBEDDING_ENV_KEYS = [ + 'GITNEXUS_EMBEDDING_URL', + 'GITNEXUS_EMBEDDING_MODEL', + 'GITNEXUS_EMBEDDING_DIMS', + 'GITNEXUS_EMBEDDING_MAX_ATTEMPTS', + 'GITNEXUS_EMBEDDING_RETRY_CAP_MS', + 'GITNEXUS_EMBEDDING_MIN_INTERVAL_MS', + 'GITNEXUS_EMBEDDING_API_KEY', +] as const; + +const deterministicEmbedding = (text: string): number[] => { + const digest = createHash('sha256').update(text).digest(); + return Array.from( + { length: EMBEDDING_DIMS }, + (_, index) => (digest[index % digest.length]! - 128) / 128, + ); +}; + describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () => { it('parks the crashed run WAL/shadow sidecars before reopening, then rebuilds clean', async () => { + const previousEnv = new Map(EMBEDDING_ENV_KEYS.map((key) => [key, process.env[key]] as const)); const repo = await setupMiniRepo(); try { const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); @@ -79,19 +100,36 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = await writeFile(`${lbugPath}.shadow`, shadowGarbage); const logs: string[] = []; - // embeddingsNodeLimit: 1 (KTD9): the recovery runs force:true - // internally, and the seeded stats would otherwise route Phase 4 into - // a real embedder in CI — the 1-node cap suppresses generation while - // leaving the preserve/restore path fully live. On linux the - // wipe-and-restore vector-index seam then fires for real (statically - // linked VECTOR): a CREATE_VECTOR_INDEX over the restored rows is - // expected and harmless here. + // Recovery runs force:true internally, so use the same deterministic + // in-process HTTP provider as the embedding canaries and admit the + // complete fixture with a real positive cap. A below-count cap now + // correctly refuses before recovery graph writes; it is no longer a + // supported way to suppress generation while continuing the rebuild. + process.env.GITNEXUS_EMBEDDING_URL = 'http://in-process.invalid/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'dirty-recovery-deterministic'; + process.env.GITNEXUS_EMBEDDING_DIMS = String(EMBEDDING_DIMS); + process.env.GITNEXUS_EMBEDDING_MAX_ATTEMPTS = '1'; + process.env.GITNEXUS_EMBEDDING_RETRY_CAP_MS = '1'; + process.env.GITNEXUS_EMBEDDING_MIN_INTERVAL_MS = '0'; + delete process.env.GITNEXUS_EMBEDDING_API_KEY; + const fetchMock = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? '{}')) as { input?: unknown }; + const inputs = Array.isArray(body.input) ? body.input.map(String) : []; + return new Response( + JSON.stringify({ + data: inputs.map((text) => ({ embedding: deterministicEmbedding(text) })), + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + }); + vi.stubGlobal('fetch', fetchMock); const recovered = await runFullAnalysis( repo.dbPath, - { skipAgentsMd: true, embeddingsNodeLimit: 1 }, + { skipAgentsMd: true, embeddingsNodeLimit: 1000 }, { onProgress: () => {}, onLog: (m) => logs.push(m) }, ); expect(recovered.alreadyUpToDate).toBeUndefined(); + expect(fetchMock).toHaveBeenCalled(); // Both sidecars were parked verbatim (renamed, never deleted) before // any open could replay them… @@ -109,17 +147,21 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = expect(joinedLogs).toContain( `--force on a repo with ${seededNodeIds.length} existing embeddings`, ); - // …with generation itself cap-suppressed (no embedder in CI): - expect(joinedLogs).toContain('exceeds the 1-node safety cap'); - // …and the rebuild completed into a clean index: dirty flag cleared, // and the seeded embeddings survived the park → open → wipe → restore // round-trip (the strongest signal the preservation open really ran: // the DB was wiped, so these rows can only come from the cache load). const after = await loadMeta(storagePath); expect(after!.incrementalInProgress).toBeUndefined(); - expect(after!.stats?.embeddings).toBe(seededNodeIds.length); + const recoveredNodeIds = await readEmbeddingNodeIds(repo.dbPath); + expect(seededNodeIds.every((nodeId) => recoveredNodeIds.includes(nodeId))).toBe(true); + expect(after!.stats?.embeddings).toBe(recoveredNodeIds.length); } finally { + vi.unstubAllGlobals(); + for (const [key, value] of previousEnv) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } await repo.cleanup(); } }, 300_000); From dc1ee545051510e6232f7a1d249724d70fa29637 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 08:39:38 +0700 Subject: [PATCH 5/9] fix(analysis): preserve cached vectors in implicit recovery --- Documentation/releases/1.6.10-electric.11.md | 26 +++++++++++---- GUARDRAILS.md | 4 +-- gitnexus/CHANGELOG.md | 15 +++++++-- gitnexus/src/core/embedding-mode.ts | 14 +++++++++ gitnexus/src/core/run-analyze.ts | 33 ++++++++++++++++---- gitnexus/test/unit/run-analyze.test.ts | 8 +++++ 6 files changed, 83 insertions(+), 17 deletions(-) diff --git a/Documentation/releases/1.6.10-electric.11.md b/Documentation/releases/1.6.10-electric.11.md index e83f0f1e28..0e1cce915d 100644 --- a/Documentation/releases/1.6.10-electric.11.md +++ b/Documentation/releases/1.6.10-electric.11.md @@ -1,8 +1,13 @@ # GitNexus 1.6.10-electric.11 `1.6.10-electric.11` carries the post-M8b release-preparation contract for bounded embedding -preservation and staged recovery. It packages the already-supported paths; it does not add a -new preservation schema, lock, service, or public preservation command surface. +preservation and staged recovery. It adds no preservation schema, service, or public +preservation command surface. + +Relative to `1.6.10-electric.10`, the artifact also includes the milestone 9 source aggregate: +shared canonical analyze ownership across supported writers, owner/generation compare-and-swap +protection for registry projections, recovery-safe implicit branch adoption, and additive Doctor +registry, freshness, identity, integrity, readiness, and strict exit-status reporting. ## Behavior contract @@ -12,6 +17,10 @@ new preservation schema, lock, service, or public preservation command surface. update, or staged promotion. Provider-free pipeline caches and prepared stage recovery files may already exist. When admitted, unaffected embedding owners retain their complete logical row payload bytes while changed or new owners are regenerated. +- Automatic `--force` recovery of an already-embedded local index may cross the implicit 50,000 + node cap. That exact implicit path restores cached vectors and continues in preserve-only mode; + explicit embedding requests and explicit positive caps still refuse the whole pass when + exceeded. - **Malformed/provenance-unproven copy:** run an explicit staged clean rebuild with a positive cap on total graph nodes: `gitnexus analyze --staged --embeddings --drop-embeddings`. The disposable staged @@ -23,13 +32,17 @@ new preservation schema, lock, service, or public preservation command surface. bytes are unchanged. - Rows inherited from older metadata without durable provider or identity proof remain unproven and are not called verified preservation. The clean staged rebuild replaces that - unclassified state rather than upgrading its claim. + unclassified state rather than upgrading its claim. Doctor's structural health result is not + provider provenance: incremental admission additionally requires an independent, recorded + provider/model/dimensions identity match. ## Safety and rollback boundary -- Staged recovery retains a canonical preimage and uses journaled promotion/rollback. The - M8b receipt proves exact logical embedding-row and metadata preimage restoration after an - injected promotion failure, with no staged, backup, or journal residue. +- Staged recovery uses a journal and a canonical database backup. The M8b receipt proves exact + logical embedding-row and metadata preimage restoration for its injected repository-identity + failure, with no staged, backup, or journal residue. Later promotion failures may deliberately + retain the new metadata, old database backup, and journal for forward recovery; do not treat + every failed promotion as an automatic exact rollback. - `1.6.10-electric.10` remains available as the immediate non-writing rollback artifact. - This source gate is local and disposable-copy evidence only. It does not prove the canonical 65-index estate, fleet, production, customer, browser, or runtime readiness. @@ -46,5 +59,6 @@ new preservation schema, lock, service, or public preservation command surface. - Exact post-M8b main: `50a0553c01fea92edff560f9e43df838235b3cf0`. - Milestone 10 closeout receipt: SHA-256 `0f6193fa5066745108a79acbeeccf485e4a52c7d08e0a5c67a6fa334ba0c94f8`. +- Receipt and retrieval record: [issue #197 milestone 10 acceptance](https://github.com/electricsheephq/evaOS-gitnexus/issues/197#issuecomment-5472113345). - The receipt is the source/disposable-copy gate carried into this release-prep PR; release, install, and runtime claims require their separate exact-head workflow evidence. diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 5f70d8aa64..426b9e78e4 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -48,8 +48,8 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Embedding state is malformed or provenance is unproven -- **Trigger:** Doctor or read-only integrity inspection reports malformed, duplicate, orphaned, wrong-dimension, or otherwise unproven embedding state, including legacy rows without durable provider or identity proof. -- **Do:** Preserve the canonical logical embedding-row and metadata preimage, then run an explicit staged clean rebuild with a positive cap on total graph nodes: `npx gitnexus analyze --staged --embeddings --drop-embeddings`. Validate the isolated generation before journaled promotion; on a promotion failure, use the staged rollback path and verify that the logical preimage is restored with no stage, backup, or journal residue. +- **Trigger:** Doctor or read-only integrity inspection reports malformed, duplicate, orphaned, wrong-dimension, or otherwise unproven embedding state, or the operator cannot independently bind the existing rows to the expected provider, model, and dimensions. Doctor structural health alone is not provider provenance. +- **Do:** Preserve the canonical logical embedding-row and metadata preimage, then run an explicit staged clean rebuild with a positive cap on total graph nodes: `npx gitnexus analyze --staged --embeddings --drop-embeddings`. Validate the isolated generation before journaled promotion. Follow the journal state after a failure: the injected milestone 10 boundary proved exact rollback, while later failures may retain the old database backup and new metadata for forward recovery. - **Why:** A staged clean rebuild gives malformed or unclassified derived state a bounded replacement path without treating old rows as verified. The deterministic proof covers logical row payloads, not raw LadybugDB container bytes. ### MCP lists no repos diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index bf66cfabee..65a4b8b277 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to GitNexus will be documented in this file. ### Changed +- **Milestone 9 writer and Doctor protections now ship in the artifact**: supported writers share + canonical analyze ownership; registry projections use owner/generation compare-and-swap and + recovery-safe implicit branch adoption; Doctor adds registry, freshness, identity, integrity, + readiness, and strict exit-status reporting. - **Healthy source-stale indexes keep their unchanged embedding payloads on the capped incremental path**: run `analyze` with a positive total-graph node cap (`--embeddings `). GitNexus refuses the whole embedding pass after the provider-free pipeline but before graph @@ -18,10 +22,15 @@ All notable changes to GitNexus will be documented in this file. same bounded snapshot contract. - **Malformed or provenance-unproven indexes use an explicit capped staged clean rebuild**: `--staged --embeddings --drop-embeddings` builds an isolated generation, keeps the - canonical generation untouched until validation and promotion, and retains the canonical - logical embedding-row and metadata preimage for journaled rollback on a promotion failure. + canonical generation untouched until validation and promotion. The journal and canonical + database backup support rollback or forward recovery; only the injected milestone 10 boundary + proved exact logical metadata-and-row restoration, not every later promotion failure. - Rows from older metadata that lack durable provider or identity proof remain unclassified; - they are not described as verified preservation. + they are not described as verified preservation. Doctor structural health is not provider + provenance; incremental admission requires separately recorded provider/model/dimensions proof. +- Automatic `--force` recovery above the implicit local 50,000-node cap continues in + preserve-only mode with cached vectors. Explicit embedding requests and explicit positive caps + remain all-or-nothing refusal gates. ### Release boundary diff --git a/gitnexus/src/core/embedding-mode.ts b/gitnexus/src/core/embedding-mode.ts index fb8168ddd4..594153faed 100644 --- a/gitnexus/src/core/embedding-mode.ts +++ b/gitnexus/src/core/embedding-mode.ts @@ -76,6 +76,20 @@ export function deriveEmbeddingCap( return { skipForCap, capDisabled, nodeLimit }; } +/** + * The implicit local-model cap is allowed to downgrade only the automatic + * `--force` top-up of an already-embedded repository to preserve-only mode. + * Explicit embedding requests and explicit positive caps remain all-or- + * nothing admission gates. + */ +export function preserveOnlyForImplicitForceCap( + skipForCap: boolean, + forceRegenerateEmbeddings: boolean, + embeddingsNodeLimit: number | undefined, +): boolean { + return skipForCap && forceRegenerateEmbeddings && embeddingsNodeLimit === undefined; +} + export function deriveEmbeddingMode( options: EmbeddingModeInput, existingEmbeddingCount: number, diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index fb685c6ad9..4d8cfc0482 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -545,6 +545,7 @@ export type { EmbeddingMode } from './embedding-mode.js'; import { deriveEmbeddingMode as _deriveEmbeddingMode, deriveEmbeddingCap, + preserveOnlyForImplicitForceCap, resolveEmbeddingNodeLimit, DEFAULT_EMBEDDING_NODE_LIMIT, } from './embedding-mode.js'; @@ -2382,9 +2383,12 @@ const runFullAnalysisImpl = async ( : 0; const streamedPdgRows = streamedPdgNodeRows + streamedPdgRelationshipRows; - // An explicit positive embedding cap is an all-or-nothing admission gate, - // not a request to finish the graph write without semantic rows. Settle it - // immediately after the provider-free pipeline has produced the complete + // Explicit embedding requests and positive caps are all-or-nothing + // admission gates, not requests to finish the graph write without semantic + // rows. The sole exception is automatic force recovery above the implicit + // local cap, which may restore cached vectors in preserve-only mode. Settle + // the decision immediately after the provider-free pipeline has produced + // the complete // prospective node count and before any LadybugDB, metadata, registry, or // staged-promotion write begins. Pipeline cache reconciliation and prepared // staging files may already exist; the refusal deliberately preserves them @@ -2401,7 +2405,12 @@ const runFullAnalysisImpl = async ( resolveEmbeddingNodeLimit(options.embeddingsNodeLimit, resumeEmbeddingCheckpoint), httpMode, ); - if (skipForCap) { + const preserveOnlyForRecovery = preserveOnlyForImplicitForceCap( + skipForCap, + forceRegenerateEmbeddings, + options.embeddingsNodeLimit, + ); + if (skipForCap && !preserveOnlyForRecovery) { throw new Error( `Embedding generation refused: ${embeddingAdmissionNodeCount.toLocaleString()} nodes exceeds ` + `the ${nodeLimit.toLocaleString()}-node safety cap. ` + @@ -2410,8 +2419,20 @@ const runFullAnalysisImpl = async ( ); } - embeddingSkipped = false; - if (capDisabled && embeddingAdmissionNodeCount > DEFAULT_EMBEDDING_NODE_LIMIT) { + if (preserveOnlyForRecovery) { + log( + `Automatic embedding regeneration exceeds the implicit ${nodeLimit.toLocaleString()}-node ` + + 'local-model safety cap; continuing recovery in preserve-only mode with cached vectors. ' + + 'Use an explicit positive `--embeddings ` cap to admit regeneration.', + ); + } else { + embeddingSkipped = false; + } + if ( + !preserveOnlyForRecovery && + capDisabled && + embeddingAdmissionNodeCount > DEFAULT_EMBEDDING_NODE_LIMIT + ) { if (httpMode) { log( `Remote embedding endpoint selected — generating embeddings for ` + diff --git a/gitnexus/test/unit/run-analyze.test.ts b/gitnexus/test/unit/run-analyze.test.ts index 9c58007946..b53dde0201 100644 --- a/gitnexus/test/unit/run-analyze.test.ts +++ b/gitnexus/test/unit/run-analyze.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect, vi } from 'vitest'; import { deriveEmbeddingMode, deriveEmbeddingCap, + preserveOnlyForImplicitForceCap, resolveEmbeddingNodeLimit, DEFAULT_EMBEDDING_NODE_LIMIT, } from '../../src/core/embedding-mode.js'; @@ -1190,6 +1191,13 @@ describe('deriveEmbeddingCap', () => { it('custom cap below default still applies', () => { expect(deriveEmbeddingCap(15_000, 10_000).skipForCap).toBe(true); }); + + it('downgrades only an implicit force top-up to preserve-only recovery', () => { + expect(preserveOnlyForImplicitForceCap(true, true, undefined)).toBe(true); + expect(preserveOnlyForImplicitForceCap(true, true, 50_000)).toBe(false); + expect(preserveOnlyForImplicitForceCap(true, false, undefined)).toBe(false); + expect(preserveOnlyForImplicitForceCap(false, true, undefined)).toBe(false); + }); }); describe('pdgModeMismatch / resolvePdgConfig (#2099 F1)', () => { From 8230100022a90f3905b43b18f29aa859a513c823 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 08:50:59 +0700 Subject: [PATCH 6/9] test(analysis): prove cached recovery row fingerprints --- .../unit/incremental-dirty-recovery.test.ts | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/gitnexus/test/unit/incremental-dirty-recovery.test.ts b/gitnexus/test/unit/incremental-dirty-recovery.test.ts index 521b26813a..34d6f4fc93 100644 --- a/gitnexus/test/unit/incremental-dirty-recovery.test.ts +++ b/gitnexus/test/unit/incremental-dirty-recovery.test.ts @@ -32,7 +32,7 @@ import { setupMiniRepo as setupSharedMiniRepo } from '../helpers/mini-repo.js'; // zero-vector seeding pattern previously lived here as a divergent copy of // incremental-orchestration.test.ts's (a helper module has no // describe-registration problem, unlike importing a sibling test file). -import { readEmbeddingNodeIds, seedEmbeddingsForFiles } from '../helpers/embedding-seed.js'; +import { readEmbeddingRowFingerprints, seedEmbeddingsForFiles } from '../helpers/embedding-seed.js'; const setupMiniRepo = () => setupSharedMiniRepo('gitnexus-incr-dirty-rec-'); @@ -55,6 +55,50 @@ const deterministicEmbedding = (text: string): number[] => { ); }; +const stampCurrentContentHashes = async (repoPath: string, nodeIds: readonly string[]) => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const { contentHashForNode } = await import('../../src/core/embeddings/embedding-pipeline.js'); + const { escapeCypherString } = await import('../../src/core/lbug/cypher-escape.js'); + const { lbugPath } = getStoragePaths(repoPath); + await adapter.initLbug(lbugPath); + try { + const wanted = `[${nodeIds.map((id) => `'${escapeCypherString(id)}'`).join(', ')}]`; + const rows = (await adapter.executeQuery( + `MATCH (n:Function) WHERE n.id IN ${wanted} RETURN n.id AS id, n.name AS name, ` + + `'Function' AS label, n.filePath AS filePath, n.content AS content, ` + + 'n.startLine AS startLine, n.endLine AS endLine, n.isExported AS isExported, ' + + 'n.description AS description', + )) as Array>; + expect(rows).toHaveLength(nodeIds.length); + for (const row of rows) { + const nodeId = String(row.id ?? ''); + const contentHash = contentHashForNode({ + id: nodeId, + name: String(row.name ?? ''), + label: 'Function', + filePath: String(row.filePath ?? ''), + content: String(row.content ?? ''), + startLine: Number(row.startLine ?? 0), + endLine: Number(row.endLine ?? 0), + isExported: + row.isExported === undefined || row.isExported === null + ? undefined + : Boolean(row.isExported), + description: + row.description === undefined || row.description === null + ? undefined + : String(row.description), + }); + await adapter.executeQuery( + `MATCH (e:CodeEmbedding) WHERE e.nodeId = '${escapeCypherString(nodeId)}' ` + + `SET e.contentHash = '${escapeCypherString(contentHash)}'`, + ); + } + } finally { + await adapter.closeLbug(); + } +}; + describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () => { it('parks the crashed run WAL/shadow sidecars before reopening, then rebuilds clean', async () => { const previousEnv = new Map(EMBEDDING_ENV_KEYS.map((key) => [key, process.env[key]] as const)); @@ -78,6 +122,12 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = ); const seededNodeIds = [...seededIdsByFile.values()].flat(); expect(seededNodeIds.length).toBeGreaterThan(0); + await stampCurrentContentHashes(repo.dbPath, seededNodeIds); + const seededNodeIdSet = new Set(seededNodeIds); + const seededFingerprintsBefore = (await readEmbeddingRowFingerprints(repo.dbPath)).filter( + (row) => seededNodeIdSet.has(row.nodeId), + ); + expect(seededFingerprintsBefore).toHaveLength(seededNodeIds.length); // Simulate a crashed incremental writeback: dirty flag in meta plus // leftover sidecars whose bytes must never be replayed. 8KB puts the @@ -153,9 +203,11 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = // the DB was wiped, so these rows can only come from the cache load). const after = await loadMeta(storagePath); expect(after!.incrementalInProgress).toBeUndefined(); - const recoveredNodeIds = await readEmbeddingNodeIds(repo.dbPath); - expect(seededNodeIds.every((nodeId) => recoveredNodeIds.includes(nodeId))).toBe(true); - expect(after!.stats?.embeddings).toBe(recoveredNodeIds.length); + const recoveredFingerprints = await readEmbeddingRowFingerprints(repo.dbPath); + expect(recoveredFingerprints.filter((row) => seededNodeIdSet.has(row.nodeId))).toEqual( + seededFingerprintsBefore, + ); + expect(after!.stats?.embeddings).toBe(recoveredFingerprints.length); } finally { vi.unstubAllGlobals(); for (const [key, value] of previousEnv) { From 1fe8d1ef552b31b33a7954ce2ebbeabc5c5bbbf7 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 09:02:56 +0700 Subject: [PATCH 7/9] fix: clean streamed PDG files on cap refusal --- gitnexus/src/core/lbug/pdg-emit-sink.ts | 33 +++++++++++++++++++ gitnexus/src/core/run-analyze.ts | 2 ++ gitnexus/test/unit/lbug/pdg-emit-sink.test.ts | 22 ++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/gitnexus/src/core/lbug/pdg-emit-sink.ts b/gitnexus/src/core/lbug/pdg-emit-sink.ts index 79cc05f585..b716bad3c7 100644 --- a/gitnexus/src/core/lbug/pdg-emit-sink.ts +++ b/gitnexus/src/core/lbug/pdg-emit-sink.ts @@ -184,6 +184,39 @@ export interface PdgEmitManifest { readonly relsByPair: Map; } +/** + * Remove only the temporary CSV files named by a completed PDG emit manifest. + * Cap admission happens after the provider-free pipeline so a refused run can + * otherwise leave a large streamed PDG layer behind. The manifest files are + * not recovery artifacts: a retry always re-emits them. Cleanup is best-effort + * because the original admission refusal remains the primary error. + */ +export function cleanupPdgEmitManifestFiles(manifest: PdgEmitManifest | undefined): void { + if (!manifest) return; + + const directories = new Set(); + const files = new Set(); + for (const { csvPath } of manifest.nodeFiles.values()) files.add(csvPath); + for (const { csvPath } of manifest.relsByPair.values()) files.add(csvPath); + + for (const csvPath of files) { + directories.add(path.dirname(csvPath)); + try { + fs.unlinkSync(csvPath); + } catch { + // Best-effort: preserve the cap-refusal error and never widen deletion + // beyond the exact manifest paths. + } + } + for (const directory of directories) { + try { + fs.rmdirSync(directory); + } catch { + // A non-empty directory may contain unrelated data; leave it intact. + } + } +} + /** * Write-routing graph façade. Construct one per analyze run, thread it into the * per-language `runScopeResolution` calls in place of the real graph during the diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 4d8cfc0482..bde18aa8b5 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -45,6 +45,7 @@ import { type EmbeddingIntegrityReport, } from './lbug/lbug-adapter.js'; import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js'; +import { cleanupPdgEmitManifestFiles } from './lbug/pdg-emit-sink.js'; import { escapeCypherString } from './lbug/cypher-escape.js'; import { createSearchFTSIndexes, @@ -2411,6 +2412,7 @@ const runFullAnalysisImpl = async ( options.embeddingsNodeLimit, ); if (skipForCap && !preserveOnlyForRecovery) { + cleanupPdgEmitManifestFiles(pipelineResult.pdgEmitManifest); throw new Error( `Embedding generation refused: ${embeddingAdmissionNodeCount.toLocaleString()} nodes exceeds ` + `the ${nodeLimit.toLocaleString()}-node safety cap. ` + diff --git a/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts b/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts index 7d4699529e..d3f85c9336 100644 --- a/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts +++ b/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts @@ -17,7 +17,7 @@ import path from 'node:path'; import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { streamAllCSVsToDisk, buildBasicBlockRow } from '../../../src/core/lbug/csv-generator.js'; -import { PdgEmitSink } from '../../../src/core/lbug/pdg-emit-sink.js'; +import { cleanupPdgEmitManifestFiles, PdgEmitSink } from '../../../src/core/lbug/pdg-emit-sink.js'; import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; const bbNode = (fp: string, idx: number, line: number): GraphNode => ({ @@ -61,6 +61,26 @@ afterEach(() => { }); describe('PdgEmitSink — routing', () => { + it('removes only exact manifest files after a pre-COPY refusal', () => { + const pdgDir = path.join(tmpRoot, 'pdg-csv'); + const real = createKnowledgeGraph(); + const sink = new PdgEmitSink(real, pdgDir); + sink.addNode(bbNode('a.ts', 0, 1)); + sink.addNode(bbNode('a.ts', 1, 5)); + sink.addRelationship(pdgEdge('a.ts', 0, 1, 'CFG', 'seq')); + const manifest = sink.finalize(); + const unrelated = path.join(pdgDir, 'unrelated.keep'); + fs.writeFileSync(unrelated, 'keep'); + + cleanupPdgEmitManifestFiles(manifest); + + for (const { csvPath } of manifest.nodeFiles.values()) + expect(fs.existsSync(csvPath)).toBe(false); + for (const { csvPath } of manifest.relsByPair.values()) + expect(fs.existsSync(csvPath)).toBe(false); + expect(fs.readFileSync(unrelated, 'utf8')).toBe('keep'); + }); + it('routes BasicBlock nodes and PDG edges to CSV, never to the real graph', () => { const real = createKnowledgeGraph(); const sink = new PdgEmitSink(real, path.join(tmpRoot, 'pdg-csv')); From 3e2afe823af8ea255efe60882c62f4bcb0b4739d Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 09:11:35 +0700 Subject: [PATCH 8/9] fix: distinguish explicit embedding requests --- gitnexus/src/core/embedding-mode.ts | 8 +++++++- gitnexus/src/core/run-analyze.ts | 1 + gitnexus/test/unit/run-analyze-fts-repair.test.ts | 7 ++++++- gitnexus/test/unit/run-analyze.test.ts | 9 +++++---- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/gitnexus/src/core/embedding-mode.ts b/gitnexus/src/core/embedding-mode.ts index 594153faed..91f60655ac 100644 --- a/gitnexus/src/core/embedding-mode.ts +++ b/gitnexus/src/core/embedding-mode.ts @@ -86,8 +86,14 @@ export function preserveOnlyForImplicitForceCap( skipForCap: boolean, forceRegenerateEmbeddings: boolean, embeddingsNodeLimit: number | undefined, + explicitEmbeddingRequest: boolean, ): boolean { - return skipForCap && forceRegenerateEmbeddings && embeddingsNodeLimit === undefined; + return ( + skipForCap && + forceRegenerateEmbeddings && + embeddingsNodeLimit === undefined && + !explicitEmbeddingRequest + ); } export function deriveEmbeddingMode( diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index bde18aa8b5..e801b6354c 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -2410,6 +2410,7 @@ const runFullAnalysisImpl = async ( skipForCap, forceRegenerateEmbeddings, options.embeddingsNodeLimit, + options.embeddings === true, ); if (skipForCap && !preserveOnlyForRecovery) { cleanupPdgEmitManifestFiles(pipelineResult.pdgEmitManifest); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 83f1704dec..826edfa28d 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -731,6 +731,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 const wipeLbugDbFiles = vi.fn(async () => undefined); const buildVectorIndex = vi.fn(async () => false); const executeWithReusedStatement = vi.fn(async () => []); + let pdgCsvPath = ''; vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ initLbug, loadGraphToLbug, @@ -791,7 +792,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 repoPath, totalFileCount: 1, pdgEmitManifest: { - nodeFiles: new Map([['chunk-0', { rows: 1 }]]), + nodeFiles: new Map([['chunk-0', { csvPath: pdgCsvPath, rows: 1 }]]), relsByPair: new Map(), }, graph: { @@ -820,6 +821,9 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 try { const { storagePath } = getStoragePaths(tmpRepo.dbPath); await fs.mkdir(storagePath, { recursive: true }); + pdgCsvPath = `${storagePath}/pdg-csv/basicblock.csv`; + await fs.mkdir(`${storagePath}/pdg-csv`, { recursive: true }); + await fs.writeFile(pdgCsvPath, 'id:ID(BasicBlock)\n'); const originalMeta: RepoMeta = { repoPath: tmpRepo.dbPath, lastCommit: '', @@ -844,6 +848,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 expect(wipeLbugDbFiles).not.toHaveBeenCalled(); expect(buildVectorIndex).not.toHaveBeenCalled(); expect(executeWithReusedStatement).not.toHaveBeenCalled(); + await expect(fs.access(pdgCsvPath)).rejects.toMatchObject({ code: 'ENOENT' }); const meta = JSON.parse(await fs.readFile(`${storagePath}/meta.json`, 'utf-8')) as RepoMeta; expect(meta).toEqual(originalMeta); } finally { diff --git a/gitnexus/test/unit/run-analyze.test.ts b/gitnexus/test/unit/run-analyze.test.ts index b53dde0201..36ccf59097 100644 --- a/gitnexus/test/unit/run-analyze.test.ts +++ b/gitnexus/test/unit/run-analyze.test.ts @@ -1193,10 +1193,11 @@ describe('deriveEmbeddingCap', () => { }); it('downgrades only an implicit force top-up to preserve-only recovery', () => { - expect(preserveOnlyForImplicitForceCap(true, true, undefined)).toBe(true); - expect(preserveOnlyForImplicitForceCap(true, true, 50_000)).toBe(false); - expect(preserveOnlyForImplicitForceCap(true, false, undefined)).toBe(false); - expect(preserveOnlyForImplicitForceCap(false, true, undefined)).toBe(false); + expect(preserveOnlyForImplicitForceCap(true, true, undefined, false)).toBe(true); + expect(preserveOnlyForImplicitForceCap(true, true, undefined, true)).toBe(false); + expect(preserveOnlyForImplicitForceCap(true, true, 50_000, false)).toBe(false); + expect(preserveOnlyForImplicitForceCap(true, false, undefined, false)).toBe(false); + expect(preserveOnlyForImplicitForceCap(false, true, undefined, false)).toBe(false); }); }); From 2e25826d9d09bd4e8898c631b276e12c5424febf Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 31 Aug 2026 09:33:29 +0700 Subject: [PATCH 9/9] docs: qualify staged promotion recovery --- ARCHITECTURE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2587175a03..ec1e97ac3b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -390,8 +390,10 @@ embedding node cap, `0` disables it), `--staged` (isolated generation plus valid contract remains the same. - For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild with a positive cap on total graph nodes (`--staged --embeddings --drop-embeddings`). The staged database is disposable and the - canonical generation stays readable until validation and journaled promotion complete. A promotion - failure restores the canonical logical embedding-row and metadata preimage. + canonical generation stays readable until validation and journaled promotion complete. The injected + milestone-10 pre-promotion failure restores the exact canonical logical embedding-row and metadata + preimage. A later promotion failure may retain the new metadata, old database backup, and journal so + the next analyze can complete the recorded promotion forward. - The deterministic M8b proof compares every unaffected embedding-row payload field (identity, owner, chunk, line range, content hash, and vector). It proves logical payload-byte preservation on that supported path, not unchanged raw LadybugDB container bytes.