diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index a2ac6fa595..39e1944b4a 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "source": { "source": "local", "path": "./gitnexus-claude-plugin" diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 0592aee5bf..4dff9ff269 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "source": "./gitnexus-claude-plugin", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase." } diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4390cae8ce..ec1e97ac3b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -362,19 +362,43 @@ Inheritance is captured by the `@reference.inherits` tag and emitted by the scop ``` CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) - 1. Early exit if lastCommit == HEAD (unless --force) [0%] - 2. Cache existing embeddings from prior index [0%] - 3. runPipelineFromRepo() → KnowledgeGraph [0-60%] - 4. Clean up legacy KuzuDB files [60%] - 5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%] - 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] - 7. Restore cached embeddings (batch insert) [88%] - 8. Generate new embeddings if --embeddings [90-98%] - 9. Save metadata + register repo + update .gitignore [98-100%] - 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] + 1. Early exit if lastCommit == HEAD (unless --force) [0%] + 2. Read metadata and select surgical incremental, full, or staged [0%] + 3. Capture a bounded embedding snapshot when a rebuild needs restore [0%] + 4. runPipelineFromRepo() → KnowledgeGraph [0-60%] + 5. Write changed rows surgically, or wipe + bulk-COPY a full graph [60-85%] + 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] + 7. Restore cached embeddings in bounded batches [88%] + 8. Generate new embeddings within the requested cap [90-98%] + 9. Validate, save metadata, register repo, and update .gitignore [98-100%] + 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] ``` -**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`. +**Options:** `--force` (rebuild regardless), `--embeddings [n]` (opt-in; a positive `n` is the +embedding node cap, `0` disables it), `--staged` (isolated generation plus validated promotion), +`--skipGit`, `--noStats`. + +### Bounded embedding preservation and staged recovery + +- For a healthy index that is stale against its source, a positive-cap incremental run (`--embeddings `) + first admits or refuses the entire embedding pass against the provider-free pipeline's total graph + node count, before graph persistence, final metadata, registry update, or staged promotion. Transient + pipeline caches and prepared stage recovery files may already exist. When admitted, + it rewrites only the changed/effective file set and regenerates changed or new embedding owners. + Unaffected owners retain their complete logical embedding-row payload. If the effective write set is + large, the existing escalation valve selects the full-write plan; the bounded snapshot and restore + contract remains the same. +- For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild with a + positive cap on total graph nodes (`--staged --embeddings --drop-embeddings`). The staged database is disposable and the + canonical generation stays readable until validation and journaled promotion complete. The injected + milestone-10 pre-promotion failure restores the exact canonical logical embedding-row and metadata + preimage. A later promotion failure may retain the new metadata, old database backup, and journal so + the next analyze can complete the recorded promotion forward. +- The deterministic M8b proof compares every unaffected embedding-row payload field (identity, owner, + chunk, line range, content hash, and vector). It proves logical payload-byte preservation on that + supported path, not unchanged raw LadybugDB container bytes. +- Legacy rows whose metadata lacks durable provider or identity proof remain unproven and are not + described as verified preservation. A clean staged rebuild replaces that unclassified state. ## Storage @@ -385,6 +409,8 @@ CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) ├── lbug.shadow # Shadow sidecar (checkpoint staging) ├── lbug.lock # Single-writer lock ├── lbug.{wal,shadow}.dirty-recovery # parked sidecars from a crashed run; safe to delete + ├── embedding-preservation.jsonl # transient checksummed restore snapshot + ├── embedding-table-rebuild.json # transient staged-table recovery marker ├── gitnexus.json # lastCommit, indexedAt, stats (primary metadata file) └── meta.json # legacy mirror of gitnexus.json, kept in sync (see MIGRATION.md) diff --git a/Documentation/releases/1.6.10-electric.11.md b/Documentation/releases/1.6.10-electric.11.md new file mode 100644 index 0000000000..0e1cce915d --- /dev/null +++ b/Documentation/releases/1.6.10-electric.11.md @@ -0,0 +1,64 @@ +# GitNexus 1.6.10-electric.11 + +`1.6.10-electric.11` carries the post-M8b release-preparation contract for bounded embedding +preservation and staged recovery. It adds no preservation schema, service, or public +preservation command surface. + +Relative to `1.6.10-electric.10`, the artifact also includes the milestone 9 source aggregate: +shared canonical analyze ownership across supported writers, owner/generation compare-and-swap +protection for registry projections, recovery-safe implicit branch adoption, and additive Doctor +registry, freshness, identity, integrity, readiness, and strict exit-status reporting. + +## Behavior contract + +- **Healthy/source-stale copy:** run incremental analysis with a positive total-graph node cap, + for example `gitnexus analyze --embeddings `. GitNexus refuses the whole embedding pass + when the graph has more than `` nodes, before graph persistence, final metadata, registry + update, or staged promotion. Provider-free pipeline caches and prepared stage recovery files may + already exist. When admitted, unaffected embedding owners retain their complete logical row + payload bytes while changed or new owners are regenerated. +- Automatic `--force` recovery of an already-embedded local index may cross the implicit 50,000 + node cap. That exact implicit path restores cached vectors and continues in preserve-only mode; + explicit embedding requests and explicit positive caps still refuse the whole pass when + exceeded. +- **Malformed/provenance-unproven copy:** run an explicit staged clean rebuild with a positive cap + on total graph nodes: + `gitnexus analyze --staged --embeddings --drop-embeddings`. The disposable staged + generation is validated before promotion; the canonical generation remains readable and + unchanged until that boundary. +- The M8b disposable canary deterministically compared the complete unaffected embedding-row + payload (identity, owner, chunk, line range, content hash, and vector). This proves logical + row-payload preservation for that supported path; it does not claim raw LadybugDB container + bytes are unchanged. +- Rows inherited from older metadata without durable provider or identity proof remain + unproven and are not called verified preservation. The clean staged rebuild replaces that + unclassified state rather than upgrading its claim. Doctor's structural health result is not + provider provenance: incremental admission additionally requires an independent, recorded + provider/model/dimensions identity match. + +## Safety and rollback boundary + +- Staged recovery uses a journal and a canonical database backup. The M8b receipt proves exact + logical embedding-row and metadata preimage restoration for its injected repository-identity + failure, with no staged, backup, or journal residue. Later promotion failures may deliberately + retain the new metadata, old database backup, and journal for forward recovery; do not treat + every failed promotion as an automatic exact rollback. +- `1.6.10-electric.10` remains available as the immediate non-writing rollback artifact. +- This source gate is local and disposable-copy evidence only. It does not prove the canonical + 65-index estate, fleet, production, customer, browser, or runtime readiness. + +## Distribution boundary + +- Electric releases are GitHub Release assets only. `.11` is not published to npm, a container + registry, or any other package registry, and no floating tag is used. +- This release-prep change does not install, dispatch the release workflow, mutate a registry or + live index, or perform a runtime/customer rollout. + +## Source-gate evidence + +- Exact post-M8b main: `50a0553c01fea92edff560f9e43df838235b3cf0`. +- Milestone 10 closeout receipt: SHA-256 + `0f6193fa5066745108a79acbeeccf485e4a52c7d08e0a5c67a6fa334ba0c94f8`. +- Receipt and retrieval record: [issue #197 milestone 10 acceptance](https://github.com/electricsheephq/evaOS-gitnexus/issues/197#issuecomment-5472113345). +- The receipt is the source/disposable-copy gate carried into this release-prep PR; release, + install, and runtime claims require their separate exact-head workflow evidence. diff --git a/GUARDRAILS.md b/GUARDRAILS.md index a6297f4f43..426b9e78e4 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -20,6 +20,7 @@ Maintainer may widen scope per task. 3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. 4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. 5. **Preserve embeddings** — plain `npx gitnexus analyze` now preserves any embeddings recorded in the index metadata (`.gitnexus/gitnexus.json`, mirrored to the legacy `meta.json`) — the previous behavior wiped them. Use `--embeddings` to also generate vectors for new/changed nodes; use `--drop-embeddings` only when an explicit wipe is intended (e.g., model swap). +6. **Use the bounded recovery contract** — for healthy indexes that are stale against source, use a positive cap on total graph nodes (`--embeddings `). The cap admits or refuses the whole embedding pass after the provider-free pipeline but before graph persistence, final metadata, registry update, or staged promotion; transient caches or prepared stage recovery files may already exist. When admitted, the incremental path retains unaffected logical embedding-row payloads and regenerates only changed/new owners. For malformed or provenance-unproven state, use an explicit staged clean rebuild with the same positive total-graph cap (`--staged --embeddings --drop-embeddings`) and keep the canonical generation as the rollback preimage until validated promotion completes. Older rows without durable provider or identity proof remain unproven; never call them verified preservation. --- @@ -45,6 +46,12 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Do:** Re-run `npx gitnexus analyze --embeddings` to regenerate. Check the analyze log for a `Warning: could not load cached embeddings` line — if present, the cache restore failed (corrupt DB / schema mismatch) and the rebuild had nothing to preserve. If you intentionally passed `--drop-embeddings`, this is expected. - **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; the only ways to end up at zero are an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. +### Embedding state is malformed or provenance is unproven + +- **Trigger:** Doctor or read-only integrity inspection reports malformed, duplicate, orphaned, wrong-dimension, or otherwise unproven embedding state, or the operator cannot independently bind the existing rows to the expected provider, model, and dimensions. Doctor structural health alone is not provider provenance. +- **Do:** Preserve the canonical logical embedding-row and metadata preimage, then run an explicit staged clean rebuild with a positive cap on total graph nodes: `npx gitnexus analyze --staged --embeddings --drop-embeddings`. Validate the isolated generation before journaled promotion. Follow the journal state after a failure: the injected milestone 10 boundary proved exact rollback, while later failures may retain the old database backup and new metadata for forward recovery. +- **Why:** A staged clean rebuild gives malformed or unclassified derived state a bounded replacement path without treating old rows as verified. The deterministic proof covers logical row payloads, not raw LadybugDB container bytes. + ### MCP lists no repos - **Trigger:** MCP stderr says no indexed repos. @@ -74,6 +81,7 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ## Publishing & supply chain - **npm:** Do not publish from unreviewed automation. Bump version intentionally; tag releases to match `package.json`. +- **Electric releases:** Distribution is GitHub Release assets only. Do not publish an Electric version to npm, a container registry, or another package registry, and do not use a floating tag. Keep `1.6.10-electric.10` available as the immediate non-writing rollback artifact while `.11` release/install/runtime work is proved separately. - **Dependencies:** Minimal, auditable `package.json` changes; run tests and CI after lockfile updates. - **License:** PolyForm Noncommercial 1.0.0 — do not relicense without maintainer approval. diff --git a/MIGRATION.md b/MIGRATION.md index f6af6c6a72..4141d7c44c 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -110,3 +110,25 @@ repo as never analyzed. The `meta.json` mirror will remain until a future major version. Removal will be announced in this file and in the changelog before it happens. + +## Bounded embedding preservation and staged recovery (`1.6.10-electric.11`) + +There is no embedding-schema migration. The release documents two existing paths: + +- A healthy index that is stale against its source can use an incremental run with a positive cap + on total graph nodes, + run, `gitnexus analyze --embeddings `. The pass is refused when the graph has more than + `` nodes. Refusal happens before graph persistence, final metadata, registry update, or staged + promotion, although transient pipeline caches or prepared stage recovery files may already exist. + When admitted, unaffected logical embedding-row payloads remain intact and changed or new owners + are regenerated. +- Malformed or provenance-unproven embedding state uses an explicit staged clean rebuild with the + same positive total-graph cap, `gitnexus analyze --staged --embeddings --drop-embeddings`. The staged generation is + validated before promotion, and the canonical logical embedding-row and metadata preimage stays + available for journaled rollback. + +Rows inherited from metadata without durable provider or identity proof remain unproven. They are +not called verified preservation; the clean staged rebuild is the replacement path for that state. +The deterministic proof covers logical row payloads, not raw LadybugDB container bytes. Existing +`1.6.10-electric.10` installation bytes remain suitable for non-writing rollback while `.11` +release and runtime steps are handled separately. diff --git a/RUNBOOK.md b/RUNBOOK.md index f5ecd0d91c..0e157e8fd7 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -56,7 +56,11 @@ npx gitnexus list npx gitnexus analyze --embeddings ``` -**Important:** If you already had embeddings, **always** pass `--embeddings` on later analyzes, or they can be dropped. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none). +**Important:** A plain `analyze` preserves existing embeddings. Pass `--embeddings ` when +you also want changed or new owners regenerated under a positive node cap; use +`--drop-embeddings` only when intentionally replacing the existing vectors. See +`stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means +none). **Large repos:** Analyze may skip or limit embedding work when node counts are very high; watch CLI output. @@ -164,6 +168,35 @@ If the error text is `"Only one write transaction at a time is allowed in the sy - Agent safety rules: [GUARDRAILS.md](GUARDRAILS.md) - Tests: [TESTING.md](TESTING.md) +## Bounded embedding preservation and staged recovery + +For a healthy index that is stale against its source, run incremental analysis with a positive +total-graph node cap: + +```bash +npx gitnexus analyze --embeddings +``` + +GitNexus refuses the whole embedding pass when the provider-free pipeline reports more than `` +nodes. Refusal happens before graph persistence, final metadata, registry update, or staged +promotion; transient pipeline caches or prepared stage recovery files may already exist. When +admitted, the incremental path rewrites the changed/effective file set, regenerates changed or new +embedding owners, and retains unaffected logical embedding-row payloads. If the effective write set +is large, the existing escalation valve may select a full write while retaining the same bounded +snapshot and restore contract. + +For malformed or provenance-unproven embedding state, use an explicit staged clean rebuild: + +```bash +npx gitnexus analyze --staged --embeddings --drop-embeddings +``` + +The staged generation is disposable and the canonical generation remains readable until +validation and journaled promotion complete. Keep the canonical logical embedding-row and +metadata preimage available for rollback. The deterministic proof covers logical row payloads, +not raw LadybugDB container bytes; legacy rows without durable provider or identity proof remain +unproven and are not called verified preservation. + --- ## Electric fork releases @@ -173,8 +206,8 @@ to npm, GitHub Container Registry, Docker Hub, or another package registry. Prepare each version through a reviewed release PR: -1. Set `gitnexus/package.json` and `gitnexus/package-lock.json` to a version such - as `1.6.10-electric.1`. +1. Set `gitnexus/package.json` and `gitnexus/package-lock.json` to the reviewed + fork version, such as `1.6.10-electric.11`. 2. Run `node gitnexus/scripts/sync-plugin-versions.mjs` from the repository root. 3. Add `Documentation/releases/.md` and update `gitnexus/CHANGELOG.md`. 4. Run `npm run check:electric-release-policy`, the plugin-version test, build, @@ -187,7 +220,7 @@ Create the release from the current `main` branch with the protected workflow: gh workflow run electric-release.yml \ --repo electricsheephq/evaOS-gitnexus \ --ref main \ - -f expected_version=1.6.10-electric.1 \ + -f expected_version=1.6.10-electric.11 \ -f prerelease=false ``` @@ -219,10 +252,11 @@ After downloading both assets, verify and install locally: ```bash shasum -a 256 --check SHA256SUMS -npm install --global ./gitnexus-1.6.10-electric.1.tgz +npm install --global ./gitnexus-1.6.10-electric.11.tgz gitnexus --version ``` A GitHub Release does not authorize an OpenClaw/evaOS rollout. Keep the prior -installation available for rollback, switch runtime source separately, and do -not rebuild or migrate a live index unless that rollout explicitly requires it. +`1.6.10-electric.10` installation available as non-writing rollback bytes, switch +runtime source separately, and do not rebuild or migrate a live index unless that +rollout explicitly requires it. diff --git a/gitnexus-claude-plugin/.claude-plugin/plugin.json b/gitnexus-claude-plugin/.claude-plugin/plugin.json index caac21b88f..2d6b067dda 100644 --- a/gitnexus-claude-plugin/.claude-plugin/plugin.json +++ b/gitnexus-claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "author": { "name": "GitNexus" }, diff --git a/gitnexus-claude-plugin/.codex-plugin/plugin.json b/gitnexus-claude-plugin/.codex-plugin/plugin.json index d57f9f52ec..d7969761bd 100644 --- a/gitnexus-claude-plugin/.codex-plugin/plugin.json +++ b/gitnexus-claude-plugin/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "skills": "./skills", "mcpServers": "./.mcp.json", "hooks": "./hooks/hooks.json", diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index b046c92bf6..65a4b8b277 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -4,6 +4,42 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +## [1.6.10-electric.11] - 2026-08-31 + +### Changed + +- **Milestone 9 writer and Doctor protections now ship in the artifact**: supported writers share + canonical analyze ownership; registry projections use owner/generation compare-and-swap and + recovery-safe implicit branch adoption; Doctor adds registry, freshness, identity, integrity, + readiness, and strict exit-status reporting. +- **Healthy source-stale indexes keep their unchanged embedding payloads on the capped + incremental path**: run `analyze` with a positive total-graph node cap (`--embeddings `). + GitNexus refuses the whole embedding pass after the provider-free pipeline but before graph + persistence, final metadata, registry update, or staged promotion; transient caches or prepared + stage recovery files may already exist. When admitted, it + regenerates changed or new owners while retaining unaffected logical embedding-row payload bytes. + Large effective write sets may still use the existing full-write escalation while preserving the + same bounded snapshot contract. +- **Malformed or provenance-unproven indexes use an explicit capped staged clean rebuild**: + `--staged --embeddings --drop-embeddings` builds an isolated generation, keeps the + canonical generation untouched until validation and promotion. The journal and canonical + database backup support rollback or forward recovery; only the injected milestone 10 boundary + proved exact logical metadata-and-row restoration, not every later promotion failure. +- Rows from older metadata that lack durable provider or identity proof remain unclassified; + they are not described as verified preservation. Doctor structural health is not provider + provenance; incremental admission requires separately recorded provider/model/dimensions proof. +- Automatic `--force` recovery above the implicit local 50,000-node cap continues in + preserve-only mode with cached vectors. Explicit embedding requests and explicit positive caps + remain all-or-nothing refusal gates. + +### Release boundary + +- Electric distribution remains GitHub Release assets only. This version does not publish to + npm or a container registry, follow a floating tag, or perform runtime, index, fleet, or + customer mutation. +- The installed `1.6.10-electric.10` artifact remains the immediate non-writing rollback + runtime while `.11` release and install proof are completed separately. + ## [1.6.10-electric.10] - 2026-08-20 ### Fixed diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index ad6e7701f4..99603a85dc 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,12 +1,12 @@ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { diff --git a/gitnexus/package.json b/gitnexus/package.json index a0b7eb9884..6a73c1c979 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.6.10-electric.10", + "version": "1.6.10-electric.11", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", diff --git a/gitnexus/src/core/embedding-mode.ts b/gitnexus/src/core/embedding-mode.ts index fb8168ddd4..91f60655ac 100644 --- a/gitnexus/src/core/embedding-mode.ts +++ b/gitnexus/src/core/embedding-mode.ts @@ -76,6 +76,26 @@ export function deriveEmbeddingCap( return { skipForCap, capDisabled, nodeLimit }; } +/** + * The implicit local-model cap is allowed to downgrade only the automatic + * `--force` top-up of an already-embedded repository to preserve-only mode. + * Explicit embedding requests and explicit positive caps remain all-or- + * nothing admission gates. + */ +export function preserveOnlyForImplicitForceCap( + skipForCap: boolean, + forceRegenerateEmbeddings: boolean, + embeddingsNodeLimit: number | undefined, + explicitEmbeddingRequest: boolean, +): boolean { + return ( + skipForCap && + forceRegenerateEmbeddings && + embeddingsNodeLimit === undefined && + !explicitEmbeddingRequest + ); +} + export function deriveEmbeddingMode( options: EmbeddingModeInput, existingEmbeddingCount: number, diff --git a/gitnexus/src/core/lbug/pdg-emit-sink.ts b/gitnexus/src/core/lbug/pdg-emit-sink.ts index 79cc05f585..b716bad3c7 100644 --- a/gitnexus/src/core/lbug/pdg-emit-sink.ts +++ b/gitnexus/src/core/lbug/pdg-emit-sink.ts @@ -184,6 +184,39 @@ export interface PdgEmitManifest { readonly relsByPair: Map; } +/** + * Remove only the temporary CSV files named by a completed PDG emit manifest. + * Cap admission happens after the provider-free pipeline so a refused run can + * otherwise leave a large streamed PDG layer behind. The manifest files are + * not recovery artifacts: a retry always re-emits them. Cleanup is best-effort + * because the original admission refusal remains the primary error. + */ +export function cleanupPdgEmitManifestFiles(manifest: PdgEmitManifest | undefined): void { + if (!manifest) return; + + const directories = new Set(); + const files = new Set(); + for (const { csvPath } of manifest.nodeFiles.values()) files.add(csvPath); + for (const { csvPath } of manifest.relsByPair.values()) files.add(csvPath); + + for (const csvPath of files) { + directories.add(path.dirname(csvPath)); + try { + fs.unlinkSync(csvPath); + } catch { + // Best-effort: preserve the cap-refusal error and never widen deletion + // beyond the exact manifest paths. + } + } + for (const directory of directories) { + try { + fs.rmdirSync(directory); + } catch { + // A non-empty directory may contain unrelated data; leave it intact. + } + } +} + /** * Write-routing graph façade. Construct one per analyze run, thread it into the * per-language `runScopeResolution` calls in place of the real graph during the diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 3da3e6e4ad..e801b6354c 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -45,6 +45,7 @@ import { type EmbeddingIntegrityReport, } from './lbug/lbug-adapter.js'; import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js'; +import { cleanupPdgEmitManifestFiles } from './lbug/pdg-emit-sink.js'; import { escapeCypherString } from './lbug/cypher-escape.js'; import { createSearchFTSIndexes, @@ -545,6 +546,7 @@ export type { EmbeddingMode } from './embedding-mode.js'; import { deriveEmbeddingMode as _deriveEmbeddingMode, deriveEmbeddingCap, + preserveOnlyForImplicitForceCap, resolveEmbeddingNodeLimit, DEFAULT_EMBEDDING_NODE_LIMIT, } from './embedding-mode.js'; @@ -2368,6 +2370,90 @@ const runFullAnalysisImpl = async ( }, ); + const streamedPdgNodeRows = pipelineResult.pdgEmitManifest + ? [...pipelineResult.pdgEmitManifest.nodeFiles.values()].reduce( + (total, entry) => total + entry.rows, + 0, + ) + : 0; + const streamedPdgRelationshipRows = pipelineResult.pdgEmitManifest + ? [...pipelineResult.pdgEmitManifest.relsByPair.values()].reduce( + (total, entry) => total + entry.rows, + 0, + ) + : 0; + const streamedPdgRows = streamedPdgNodeRows + streamedPdgRelationshipRows; + + // Explicit embedding requests and positive caps are all-or-nothing + // admission gates, not requests to finish the graph write without semantic + // rows. The sole exception is automatic force recovery above the implicit + // local cap, which may restore cached vectors in preserve-only mode. Settle + // the decision immediately after the provider-free pipeline has produced + // the complete + // prospective node count and before any LadybugDB, metadata, registry, or + // staged-promotion write begins. Pipeline cache reconciliation and prepared + // staging files may already exist; the refusal deliberately preserves them + // for the existing recovery path. + let embeddingSkipped = true; + let semanticMode: 'vector-index' | 'exact-scan' | undefined; + let httpMode = false; + if (shouldGenerateEmbeddings) { + const { isHttpMode } = await import('./embeddings/http-client.js'); + httpMode = isHttpMode(); + const embeddingAdmissionNodeCount = pipelineResult.graph.nodeCount + streamedPdgNodeRows; + const { skipForCap, capDisabled, nodeLimit } = deriveEmbeddingCap( + embeddingAdmissionNodeCount, + resolveEmbeddingNodeLimit(options.embeddingsNodeLimit, resumeEmbeddingCheckpoint), + httpMode, + ); + const preserveOnlyForRecovery = preserveOnlyForImplicitForceCap( + skipForCap, + forceRegenerateEmbeddings, + options.embeddingsNodeLimit, + options.embeddings === true, + ); + if (skipForCap && !preserveOnlyForRecovery) { + cleanupPdgEmitManifestFiles(pipelineResult.pdgEmitManifest); + throw new Error( + `Embedding generation refused: ${embeddingAdmissionNodeCount.toLocaleString()} nodes exceeds ` + + `the ${nodeLimit.toLocaleString()}-node safety cap. ` + + 'No analyze graph write, final metadata commit, or staged promotion was started. ' + + 'Retry with `--embeddings 0` or a limit covering the full graph.', + ); + } + + if (preserveOnlyForRecovery) { + log( + `Automatic embedding regeneration exceeds the implicit ${nodeLimit.toLocaleString()}-node ` + + 'local-model safety cap; continuing recovery in preserve-only mode with cached vectors. ' + + 'Use an explicit positive `--embeddings ` cap to admit regeneration.', + ); + } else { + embeddingSkipped = false; + } + if ( + !preserveOnlyForRecovery && + capDisabled && + embeddingAdmissionNodeCount > DEFAULT_EMBEDDING_NODE_LIMIT + ) { + if (httpMode) { + log( + `Remote embedding endpoint selected — generating embeddings for ` + + `${embeddingAdmissionNodeCount.toLocaleString()} nodes; the ` + + `${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model cap ` + + `does not apply.`, + ); + } else { + log( + `Embedding node-count cap disabled — generating embeddings for ` + + `${embeddingAdmissionNodeCount.toLocaleString()} nodes. Ensure sufficient memory; the ` + + `default ${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model ` + + `cap exists to prevent OOM.`, + ); + } + } + } + // ── Phase 2: LadybugDB (60–85%) ────────────────────────────────── progress('lbug', 60, 'Loading into LadybugDB...'); @@ -2464,12 +2550,6 @@ const runFullAnalysisImpl = async ( await wipeLbugDbFiles(lbugPath); } - const streamedPdgRows = pipelineResult.pdgEmitManifest - ? [ - ...pipelineResult.pdgEmitManifest.nodeFiles.values(), - ...pipelineResult.pdgEmitManifest.relsByPair.values(), - ].reduce((total, entry) => total + entry.rows, 0) - : 0; setBufferPoolSizeHint( estimateBufferPool( pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount + streamedPdgRows, @@ -2937,50 +3017,10 @@ const runFullAnalysisImpl = async ( progress('fts', 90, 'Search indexes skipped (FTS unavailable)'); } - // Settle embedding admission before streaming the snapshot. Exact row-ID - // sidecars are needed only when Phase 4 will run; preserve-only and - // cap-refused analyses must keep the snapshot pass bounded to its batches. + // Exact row-ID sidecars are needed only when Phase 4 will run; preserve-only + // analyses keep the snapshot pass bounded to its batches. Explicit positive + // cap refusal already terminated before Phase 2. const stats = await getLbugStats(); - let embeddingSkipped = true; - let semanticMode: 'vector-index' | 'exact-scan' | undefined; - let httpMode = false; - - if (shouldGenerateEmbeddings) { - const { isHttpMode } = await import('./embeddings/http-client.js'); - httpMode = isHttpMode(); - const { skipForCap, capDisabled, nodeLimit } = deriveEmbeddingCap( - stats.nodes, - resolveEmbeddingNodeLimit(options.embeddingsNodeLimit, resumeEmbeddingCheckpoint), - httpMode, - ); - if (!skipForCap) { - embeddingSkipped = false; - if (capDisabled && stats.nodes > DEFAULT_EMBEDDING_NODE_LIMIT) { - if (httpMode) { - log( - `Remote embedding endpoint selected — generating embeddings for ` + - `${stats.nodes.toLocaleString()} nodes; the ` + - `${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model cap ` + - `does not apply.`, - ); - } else { - log( - `Embedding node-count cap disabled — generating embeddings for ` + - `${stats.nodes.toLocaleString()} nodes. Ensure sufficient memory; the ` + - `default ${DEFAULT_EMBEDDING_NODE_LIMIT.toLocaleString()}-node local-model ` + - `cap exists to prevent OOM.`, - ); - } - } - } else { - log( - `Embeddings skipped: ${stats.nodes.toLocaleString()} nodes exceeds ` + - `the ${nodeLimit.toLocaleString()}-node safety cap. ` + - `Override with \`--embeddings 0\` to disable the cap, or ` + - `\`--embeddings \` to set a custom cap.`, - ); - } - } // ── Phase 3.5: Re-insert cached embeddings ──────────────────────── // Runs on BOTH the full-rebuild path and the incremental path: diff --git a/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts b/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts index 873ff6a313..8f5a6a9a66 100644 --- a/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts +++ b/gitnexus/test/integration/m8b-incremental-embedding-restore.test.ts @@ -194,4 +194,75 @@ describe('M8b incremental embedding restore', () => { vi.unstubAllGlobals(); } }, 180_000); + + it('refuses an over-cap incremental run before canonical graph or metadata mutation', async () => { + const previousEnv = new Map(EMBEDDING_ENV_KEYS.map((key) => [key, process.env[key]] as const)); + const home = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-cap-refusal-home-')); + const repo = await setupMiniRepo('gitnexus-cap-refusal-'); + try { + process.env.GITNEXUS_HOME = home; + process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'never'; + process.env.GITNEXUS_EMBEDDING_URL = 'http://in-process.invalid/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'cap-refusal-test'; + process.env.GITNEXUS_EMBEDDING_DIMS = String(EMBEDDING_DIMS); + process.env.GITNEXUS_EMBEDDING_MAX_ATTEMPTS = '1'; + process.env.GITNEXUS_EMBEDDING_RETRY_CAP_MS = '1'; + process.env.GITNEXUS_EMBEDDING_MIN_INTERVAL_MS = '0'; + delete process.env.GITNEXUS_EMBEDDING_API_KEY; + const fetchMock = vi.fn(async () => { + throw new Error('provider must not be called on cap refusal'); + }); + vi.stubGlobal('fetch', fetchMock); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis( + repo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ); + + const { lbugPath, storagePath } = getStoragePaths(repo.dbPath); + const graphBefore = await readFile(lbugPath); + const metaPath = path.join(storagePath, 'meta.json'); + const metaBefore = await readFile(metaPath); + + const handlerPath = path.join(repo.dbPath, 'src', 'handler.ts'); + const handlerSource = await readFile(handlerPath, 'utf8'); + await writeFile( + handlerPath, + handlerSource.replace( + 'return formatResponse(saved);', + "return formatResponse(saved) + ' cap-refusal';", + ), + 'utf8', + ); + gitCommitAll(repo.dbPath, 'cap refusal source delta'); + + await expect( + runFullAnalysis( + repo.dbPath, + { + incrementalOnly: true, + skipAgentsMd: true, + skipSkills: true, + embeddings: true, + embeddingsNodeLimit: 1, + }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/Embedding generation refused: .* exceeds the 1-node safety cap/i); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(await readFile(lbugPath)).toEqual(graphBefore); + expect(await readFile(metaPath)).toEqual(metaBefore); + } finally { + await repo.cleanup(); + await rm(home, { recursive: true, force: true }); + for (const [key, value] of previousEnv) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + vi.unstubAllGlobals(); + } + }, 120_000); }); diff --git a/gitnexus/test/unit/incremental-dirty-recovery.test.ts b/gitnexus/test/unit/incremental-dirty-recovery.test.ts index eafb933ce6..34d6f4fc93 100644 --- a/gitnexus/test/unit/incremental-dirty-recovery.test.ts +++ b/gitnexus/test/unit/incremental-dirty-recovery.test.ts @@ -18,8 +18,9 @@ * Windows. */ +import { createHash } from 'node:crypto'; import { writeFile, readFile } from 'fs/promises'; -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { getStoragePaths, saveMeta, @@ -31,12 +32,76 @@ import { setupMiniRepo as setupSharedMiniRepo } from '../helpers/mini-repo.js'; // zero-vector seeding pattern previously lived here as a divergent copy of // incremental-orchestration.test.ts's (a helper module has no // describe-registration problem, unlike importing a sibling test file). -import { seedEmbeddingsForFiles } from '../helpers/embedding-seed.js'; +import { readEmbeddingRowFingerprints, seedEmbeddingsForFiles } from '../helpers/embedding-seed.js'; const setupMiniRepo = () => setupSharedMiniRepo('gitnexus-incr-dirty-rec-'); +const EMBEDDING_DIMS = 384; +const EMBEDDING_ENV_KEYS = [ + 'GITNEXUS_EMBEDDING_URL', + 'GITNEXUS_EMBEDDING_MODEL', + 'GITNEXUS_EMBEDDING_DIMS', + 'GITNEXUS_EMBEDDING_MAX_ATTEMPTS', + 'GITNEXUS_EMBEDDING_RETRY_CAP_MS', + 'GITNEXUS_EMBEDDING_MIN_INTERVAL_MS', + 'GITNEXUS_EMBEDDING_API_KEY', +] as const; + +const deterministicEmbedding = (text: string): number[] => { + const digest = createHash('sha256').update(text).digest(); + return Array.from( + { length: EMBEDDING_DIMS }, + (_, index) => (digest[index % digest.length]! - 128) / 128, + ); +}; + +const stampCurrentContentHashes = async (repoPath: string, nodeIds: readonly string[]) => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const { contentHashForNode } = await import('../../src/core/embeddings/embedding-pipeline.js'); + const { escapeCypherString } = await import('../../src/core/lbug/cypher-escape.js'); + const { lbugPath } = getStoragePaths(repoPath); + await adapter.initLbug(lbugPath); + try { + const wanted = `[${nodeIds.map((id) => `'${escapeCypherString(id)}'`).join(', ')}]`; + const rows = (await adapter.executeQuery( + `MATCH (n:Function) WHERE n.id IN ${wanted} RETURN n.id AS id, n.name AS name, ` + + `'Function' AS label, n.filePath AS filePath, n.content AS content, ` + + 'n.startLine AS startLine, n.endLine AS endLine, n.isExported AS isExported, ' + + 'n.description AS description', + )) as Array>; + expect(rows).toHaveLength(nodeIds.length); + for (const row of rows) { + const nodeId = String(row.id ?? ''); + const contentHash = contentHashForNode({ + id: nodeId, + name: String(row.name ?? ''), + label: 'Function', + filePath: String(row.filePath ?? ''), + content: String(row.content ?? ''), + startLine: Number(row.startLine ?? 0), + endLine: Number(row.endLine ?? 0), + isExported: + row.isExported === undefined || row.isExported === null + ? undefined + : Boolean(row.isExported), + description: + row.description === undefined || row.description === null + ? undefined + : String(row.description), + }); + await adapter.executeQuery( + `MATCH (e:CodeEmbedding) WHERE e.nodeId = '${escapeCypherString(nodeId)}' ` + + `SET e.contentHash = '${escapeCypherString(contentHash)}'`, + ); + } + } finally { + await adapter.closeLbug(); + } +}; + describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () => { it('parks the crashed run WAL/shadow sidecars before reopening, then rebuilds clean', async () => { + const previousEnv = new Map(EMBEDDING_ENV_KEYS.map((key) => [key, process.env[key]] as const)); const repo = await setupMiniRepo(); try { const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); @@ -57,6 +122,12 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = ); const seededNodeIds = [...seededIdsByFile.values()].flat(); expect(seededNodeIds.length).toBeGreaterThan(0); + await stampCurrentContentHashes(repo.dbPath, seededNodeIds); + const seededNodeIdSet = new Set(seededNodeIds); + const seededFingerprintsBefore = (await readEmbeddingRowFingerprints(repo.dbPath)).filter( + (row) => seededNodeIdSet.has(row.nodeId), + ); + expect(seededFingerprintsBefore).toHaveLength(seededNodeIds.length); // Simulate a crashed incremental writeback: dirty flag in meta plus // leftover sidecars whose bytes must never be replayed. 8KB puts the @@ -79,19 +150,36 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = await writeFile(`${lbugPath}.shadow`, shadowGarbage); const logs: string[] = []; - // embeddingsNodeLimit: 1 (KTD9): the recovery runs force:true - // internally, and the seeded stats would otherwise route Phase 4 into - // a real embedder in CI — the 1-node cap suppresses generation while - // leaving the preserve/restore path fully live. On linux the - // wipe-and-restore vector-index seam then fires for real (statically - // linked VECTOR): a CREATE_VECTOR_INDEX over the restored rows is - // expected and harmless here. + // Recovery runs force:true internally, so use the same deterministic + // in-process HTTP provider as the embedding canaries and admit the + // complete fixture with a real positive cap. A below-count cap now + // correctly refuses before recovery graph writes; it is no longer a + // supported way to suppress generation while continuing the rebuild. + process.env.GITNEXUS_EMBEDDING_URL = 'http://in-process.invalid/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'dirty-recovery-deterministic'; + process.env.GITNEXUS_EMBEDDING_DIMS = String(EMBEDDING_DIMS); + process.env.GITNEXUS_EMBEDDING_MAX_ATTEMPTS = '1'; + process.env.GITNEXUS_EMBEDDING_RETRY_CAP_MS = '1'; + process.env.GITNEXUS_EMBEDDING_MIN_INTERVAL_MS = '0'; + delete process.env.GITNEXUS_EMBEDDING_API_KEY; + const fetchMock = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? '{}')) as { input?: unknown }; + const inputs = Array.isArray(body.input) ? body.input.map(String) : []; + return new Response( + JSON.stringify({ + data: inputs.map((text) => ({ embedding: deterministicEmbedding(text) })), + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + }); + vi.stubGlobal('fetch', fetchMock); const recovered = await runFullAnalysis( repo.dbPath, - { skipAgentsMd: true, embeddingsNodeLimit: 1 }, + { skipAgentsMd: true, embeddingsNodeLimit: 1000 }, { onProgress: () => {}, onLog: (m) => logs.push(m) }, ); expect(recovered.alreadyUpToDate).toBeUndefined(); + expect(fetchMock).toHaveBeenCalled(); // Both sidecars were parked verbatim (renamed, never deleted) before // any open could replay them… @@ -109,17 +197,23 @@ describe('runFullAnalysis — dirty-flag recovery sidecar parking (#2409)', () = expect(joinedLogs).toContain( `--force on a repo with ${seededNodeIds.length} existing embeddings`, ); - // …with generation itself cap-suppressed (no embedder in CI): - expect(joinedLogs).toContain('exceeds the 1-node safety cap'); - // …and the rebuild completed into a clean index: dirty flag cleared, // and the seeded embeddings survived the park → open → wipe → restore // round-trip (the strongest signal the preservation open really ran: // the DB was wiped, so these rows can only come from the cache load). const after = await loadMeta(storagePath); expect(after!.incrementalInProgress).toBeUndefined(); - expect(after!.stats?.embeddings).toBe(seededNodeIds.length); + const recoveredFingerprints = await readEmbeddingRowFingerprints(repo.dbPath); + expect(recoveredFingerprints.filter((row) => seededNodeIdSet.has(row.nodeId))).toEqual( + seededFingerprintsBefore, + ); + expect(after!.stats?.embeddings).toBe(recoveredFingerprints.length); } finally { + vi.unstubAllGlobals(); + for (const [key, value] of previousEnv) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } await repo.cleanup(); } }, 300_000); diff --git a/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts b/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts index 7d4699529e..d3f85c9336 100644 --- a/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts +++ b/gitnexus/test/unit/lbug/pdg-emit-sink.test.ts @@ -17,7 +17,7 @@ import path from 'node:path'; import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { streamAllCSVsToDisk, buildBasicBlockRow } from '../../../src/core/lbug/csv-generator.js'; -import { PdgEmitSink } from '../../../src/core/lbug/pdg-emit-sink.js'; +import { cleanupPdgEmitManifestFiles, PdgEmitSink } from '../../../src/core/lbug/pdg-emit-sink.js'; import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; const bbNode = (fp: string, idx: number, line: number): GraphNode => ({ @@ -61,6 +61,26 @@ afterEach(() => { }); describe('PdgEmitSink — routing', () => { + it('removes only exact manifest files after a pre-COPY refusal', () => { + const pdgDir = path.join(tmpRoot, 'pdg-csv'); + const real = createKnowledgeGraph(); + const sink = new PdgEmitSink(real, pdgDir); + sink.addNode(bbNode('a.ts', 0, 1)); + sink.addNode(bbNode('a.ts', 1, 5)); + sink.addRelationship(pdgEdge('a.ts', 0, 1, 'CFG', 'seq')); + const manifest = sink.finalize(); + const unrelated = path.join(pdgDir, 'unrelated.keep'); + fs.writeFileSync(unrelated, 'keep'); + + cleanupPdgEmitManifestFiles(manifest); + + for (const { csvPath } of manifest.nodeFiles.values()) + expect(fs.existsSync(csvPath)).toBe(false); + for (const { csvPath } of manifest.relsByPair.values()) + expect(fs.existsSync(csvPath)).toBe(false); + expect(fs.readFileSync(unrelated, 'utf8')).toBe('keep'); + }); + it('routes BasicBlock nodes and PDG edges to CSV, never to the real graph', () => { const real = createKnowledgeGraph(); const sink = new PdgEmitSink(real, path.join(tmpRoot, 'pdg-csv')); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 3eae363218..826edfa28d 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -718,7 +718,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 vi.unstubAllEnvs(); }); - it('stamps capabilities.vectorSearch.status = exact-scan when post-restore index recreation reports failure', async () => { + it('refuses an over-cap rebuild before graph writes or cached-row restoration', async () => { const RESTORED_NODE_ID = 'Function:src/app.ts:handler:1'; const stubNode = { id: RESTORED_NODE_ID, @@ -726,11 +726,15 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 name: 'handler', properties: { filePath: 'src/app.ts' }, }; + const initLbug = vi.fn(async () => undefined); + const loadGraphToLbug = vi.fn(async () => undefined); + const wipeLbugDbFiles = vi.fn(async () => undefined); const buildVectorIndex = vi.fn(async () => false); const executeWithReusedStatement = vi.fn(async () => []); + let pdgCsvPath = ''; vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ - initLbug: vi.fn(async () => undefined), - loadGraphToLbug: vi.fn(async () => undefined), + initLbug, + loadGraphToLbug, getLbugStats: vi.fn(async () => ({ nodes: 2, edges: 0, communities: 0, processes: 0 })), // The finalize embedding count answers 1 (the restored row) — a zero // count would stamp 'unavailable' and the exact-scan assertion below @@ -747,7 +751,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 embeddingIntegrityFailures: vi.fn(() => 0), // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. - wipeLbugDbFiles: vi.fn(async () => undefined), + wipeLbugDbFiles, // ≥1 cached row with a real-dims embedding: the harness default (empty // cache) would leave restoredEmbeddingCount at 0 and the recreation // gate shut — this test would then assert nothing. @@ -787,7 +791,13 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, totalFileCount: 1, + pdgEmitManifest: { + nodeFiles: new Map([['chunk-0', { csvPath: pdgCsvPath, rows: 1 }]]), + relsByPair: new Map(), + }, graph: { + nodeCount: 1, + relationshipCount: 0, forEachNode: (fn: (node: typeof stubNode) => void) => fn(stubNode), getNode: (id: string) => (id === RESTORED_NODE_ID ? stubNode : undefined), }, @@ -811,35 +821,36 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 try { const { storagePath } = getStoragePaths(tmpRepo.dbPath); await fs.mkdir(storagePath, { recursive: true }); - // stats.embeddings > 0 → deriveEmbeddingMode loads the cache; force + - // embeddingsNodeLimit(1) < getLbugStats().nodes(2) → generation is - // cap-skipped. That makes this a wiped PRESERVE-shaped run — exactly - // the KTD1 case where a naive `!shouldGenerateEmbeddings` gate would - // wrongly stay shut (shouldGenerate is TRUE here, yet the Phase 4 - // pipeline never runs). - await saveMeta(storagePath, { + pdgCsvPath = `${storagePath}/pdg-csv/basicblock.csv`; + await fs.mkdir(`${storagePath}/pdg-csv`, { recursive: true }); + await fs.writeFile(pdgCsvPath, 'id:ID(BasicBlock)\n'); + const originalMeta: RepoMeta = { repoPath: tmpRepo.dbPath, lastCommit: '', indexedAt: new Date().toISOString(), stats: { embeddings: 1 }, - }); + }; + await saveMeta(storagePath, originalMeta); const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); - await runFullAnalysis( - tmpRepo.dbPath, - { force: true, embeddingsNodeLimit: 1 }, - { onProgress: () => {} }, - ); + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { force: true, embeddingsNodeLimit: 1 }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/Embedding generation refused: 2 nodes exceeds the 1-node safety cap/i); - // The recreation seam fired exactly once… - expect(buildVectorIndex).toHaveBeenCalledTimes(1); - // …the restore actually submitted the cached row (one 200-row batch)… - expect(executeWithReusedStatement).toHaveBeenCalledTimes(1); - // …and the persisted stamp reflects the DB's ACTUAL state, not the - // platform capability fallback. + // The bounded pre-pipeline embedding snapshot may open the existing DB. + // Refusal must still happen before any graph load, wipe, or restore write. + expect(initLbug).toHaveBeenCalledTimes(1); + expect(loadGraphToLbug).not.toHaveBeenCalled(); + expect(wipeLbugDbFiles).not.toHaveBeenCalled(); + expect(buildVectorIndex).not.toHaveBeenCalled(); + expect(executeWithReusedStatement).not.toHaveBeenCalled(); + await expect(fs.access(pdgCsvPath)).rejects.toMatchObject({ code: 'ENOENT' }); const meta = JSON.parse(await fs.readFile(`${storagePath}/meta.json`, 'utf-8')) as RepoMeta; - expect(meta.capabilities?.vectorSearch.status).toBe('exact-scan'); - expect(meta.stats?.embeddings).toBe(1); + expect(meta).toEqual(originalMeta); } finally { await tmpRepo.cleanup(); } diff --git a/gitnexus/test/unit/run-analyze-staged.test.ts b/gitnexus/test/unit/run-analyze-staged.test.ts index e0b1dbff4c..818856b672 100644 --- a/gitnexus/test/unit/run-analyze-staged.test.ts +++ b/gitnexus/test/unit/run-analyze-staged.test.ts @@ -218,6 +218,59 @@ describe('runFullAnalysis --staged', () => { await expect(fs.access(staged.stageRoot)).rejects.toMatchObject({ code: 'ENOENT' }); }, 120_000); + it('refuses an over-cap staged rebuild before canonical mutation or promotion', async () => { + const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-staged-cap-refusal-')); + tempDirs.push(repo); + process.env.GITNEXUS_HOME = path.join(repo, '.registry-home'); + process.env.GITNEXUS_EMBEDDING_URL = 'http://test.invalid/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'cap-refusal-test'; + process.env.GITNEXUS_EMBEDDING_DIMS = String(EMBEDDING_DIMS); + const fetchMock = vi.fn(async () => { + throw new Error('provider must not be called on cap refusal'); + }); + vi.stubGlobal('fetch', fetchMock); + await fs.writeFile( + path.join(repo, 'index.ts'), + 'export function stagedCapRefusal() { return 1; }\n', + ); + execFileSync('git', ['init'], { cwd: repo }); + execFileSync('git', ['add', 'index.ts'], { cwd: repo }); + execFileSync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@test', 'commit', '-m', 'initial'], + { cwd: repo }, + ); + + await runFullAnalysis(repo, { skipAgentsMd: true, skipSkills: true }, { onProgress: () => {} }); + const canonical = getStoragePaths(repo); + const canonicalDbBefore = await fs.readFile(canonical.lbugPath); + const canonicalMetaBefore = await fs.readFile(path.join(canonical.storagePath, 'meta.json')); + const staged = getStagedAnalyzePaths(canonical.lbugPath, canonical.storagePath); + + await expect( + runFullAnalysis( + repo, + { + staged: true, + embeddings: true, + embeddingsNodeLimit: 1, + dropEmbeddings: true, + skipAgentsMd: true, + skipSkills: true, + }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/Embedding generation refused: .* exceeds the 1-node safety cap/i); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(await fs.readFile(canonical.lbugPath)).toEqual(canonicalDbBefore); + expect(await fs.readFile(path.join(canonical.storagePath, 'meta.json'))).toEqual( + canonicalMetaBefore, + ); + await expect(fs.access(staged.backupLbugPath)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(fs.access(staged.journalPath)).rejects.toMatchObject({ code: 'ENOENT' }); + }, 120_000); + it('never resumes a staged embedding checkpoint without an explicit clean rebuild', async () => { const repo = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-staged-checkpoint-refusal-')); tempDirs.push(repo); diff --git a/gitnexus/test/unit/run-analyze.test.ts b/gitnexus/test/unit/run-analyze.test.ts index 9c58007946..36ccf59097 100644 --- a/gitnexus/test/unit/run-analyze.test.ts +++ b/gitnexus/test/unit/run-analyze.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect, vi } from 'vitest'; import { deriveEmbeddingMode, deriveEmbeddingCap, + preserveOnlyForImplicitForceCap, resolveEmbeddingNodeLimit, DEFAULT_EMBEDDING_NODE_LIMIT, } from '../../src/core/embedding-mode.js'; @@ -1190,6 +1191,14 @@ describe('deriveEmbeddingCap', () => { it('custom cap below default still applies', () => { expect(deriveEmbeddingCap(15_000, 10_000).skipForCap).toBe(true); }); + + it('downgrades only an implicit force top-up to preserve-only recovery', () => { + expect(preserveOnlyForImplicitForceCap(true, true, undefined, false)).toBe(true); + expect(preserveOnlyForImplicitForceCap(true, true, undefined, true)).toBe(false); + expect(preserveOnlyForImplicitForceCap(true, true, 50_000, false)).toBe(false); + expect(preserveOnlyForImplicitForceCap(true, false, undefined, false)).toBe(false); + expect(preserveOnlyForImplicitForceCap(false, true, undefined, false)).toBe(false); + }); }); describe('pdgModeMismatch / resolvePdgConfig (#2099 F1)', () => {