feat: paths cleared

packages/ingest-core/src/big-file/index.ts

@@ -5,6 +5,7 @@ import type { AskLlmOptions } from "@bb/llm";
 import { logger } from "@bb/logger";
 import type { ChunkAnalysisResult, HugeFileManifest } from "#src/types/big-file.ts";
 import type { CondensedFileAnalysis } from "#src/types/condensed-file-analysis.ts";
+import { metaId } from "#src/pipeline/paths.ts";
 import type { MetaPaths } from "#src/types/meta-paths.ts";
 import type { ProgressContext } from "#src/progress/types.ts";
 import { throwIfCancelled } from "#src/pipeline/cancellation.ts";
@@ -80,7 +81,7 @@ export async function processBigFile(input: ProcessBigFileInput): Promise<Conden
   throwIfCancelled(input.knowledgeId);
   const merged = await condenseChunks(input.relativePath, results, input.llmCallContext);
 
-  const chunkPaths = chunks.map((_, i) => `chunks/${encodeFolder(input.relativePath)}/chunk-${i}.json`);
+  const chunkPaths = chunks.map((_, i) => `chunks/${metaId(input.relativePath)}/chunk-${i}.json`);
   const totalTokenCount = chunks.reduce((acc, c) => acc + c.tokenCount, 0);
 
   const chunkInputTokens = results.reduce((acc, r) => acc + (r.tokenUsage?.inputTokens ?? 0), 0);
@@ -122,7 +123,3 @@ export async function processBigFile(input: ProcessBigFileInput): Promise<Conden
 function sha256(content: string): string {
   return createHash("sha256").update(content).digest("hex");
 }
-
-function encodeFolder(relativePath: string): string {
-  return relativePath.replace(/\//gu, "__SL__").replace(/\\/gu, "__BS__");
-}

packages/ingest-core/src/big-file/storage.ts

@@ -1,26 +1,26 @@
 import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
-import { encodeMetaPath } from "#src/pipeline/paths.ts";
+import { metaId } from "#src/pipeline/paths.ts";
 import type { CondensedFileAnalysis } from "#src/types/condensed-file-analysis.ts";
 import type { ChunkAnalysisResult, HugeFileManifest } from "#src/types/big-file.ts";
 import type { MetaPaths } from "#src/types/meta-paths.ts";
 
 const DIR_MODE = 0o700;
 
 function chunkDir(metaPaths: MetaPaths, relativePath: string): string {
-  return path.join(metaPaths.bigFileChunksDir, encodeMetaPath(relativePath));
+  return path.join(metaPaths.bigFileChunksDir, metaId(relativePath));
 }
 
 function chunkFile(metaPaths: MetaPaths, relativePath: string, chunkIndex: number): string {
   return path.join(chunkDir(metaPaths, relativePath), `chunk-${chunkIndex}.json`);
 }
 
 function manifestFile(metaPaths: MetaPaths, relativePath: string): string {
-  return path.join(metaPaths.bigFileAnalysisDir, `${encodeMetaPath(relativePath)}.manifest.json`);
+  return path.join(metaPaths.bigFileAnalysisDir, `${metaId(relativePath)}.manifest.json`);
 }
 
 function condensedFile(metaPaths: MetaPaths, relativePath: string): string {
-  return path.join(metaPaths.fileAnalysisDir, `${encodeMetaPath(relativePath)}.json`);
+  return path.join(metaPaths.fileAnalysisDir, `${metaId(relativePath)}.json`);
 }
 
 export async function saveChunk(metaPaths: MetaPaths, result: ChunkAnalysisResult): Promise<string> {

packages/ingest-core/src/index.ts

@@ -64,6 +64,7 @@ export {
   orgRegistryDir,
   encodeMetaPath,
   decodeMetaPath,
+  metaId,
 } from "#src/pipeline/paths.ts";
 export type { RepoLocation } from "#src/pipeline/paths.ts";
 
@@ -186,6 +187,15 @@ export { FileAnalysisCache } from "#src/file-analysis-cache.ts";
 export { analyseScannedFile, buildOversizedStub } from "#src/analyse-file.ts";
 export { directFolderOf, affectedFolderPaths } from "#src/folder-path.ts";
 export { readScanManifest, writeScanManifest, emptyManifest } from "#src/scan-manifest.ts";
+export {
+  buildPathMap,
+  writePathMap,
+  readPathMap,
+  pathMapPath,
+  PATH_MAP_RELATIVE_PATH,
+  PATH_MAP_SCHEMA_VERSION,
+} from "#src/path-map.ts";
+export type { PathMap } from "#src/path-map.ts";
 export type { ScanManifest, ScanManifestEntry, ScanManifestSummary, ScanEntryKind } from "#src/scan-manifest.ts";
 export { writeEligibleFiles, ELIGIBLE_FILES_RELATIVE_PATH } from "#src/eligible-files.ts";
 export type { EligibleFilesDocument, WriteEligibleFilesInput } from "#src/eligible-files.ts";

packages/ingest-core/src/path-map.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "bun:test";
+import { buildPathMap } from "./path-map.ts";
+import { metaId } from "./pipeline/paths.ts";
+import type { ScanManifest } from "./scan-manifest.ts";
+
+function manifest(paths: string[]): ScanManifest {
+  return {
+    generatedAt: "t",
+    summary: {
+      totalFiles: paths.length,
+      smallCount: paths.length,
+      bigCount: 0,
+      oversizedCount: 0,
+      totalTokens: 0,
+      estimatedBigChunks: 0,
+    },
+    entries: paths.map((relativePath) => ({
+      relativePath,
+      absolutePath: relativePath,
+      sizeBytes: 0,
+      tokenCount: 0,
+      kind: "small" as const,
+    })),
+  };
+}
+
+/** `buildPathMap` — the pure `id → path` inverse of `metaId`, built once from the scan manifest. */
+describe("buildPathMap", () => {
+  it("maps every file's metaId back to its path", () => {
+    const m = buildPathMap(manifest(["src/a.ts", "src/foo/b.ts"]));
+    expect(m.files[metaId("src/a.ts")]).toBe("src/a.ts");
+    expect(m.files[metaId("src/foo/b.ts")]).toBe("src/foo/b.ts");
+  });
+
+  it("includes every ancestor folder + the root sentinel", () => {
+    const m = buildPathMap(manifest(["src/foo/bar/b.ts"]));
+    expect(m.folders["__ROOT__"]).toBe("");
+    expect(m.folders[metaId("src")]).toBe("src");
+    expect(m.folders[metaId("src/foo")]).toBe("src/foo");
+    expect(m.folders[metaId("src/foo/bar")]).toBe("src/foo/bar");
+  });
+
+  it("produces one distinct key per distinct file (no collisions)", () => {
+    const paths = ["a/b.ts", "a_b.ts", "a.b.ts", "c/d.ts"];
+    const m = buildPathMap(manifest(paths));
+    expect(Object.keys(m.files)).toHaveLength(paths.length);
+  });
+
+  it("stamps the schema version + algo", () => {
+    const m = buildPathMap(manifest(["x.ts"]));
+    expect(m.version).toBe(1);
+    expect(m.algo).toBe("sha256");
+  });
+});

packages/ingest-core/src/path-map.ts

@@ -0,0 +1,58 @@
+import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { metaId } from "#src/pipeline/paths.ts";
+import type { ScanManifest } from "#src/scan-manifest.ts";
+import type { MetaPaths } from "#src/types/meta-paths.ts";
+
+/**
+ * Persisted inverse of the `metaId` storage-key scheme (see pipeline/paths.ts). Every meta-output artifact
+ * dir is named `metaId(<repo-relative path>)` — a one-way hash — so this `<metaOutputRoot>/path-map.json`
+ * records `id → path` for every file and every ancestor folder. Two roles:
+ *   1. External consumers (the VS Code extension) rebuild the repo's folder tree from it, then fetch each
+ *      artifact by recomputing `metaId(path)`.
+ *   2. Its PRESENCE marks a metaRoot as written by the hash scheme ("v2"), which the incremental baseline
+ *      gate uses to refuse seeding a new run from an old `__SL__`-encoded tree.
+ * Built ONCE from the scan manifest (all paths known before any fan-out worker starts), so no concurrency.
+ */
+export const PATH_MAP_RELATIVE_PATH = "path-map.json";
+export const PATH_MAP_SCHEMA_VERSION = 1;
+
+export interface PathMap {
+  version: number;
+  algo: "sha256";
+  /** `metaId(relativePath) → relativePath` for every scanned file. */
+  files: Record<string, string>;
+  /** `metaId(folderPath) → folderPath` for every ancestor folder; the repo root is `"__ROOT__" → ""`. */
+  folders: Record<string, string>;
+}
+
+/** PURE. Builds the `id → path` map for every file in the manifest plus every ancestor folder. */
+export function buildPathMap(manifest: ScanManifest): PathMap {
+  const files: Record<string, string> = {};
+  const folders: Record<string, string> = { __ROOT__: "" };
+  for (const entry of manifest.entries) {
+    files[metaId(entry.relativePath)] = entry.relativePath;
+    const parts = entry.relativePath.split("/");
+    for (let i = 1; i < parts.length; i += 1) {
+      const folder = parts.slice(0, i).join("/");
+      folders[metaId(folder)] = folder;
+    }
+  }
+  return { version: PATH_MAP_SCHEMA_VERSION, algo: "sha256", files, folders };
+}
+
+export function pathMapPath(metaPaths: MetaPaths): string {
+  return path.join(metaPaths.metaOutputRoot, PATH_MAP_RELATIVE_PATH);
+}
+
+export async function writePathMap(metaPaths: MetaPaths, map: PathMap): Promise<void> {
+  await writeFile(pathMapPath(metaPaths), JSON.stringify(map, null, 2), "utf8");
+}
+
+export async function readPathMap(metaPaths: MetaPaths): Promise<PathMap | null> {
+  try {
+    return JSON.parse(await readFile(pathMapPath(metaPaths), "utf8")) as PathMap;
+  } catch {
+    return null;
+  }
+}

packages/ingest-core/src/phases/analyse-big-files.ts

@@ -4,6 +4,7 @@ import { Config } from "@bb/types";
 import { getConfigValue } from "@bb/config";
 import type { AskLlmOptions } from "@bb/llm";
 import { LlmConfigError, LlmError } from "@bb/errors";
+import { metaId } from "#src/pipeline/paths.ts";
 import type { MetaPaths } from "#src/types/meta-paths.ts";
 import type { AnalyzedFileResult, SourceReader } from "#src/types/pipeline.ts";
 import type { ProgressContext } from "#src/progress/types.ts";
@@ -163,7 +164,7 @@ export async function analyseBigFiles(input: AnalyseBigFilesInput): Promise<Proc
             relativePath: state.entry.relativePath,
             totalChunks: state.chunks.length,
             totalTokenCount,
-            chunkPaths: state.chunks.map((_, i) => `chunks/${encodeFolder(state.entry.relativePath)}/chunk-${i}.json`),
+            chunkPaths: state.chunks.map((_, i) => `chunks/${metaId(state.entry.relativePath)}/chunk-${i}.json`),
             generatedAt: new Date().toISOString(),
           };
           await saveManifest(input.metaPaths, manifest);
@@ -281,7 +282,3 @@ export async function analyseBigFiles(input: AnalyseBigFilesInput): Promise<Proc
 function sha256(content: string): string {
   return createHash("sha256").update(content).digest("hex");
 }
-
-function encodeFolder(relativePath: string): string {
-  return relativePath.replace(/\//gu, "__SL__").replace(/\\/gu, "__BS__");
-}

packages/ingest-core/src/pipeline/paths.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "bun:test";
+import { metaId } from "./paths.ts";
+
+/**
+ * `metaId` — the deterministic, filesystem-safe storage-key hash that REPLACES `encodeMetaPath`. A single
+ * fixed-width component is immune to path depth (the ENAMETOOLONG fix on deeply-nested repos); the `id → path`
+ * inverse is persisted in `path-map.json` for the reverse.
+ */
+describe("metaId", () => {
+  it("is deterministic", () => {
+    expect(metaId("src/a/b.ts")).toBe(metaId("src/a/b.ts"));
+  });
+
+  it("is 64 lowercase hex chars", () => {
+    expect(metaId("src/a/b.ts")).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it("distinguishes paths the old lossy slug would collide", () => {
+    expect(metaId("a/b")).not.toBe(metaId("a_b"));
+    expect(metaId("a/b")).not.toBe(metaId("a/b/"));
+    expect(metaId("a/b")).not.toBe(metaId("a.b"));
+  });
+
+  it("normalizes backslashes to forward slashes (platform-independent)", () => {
+    expect(metaId("a\\b\\c")).toBe(metaId("a/b/c"));
+  });
+
+  it("stays 64 chars no matter how deep the path (the ENAMETOOLONG regression guard)", () => {
+    const deep = "x/".repeat(400) + "file.ts";
+    expect(metaId(deep)).toHaveLength(64);
+  });
+
+  it("matches the known SHA-256 vector (locks the algorithm)", () => {
+    expect(metaId("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+  });
+});

packages/ingest-core/src/pipeline/paths.ts

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import { mkdir } from "node:fs/promises";
 import path from "node:path";
 import { getBytebellHome, getConfigValue } from "@bb/config";
@@ -192,3 +193,17 @@ export function encodeMetaPath(relativePath: string): string {
 export function decodeMetaPath(encoded: string): string {
   return encoded.replace(ENCODED_SLASH_RE, "/").replace(ENCODED_BACKSLASH_RE, "\\");
 }
+
+/**
+ * Deterministic, collision-safe, filesystem-safe id for a repo-relative path (or any path-shaped key:
+ * folder paths, per-unit `<rel>__<qn>` / `<rel>:chunk-N__<qn>` ids). The storage-key scheme that REPLACES
+ * `encodeMetaPath`: a single 64-hex SHA-256 component is always well under NAME_MAX (255) no matter how
+ * deep the repo, so deeply-nested trees never hit `ENAMETOOLONG` (which `encodeMetaPath` does — it collapses
+ * the whole path into one component). Deterministic (not a UUID) so a record stays locatable by recomputing
+ * the hash and the cross-commit incremental cache still matches by name. Backslashes are normalised to `/`
+ * so a key hashes identically regardless of scan platform; nothing else is normalised (paths are
+ * case-sensitive — no lowercasing). The `<id> → path` inverse is persisted in `path-map.json` (see path-map.ts).
+ */
+export function metaId(key: string): string {
+  return crypto.createHash("sha256").update(key.replace(BACKSLASH_RE, "/")).digest("hex");
+}

packages/ingest-strategies/src/flat-folder/folder-summary-api.ts

@@ -7,7 +7,7 @@ import { Config } from "@bb/types";
 import { getConfigValue } from "@bb/config";
 import type { CondensedFileAnalysis } from "@bb/ingest-core";
 import type { MetaPaths } from "@bb/ingest-core";
-import { encodeMetaPath } from "@bb/ingest-core";
+import { metaId } from "@bb/ingest-core";
 import {
   FOLDER_ANALYSIS_SYSTEM_PROMPT,
   FOLDER_BATCH_SYSTEM_PROMPT,
@@ -183,7 +183,7 @@ export async function summariseFolderBatch(
 }
 
 export async function persistFolderSummary(metaPaths: MetaPaths, summary: FolderSummary): Promise<void> {
-  const file = path.join(metaPaths.folderSummariesDir, `${encodeMetaPath(summary.folderPath || "__ROOT__")}.json`);
+  const file = path.join(metaPaths.folderSummariesDir, `${metaId(summary.folderPath || "__ROOT__")}.json`);
   await writeFile(file, JSON.stringify(summary, null, 2), "utf8");
 }