diff --git a/packages/ingest-core/src/big-file/index.ts b/packages/ingest-core/src/big-file/index.ts index 13c7f70..fea18c5 100644 --- a/packages/ingest-core/src/big-file/index.ts +++ b/packages/ingest-core/src/big-file/index.ts @@ -5,6 +5,7 @@ import type { AskLlmOptions } from "@bb/llm"; import { logger } from "@bb/logger"; import type { ChunkAnalysisResult, HugeFileManifest } from "#src/types/big-file.ts"; import type { CondensedFileAnalysis } from "#src/types/condensed-file-analysis.ts"; +import { metaId } from "#src/pipeline/paths.ts"; import type { MetaPaths } from "#src/types/meta-paths.ts"; import type { ProgressContext } from "#src/progress/types.ts"; import { throwIfCancelled } from "#src/pipeline/cancellation.ts"; @@ -80,7 +81,7 @@ export async function processBigFile(input: ProcessBigFileInput): Promise `chunks/${encodeFolder(input.relativePath)}/chunk-${i}.json`); + const chunkPaths = chunks.map((_, i) => `chunks/${metaId(input.relativePath)}/chunk-${i}.json`); const totalTokenCount = chunks.reduce((acc, c) => acc + c.tokenCount, 0); const chunkInputTokens = results.reduce((acc, r) => acc + (r.tokenUsage?.inputTokens ?? 0), 0); @@ -122,7 +123,3 @@ export async function processBigFile(input: ProcessBigFileInput): Promise { diff --git a/packages/ingest-core/src/index.ts b/packages/ingest-core/src/index.ts index 79625bd..de6652d 100644 --- a/packages/ingest-core/src/index.ts +++ b/packages/ingest-core/src/index.ts @@ -64,6 +64,7 @@ export { orgRegistryDir, encodeMetaPath, decodeMetaPath, + metaId, } from "#src/pipeline/paths.ts"; export type { RepoLocation } from "#src/pipeline/paths.ts"; @@ -186,6 +187,15 @@ export { FileAnalysisCache } from "#src/file-analysis-cache.ts"; export { analyseScannedFile, buildOversizedStub } from "#src/analyse-file.ts"; export { directFolderOf, affectedFolderPaths } from "#src/folder-path.ts"; export { readScanManifest, writeScanManifest, emptyManifest } from "#src/scan-manifest.ts"; +export { + buildPathMap, + writePathMap, + readPathMap, + pathMapPath, + PATH_MAP_RELATIVE_PATH, + PATH_MAP_SCHEMA_VERSION, +} from "#src/path-map.ts"; +export type { PathMap } from "#src/path-map.ts"; export type { ScanManifest, ScanManifestEntry, ScanManifestSummary, ScanEntryKind } from "#src/scan-manifest.ts"; export { writeEligibleFiles, ELIGIBLE_FILES_RELATIVE_PATH } from "#src/eligible-files.ts"; export type { EligibleFilesDocument, WriteEligibleFilesInput } from "#src/eligible-files.ts"; diff --git a/packages/ingest-core/src/path-map.test.ts b/packages/ingest-core/src/path-map.test.ts new file mode 100644 index 0000000..de92abc --- /dev/null +++ b/packages/ingest-core/src/path-map.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "bun:test"; +import { buildPathMap } from "./path-map.ts"; +import { metaId } from "./pipeline/paths.ts"; +import type { ScanManifest } from "./scan-manifest.ts"; + +function manifest(paths: string[]): ScanManifest { + return { + generatedAt: "t", + summary: { + totalFiles: paths.length, + smallCount: paths.length, + bigCount: 0, + oversizedCount: 0, + totalTokens: 0, + estimatedBigChunks: 0, + }, + entries: paths.map((relativePath) => ({ + relativePath, + absolutePath: relativePath, + sizeBytes: 0, + tokenCount: 0, + kind: "small" as const, + })), + }; +} + +/** `buildPathMap` — the pure `id → path` inverse of `metaId`, built once from the scan manifest. */ +describe("buildPathMap", () => { + it("maps every file's metaId back to its path", () => { + const m = buildPathMap(manifest(["src/a.ts", "src/foo/b.ts"])); + expect(m.files[metaId("src/a.ts")]).toBe("src/a.ts"); + expect(m.files[metaId("src/foo/b.ts")]).toBe("src/foo/b.ts"); + }); + + it("includes every ancestor folder + the root sentinel", () => { + const m = buildPathMap(manifest(["src/foo/bar/b.ts"])); + expect(m.folders["__ROOT__"]).toBe(""); + expect(m.folders[metaId("src")]).toBe("src"); + expect(m.folders[metaId("src/foo")]).toBe("src/foo"); + expect(m.folders[metaId("src/foo/bar")]).toBe("src/foo/bar"); + }); + + it("produces one distinct key per distinct file (no collisions)", () => { + const paths = ["a/b.ts", "a_b.ts", "a.b.ts", "c/d.ts"]; + const m = buildPathMap(manifest(paths)); + expect(Object.keys(m.files)).toHaveLength(paths.length); + }); + + it("stamps the schema version + algo", () => { + const m = buildPathMap(manifest(["x.ts"])); + expect(m.version).toBe(1); + expect(m.algo).toBe("sha256"); + }); +}); diff --git a/packages/ingest-core/src/path-map.ts b/packages/ingest-core/src/path-map.ts new file mode 100644 index 0000000..580ac1a --- /dev/null +++ b/packages/ingest-core/src/path-map.ts @@ -0,0 +1,58 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { metaId } from "#src/pipeline/paths.ts"; +import type { ScanManifest } from "#src/scan-manifest.ts"; +import type { MetaPaths } from "#src/types/meta-paths.ts"; + +/** + * Persisted inverse of the `metaId` storage-key scheme (see pipeline/paths.ts). Every meta-output artifact + * dir is named `metaId()` — a one-way hash — so this `/path-map.json` + * records `id → path` for every file and every ancestor folder. Two roles: + * 1. External consumers (the VS Code extension) rebuild the repo's folder tree from it, then fetch each + * artifact by recomputing `metaId(path)`. + * 2. Its PRESENCE marks a metaRoot as written by the hash scheme ("v2"), which the incremental baseline + * gate uses to refuse seeding a new run from an old `__SL__`-encoded tree. + * Built ONCE from the scan manifest (all paths known before any fan-out worker starts), so no concurrency. + */ +export const PATH_MAP_RELATIVE_PATH = "path-map.json"; +export const PATH_MAP_SCHEMA_VERSION = 1; + +export interface PathMap { + version: number; + algo: "sha256"; + /** `metaId(relativePath) → relativePath` for every scanned file. */ + files: Record; + /** `metaId(folderPath) → folderPath` for every ancestor folder; the repo root is `"__ROOT__" → ""`. */ + folders: Record; +} + +/** PURE. Builds the `id → path` map for every file in the manifest plus every ancestor folder. */ +export function buildPathMap(manifest: ScanManifest): PathMap { + const files: Record = {}; + const folders: Record = { __ROOT__: "" }; + for (const entry of manifest.entries) { + files[metaId(entry.relativePath)] = entry.relativePath; + const parts = entry.relativePath.split("/"); + for (let i = 1; i < parts.length; i += 1) { + const folder = parts.slice(0, i).join("/"); + folders[metaId(folder)] = folder; + } + } + return { version: PATH_MAP_SCHEMA_VERSION, algo: "sha256", files, folders }; +} + +export function pathMapPath(metaPaths: MetaPaths): string { + return path.join(metaPaths.metaOutputRoot, PATH_MAP_RELATIVE_PATH); +} + +export async function writePathMap(metaPaths: MetaPaths, map: PathMap): Promise { + await writeFile(pathMapPath(metaPaths), JSON.stringify(map, null, 2), "utf8"); +} + +export async function readPathMap(metaPaths: MetaPaths): Promise { + try { + return JSON.parse(await readFile(pathMapPath(metaPaths), "utf8")) as PathMap; + } catch { + return null; + } +} diff --git a/packages/ingest-core/src/phases/analyse-big-files.ts b/packages/ingest-core/src/phases/analyse-big-files.ts index 18c18d6..71d8a45 100644 --- a/packages/ingest-core/src/phases/analyse-big-files.ts +++ b/packages/ingest-core/src/phases/analyse-big-files.ts @@ -4,6 +4,7 @@ import { Config } from "@bb/types"; import { getConfigValue } from "@bb/config"; import type { AskLlmOptions } from "@bb/llm"; import { LlmConfigError, LlmError } from "@bb/errors"; +import { metaId } from "#src/pipeline/paths.ts"; import type { MetaPaths } from "#src/types/meta-paths.ts"; import type { AnalyzedFileResult, SourceReader } from "#src/types/pipeline.ts"; import type { ProgressContext } from "#src/progress/types.ts"; @@ -163,7 +164,7 @@ export async function analyseBigFiles(input: AnalyseBigFilesInput): Promise `chunks/${encodeFolder(state.entry.relativePath)}/chunk-${i}.json`), + chunkPaths: state.chunks.map((_, i) => `chunks/${metaId(state.entry.relativePath)}/chunk-${i}.json`), generatedAt: new Date().toISOString(), }; await saveManifest(input.metaPaths, manifest); @@ -281,7 +282,3 @@ export async function analyseBigFiles(input: AnalyseBigFilesInput): Promise { + it("is deterministic", () => { + expect(metaId("src/a/b.ts")).toBe(metaId("src/a/b.ts")); + }); + + it("is 64 lowercase hex chars", () => { + expect(metaId("src/a/b.ts")).toMatch(/^[0-9a-f]{64}$/); + }); + + it("distinguishes paths the old lossy slug would collide", () => { + expect(metaId("a/b")).not.toBe(metaId("a_b")); + expect(metaId("a/b")).not.toBe(metaId("a/b/")); + expect(metaId("a/b")).not.toBe(metaId("a.b")); + }); + + it("normalizes backslashes to forward slashes (platform-independent)", () => { + expect(metaId("a\\b\\c")).toBe(metaId("a/b/c")); + }); + + it("stays 64 chars no matter how deep the path (the ENAMETOOLONG regression guard)", () => { + const deep = "x/".repeat(400) + "file.ts"; + expect(metaId(deep)).toHaveLength(64); + }); + + it("matches the known SHA-256 vector (locks the algorithm)", () => { + expect(metaId("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + }); +}); diff --git a/packages/ingest-core/src/pipeline/paths.ts b/packages/ingest-core/src/pipeline/paths.ts index 88f76d9..6c0fa18 100644 --- a/packages/ingest-core/src/pipeline/paths.ts +++ b/packages/ingest-core/src/pipeline/paths.ts @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import { mkdir } from "node:fs/promises"; import path from "node:path"; import { getBytebellHome, getConfigValue } from "@bb/config"; @@ -192,3 +193,17 @@ export function encodeMetaPath(relativePath: string): string { export function decodeMetaPath(encoded: string): string { return encoded.replace(ENCODED_SLASH_RE, "/").replace(ENCODED_BACKSLASH_RE, "\\"); } + +/** + * Deterministic, collision-safe, filesystem-safe id for a repo-relative path (or any path-shaped key: + * folder paths, per-unit `__` / `:chunk-N__` ids). The storage-key scheme that REPLACES + * `encodeMetaPath`: a single 64-hex SHA-256 component is always well under NAME_MAX (255) no matter how + * deep the repo, so deeply-nested trees never hit `ENAMETOOLONG` (which `encodeMetaPath` does — it collapses + * the whole path into one component). Deterministic (not a UUID) so a record stays locatable by recomputing + * the hash and the cross-commit incremental cache still matches by name. Backslashes are normalised to `/` + * so a key hashes identically regardless of scan platform; nothing else is normalised (paths are + * case-sensitive — no lowercasing). The ` → path` inverse is persisted in `path-map.json` (see path-map.ts). + */ +export function metaId(key: string): string { + return crypto.createHash("sha256").update(key.replace(BACKSLASH_RE, "/")).digest("hex"); +} diff --git a/packages/ingest-strategies/src/flat-folder/folder-summary-api.ts b/packages/ingest-strategies/src/flat-folder/folder-summary-api.ts index 6a18350..8f38bd9 100644 --- a/packages/ingest-strategies/src/flat-folder/folder-summary-api.ts +++ b/packages/ingest-strategies/src/flat-folder/folder-summary-api.ts @@ -7,7 +7,7 @@ import { Config } from "@bb/types"; import { getConfigValue } from "@bb/config"; import type { CondensedFileAnalysis } from "@bb/ingest-core"; import type { MetaPaths } from "@bb/ingest-core"; -import { encodeMetaPath } from "@bb/ingest-core"; +import { metaId } from "@bb/ingest-core"; import { FOLDER_ANALYSIS_SYSTEM_PROMPT, FOLDER_BATCH_SYSTEM_PROMPT, @@ -183,7 +183,7 @@ export async function summariseFolderBatch( } export async function persistFolderSummary(metaPaths: MetaPaths, summary: FolderSummary): Promise { - const file = path.join(metaPaths.folderSummariesDir, `${encodeMetaPath(summary.folderPath || "__ROOT__")}.json`); + const file = path.join(metaPaths.folderSummariesDir, `${metaId(summary.folderPath || "__ROOT__")}.json`); await writeFile(file, JSON.stringify(summary, null, 2), "utf8"); }