From 03035672a1d0d6b4b6771714dd8569162e54e111 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 07:49:14 +0800 Subject: [PATCH] fix(analyzer): stop hangs on large files and ground detection in project evidence Stability - ts-morph symbol lines come from a LineIndex. getStartLineNumber() recounts newlines from the top of the file on every call, so a 4 MB file with 50k functions never finished; it now takes about 3 s. - The four import/export regexes take their clause from IMPORT_CLAUSE. The old unbounded [^'"]+ rescanned a quote-free file from every `export` (20k exports cost 15 s). - camelCase splitting uses lookarounds: 100k capitals cost 3.4 s, now none. - Scanner: files over 5 MB are listed but not read (reported in warnings, size goes into the fingerprint), so six 400 MB data files no longer exhaust the heap. Symlinks that resolve outside the project are skipped, lines are counted without splitting. - Every text sort goes through compareText (one Intl.Collator). localeCompare follows LANG and changed snapshots under cs_CZ and tr_TR. Grounding - Language is "unknown" for projects without TS/JS files (Django was typescript). - A server framework that is only a devDependency counts when production code imports it (ky was reported as an Express api-service). - .d.ts files are never entry points; test-d/ is non-production source. - workspaceType comes from declared members (workspaces, pnpm-workspace.yaml packages), not from the file existing or a second package.json. - Registry signals keep only the search terms found in the project's evidence. - Duplicate feature names get a folder hint and agent-navigation ids are unique, so no feature map is silently dropped. - Domain inference validates the model answer and the cache with zod, only keeps features that name known entities, and merges on a copy. - The snapshot fingerprint includes the DevMap version. 30 new tests (538 total). 13 of the 18 grounding tests fail on main; the 12 stability tests need the new modules. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01AgTNRWDMVo2d462doRNBUc --- CLAUDE.md | 10 +- packages/cli/src/ai/contextBuilder.ts | 5 +- packages/cli/src/ai/custom.ts | 3 +- packages/cli/src/ai/groq.ts | 3 +- .../cli/src/analyzers/analysis/fileScanner.ts | 46 ++- .../src/analyzers/analysis/importClause.ts | 8 + packages/cli/src/analyzers/analysis/index.ts | 1 + .../cli/src/analyzers/analysis/lineIndex.ts | 26 ++ .../src/analyzers/analysis/tsMorphAnalyzer.ts | 40 ++- .../analyzers/detectors/frameworkDetector.ts | 47 ++- .../detectors/frontendFeatureDetector.ts | 3 +- .../analyzers/detectors/nestRouteDetector.ts | 3 +- .../src/analyzers/detectors/routeDetector.ts | 5 +- .../cli/src/analyzers/detectors/routeUtils.ts | 3 +- .../analyzers/detectors/serviceDetector.ts | 6 +- .../analyzers/detectors/trpcRouteDetector.ts | 3 +- .../analyzers/features/featureCandidates.ts | 58 +++- .../src/analyzers/features/featureDetector.ts | 43 ++- .../cli/src/analyzers/graph/aliasResolver.ts | 5 +- .../src/analyzers/graph/dependencyGraph.ts | 6 +- .../cli/src/analyzers/graph/entryPoints.ts | 9 +- .../src/analyzers/graph/nuxtAutoImports.ts | 5 +- .../src/analyzers/graph/packageManifests.ts | 3 +- .../cli/src/analyzers/graph/sourceScope.ts | 1 + .../analyzers/inference/domainInference.ts | 108 ++++-- .../analyzers/pipeline/heuristicAnalyzer.ts | 11 +- .../src/analyzers/pipeline/languageReaders.ts | 3 +- .../cli/src/analyzers/pipeline/projectMap.ts | 54 ++- .../src/analyzers/pipeline/projectMetadata.ts | 148 ++++++-- packages/cli/src/cache/agentNavigation.ts | 44 +-- packages/cli/src/utils/compare.ts | 10 + packages/cli/test/analyzer-grounding.test.ts | 326 ++++++++++++++++++ packages/cli/test/analyzer-stability.test.ts | 235 +++++++++++++ 33 files changed, 1108 insertions(+), 173 deletions(-) create mode 100644 packages/cli/src/analyzers/analysis/importClause.ts create mode 100644 packages/cli/src/analyzers/analysis/lineIndex.ts create mode 100644 packages/cli/src/utils/compare.ts create mode 100644 packages/cli/test/analyzer-grounding.test.ts create mode 100644 packages/cli/test/analyzer-stability.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index caf2346..3d022bd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ devmap/ │ │ ├── mcp/ ← MCP server: tools, resources, prompts, snapshot provider, refresher + lock │ │ ├── onboarding/ ← Onboarding model and builder │ │ └── utils/ ← Config, errors, output, prompts, markdown renderer -│ ├── test/ ← 49 test files (`*.test.ts`, auto-discovered by `run-tests.ts`) + `mcp-smoke.mjs`, `package-e2e.mjs` +│ ├── test/ ← 51 test files (`*.test.ts`, auto-discovered by `run-tests.ts`) + `mcp-smoke.mjs`, `package-e2e.mjs` │ └── package.json ├── apps/web/ ← Astro marketing site (unrelated to CLI) ├── docs/ ← Public documentation @@ -129,11 +129,12 @@ Order for a non-relative specifier: `#subpath` (package.json `imports`) → tsco ## Testing -49 test files in `packages/cli/test/`. Key covered areas: +51 test files in `packages/cli/test/`. Key covered areas: - Analyzers, AI clients, agent navigation, entity extractors - Dependency graph: aliases, workspace packages, `#imports`, Nuxt auto-imports/aliases, entry points (`graph-resolution`, `dependency-graph-alias`, `js-ts-ecosystem`, `js-ts-frameworks`) - Feature similarity/merge, frontend features, framework routes (`framework-routes`; Remix/React Router/TanStack/Expo/Hono and bundler aliases in `js-ts-frameworks`; client-side routers, Hono factories, Next normalization and critical-file ranking in `js-ts-gaps`; SolidStart, Qwik City, Angular, Elysia, Koa, AdonisJS and constant-resolved paths in `js-ts-more-frameworks`), detector fixes - Heuristic language readers (`heuristic-languages`), scanner robustness, project scope/metadata +- Analyzer stability and grounding (`analyzer-stability`: linear time on generated files, quote-free text, long identifiers, size cap, symlinks, locale-free ordering; `analyzer-grounding`: language, devDependency frameworks, workspaces, entry points, search terms, duplicate feature names, domain-inference validation) - Commands (init, analyze, map, flow, explain, onboarding, config, doctor); `analyze-lock` covers `devmap analyze` holding `.devmap/analyze.lock` - MCP server (`core-api`, `mcp-server`, `mcp-freshness`, `mcp-explain`): tools, resources, prompts, stale/missing snapshots, auto-refresh and lock, offline-never-touches-AI, stdio process with a clean protocol stream. `test/mcp-smoke.mjs` runs the compiled `dist/` as a server (CI and publish workflow) - Utils (markdown terminal, JSON output, prompts, errors) @@ -174,6 +175,8 @@ MCP server (after `pnpm build:cli`): `node packages/cli/test/mcp-smoke.mjs`. - Domain inference is cached for idempotency (`.devmap/domain-cache.json`) - Snapshot is regenerated on every `devmap analyze` run - `AGENTS.md` is never overwritten by init (append-only) +- A file over `MAX_READ_BYTES` (5 MB) is listed with `oversize: true` and no content, and `warnings` names it; its fingerprint input is its size. Symlinks that resolve outside the project root are skipped +- Text is ordered with `compareText` (`utils/compare.ts`), never `String#localeCompare`, which follows `LANG` and changes snapshots between machines; a test fails on any use - `analyzeCommand` (CLI) holds `.devmap/analyze.lock` around `runAnalyze`; `runAnalyze` itself never takes it, because the MCP refresher already holds the lock when it calls it - `.devmap/` is always gitignored - `devmap mcp` owns stdout: it is the JSON-RPC stream. Never `console.log` from code the MCP server can reach; use `output.*` inside `withJsonOutput()` (silent) or stderr @@ -189,6 +192,7 @@ MCP server (after `pnpm build:cli`): `node packages/cli/test/mcp-smoke.mjs`. - Client router detection reads literal route arrays, JSX trees and `getParentRoute` chains; routes assembled by loops, spreads of computed values or runtime config are not followed - Elysia, Koa and Adonis detectors follow literal call chains and mounts; routers built inside factory functions, `.use()` of computed values and runtime-assembled paths are not followed. Angular reads literal `Routes` arrays and `loadChildren` imports - Feature similarity threshold (0.35) is tuned but may cause false merges +- Feature `confidence` measures parse quality, not evidence strength: `projectMap.ts` lifts any feature with two files, one parsed by ts-morph, to `high`, so a folder named `search/` reads as `high` like an imported vendor package does - Domain inference is single-shot — no conversation context - The MCP staleness check re-scans project files (cached for 10 s), so it is not free on very large repositories @@ -204,6 +208,8 @@ MCP server (after `pnpm build:cli`): `node packages/cli/test/mcp-smoke.mjs`. - `.js` → `.ts` extension swap in import resolution (shared in `graph/localPaths.ts`) - Database detection ranks ORM/BaaS above drivers; a cache (Redis) is only ever listed as `additional` - Singularize entity names for feature matching +- Never use `getStartLineNumber()` in a loop over a file's nodes: ts-morph recounts newlines from the top each call. Use `LineIndex` (`analysis/lineIndex.ts`) +- Regexes that read `import`/`export ... from` take their clause from `IMPORT_CLAUSE` (`analysis/importClause.ts`); an unbounded `[^'"]+` rescans a quote-free file from every `export` - REASON_TAGS constant for critical file ranking reasoning - Critical-file execution bonus (`projectMap.ts`) is name-based (router, controller, orchestrator, engine, dispatcher, scheduler, pipeline, registry, kernel, `commands/`) or fan-out-based (≥ 8 distinct local imports in a non-index file) — never tied to devmap's own file names - Angular, Elysia, Koa and Adonis detectors share `moduleBindings.ts` (`createModuleIndex`): imports/`require`, exports and re-exports, and `stringOf` (literals, templates, `+`, constants, `as const` objects and enums across files). Instances (Elysia/Koa) are keyed by `new` expression and only unmounted ones are walked from the top, like Hono diff --git a/packages/cli/src/ai/contextBuilder.ts b/packages/cli/src/ai/contextBuilder.ts index 5617130..acb3530 100644 --- a/packages/cli/src/ai/contextBuilder.ts +++ b/packages/cli/src/ai/contextBuilder.ts @@ -1,6 +1,7 @@ import { readFile, realpath } from "node:fs/promises"; import { isAbsolute, relative, resolve } from "node:path"; import type { ProjectMap } from "../analyzers/pipeline/index.js"; +import { compareText } from "../utils/compare.js"; const DEFAULT_MAX_FILES = 5; const DEFAULT_MAX_LINES_PER_FILE = 200; @@ -445,7 +446,7 @@ function rankContextFiles( .sort((left, right) => right.score - left.score || Number(right.direct) - Number(left.direct) - || left.path.localeCompare(right.path) + || compareText(left.path, right.path) ); } @@ -918,7 +919,7 @@ function tokenizeQuestion(value: string): string[] { function splitSearchTerms(value: string): Set { const spaced = value .replace(/([a-z0-9])([A-Z])/g, "$1 $2") - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2"); + .replace(/(?<=[A-Z])(?=[A-Z][a-z])/g, " "); return new Set(tokenizeQuestion(spaced)); } diff --git a/packages/cli/src/ai/custom.ts b/packages/cli/src/ai/custom.ts index 50dd36f..01799c0 100644 --- a/packages/cli/src/ai/custom.ts +++ b/packages/cli/src/ai/custom.ts @@ -10,6 +10,7 @@ import { parseSseStream, readCompletionPayload } from "./openaiCompatibleStream.js"; +import { compareText } from "../utils/compare.js"; const CUSTOM_PAYLOAD_MESSAGES = { unreadable: () => new DevmapError( @@ -220,7 +221,7 @@ async function readModelIds(response: Response): Promise { return (payload.data ?? []) .map((model) => model.id) .filter((id): id is string => typeof id === "string") - .sort((a, b) => a.localeCompare(b)); + .sort((a, b) => compareText(a, b)); } catch { throw new DevmapError( "The custom endpoint returned an unreadable model list.", diff --git a/packages/cli/src/ai/groq.ts b/packages/cli/src/ai/groq.ts index 4497296..cf3a9af 100644 --- a/packages/cli/src/ai/groq.ts +++ b/packages/cli/src/ai/groq.ts @@ -12,6 +12,7 @@ import { type PayloadUnreadableMessages, type StreamInterruptedMessages } from "./openaiCompatibleStream.js"; +import { compareText } from "../utils/compare.js"; const GROQ_MODELS_URL = "https://api.groq.com/openai/v1/models"; const GROQ_CHAT_COMPLETIONS_URL = "https://api.groq.com/openai/v1/chat/completions"; @@ -446,7 +447,7 @@ async function readModelIds(response: Response): Promise { if (aIndex !== -1 && bIndex !== -1) return aIndex - bIndex; if (aIndex !== -1) return -1; if (bIndex !== -1) return 1; - return a.localeCompare(b); + return compareText(a, b); }); } catch { throw new DevmapError( diff --git a/packages/cli/src/analyzers/analysis/fileScanner.ts b/packages/cli/src/analyzers/analysis/fileScanner.ts index 99e0cb8..81951c6 100644 --- a/packages/cli/src/analyzers/analysis/fileScanner.ts +++ b/packages/cli/src/analyzers/analysis/fileScanner.ts @@ -1,8 +1,9 @@ import type { Dirent, Stats } from "node:fs"; -import { readdir, readFile, stat } from "node:fs/promises"; -import { extname, join, relative } from "node:path"; +import { readdir, readFile, realpath, stat } from "node:fs/promises"; +import { extname, join, relative, sep } from "node:path"; import pLimit from "p-limit"; import { clearGitignoreCache, shouldIgnorePath } from "./filterEngine.js"; +import { compareText } from "../../utils/compare.js"; export type ScannedFile = { path: string; @@ -11,11 +12,21 @@ export type ScannedFile = { size: number; lines: number; content: string; + /** The file is larger than MAX_READ_BYTES: it is listed, but its content was not read. */ + oversize?: boolean; }; +/** + * Past this size a text file is a dataset, a dump or a generated bundle, and + * reading it whole is what exhausts the heap: six 400 MB CSV files in one + * project crashed the process before any analysis started. + */ +export const MAX_READ_BYTES = 5 * 1024 * 1024; + export async function scanFiles(projectRoot: string): Promise { const files: ScannedFile[] = []; const limit = pLimit(50); + const realRoot = await realpath(projectRoot).catch(() => projectRoot); // Ignore files are cached per process; re-read them for every scan. clearGitignoreCache(); @@ -34,7 +45,7 @@ export async function scanFiles(projectRoot: string): Promise { if (isRoot) throw error; return; } - entries.sort((a, b) => a.name.localeCompare(b.name)); + entries.sort((a, b) => compareText(a.name, b.name)); const tasks: Promise[] = []; @@ -62,16 +73,20 @@ export async function scanFiles(projectRoot: string): Promise { return; } if (!metadata.isFile()) return; + // A link that leaves the project would put someone else's file in the map. + if (entry.isSymbolicLink() && !(await resolvesInside(absolutePath, realRoot))) return; - const content = await readFile(absolutePath, "utf8").catch(() => ""); + const oversize = metadata.size > MAX_READ_BYTES; + const content = oversize ? "" : await readFile(absolutePath, "utf8").catch(() => ""); files.push({ path: relativePath, absolutePath, extension: extname(entry.name).toLowerCase(), size: metadata.size, - lines: content.length === 0 ? 0 : content.split(/\r?\n/).length, - content + lines: countLines(content), + content, + ...(oversize ? { oversize: true } : {}) }); })); } @@ -85,5 +100,22 @@ export async function scanFiles(projectRoot: string): Promise { // I/O completion timing, not directory structure. Sort here so every // downstream consumer (fileIndex ordering, onboarding path ties, etc.) // gets a stable, reproducible order across runs on an unchanged project. - return files.sort((a, b) => a.path.localeCompare(b.path)); + return files.sort((a, b) => compareText(a.path, b.path)); +} + +async function resolvesInside(path: string, realRoot: string): Promise { + try { + const target = await realpath(path); + return target === realRoot || target.startsWith(realRoot + sep); + } catch { + return false; + } +} + +// Same count as content.split(/\r?\n/).length without allocating a string per line. +function countLines(content: string): number { + if (content.length === 0) return 0; + let lines = 1; + for (let at = content.indexOf("\n"); at !== -1; at = content.indexOf("\n", at + 1)) lines += 1; + return lines; } diff --git a/packages/cli/src/analyzers/analysis/importClause.ts b/packages/cli/src/analyzers/analysis/importClause.ts new file mode 100644 index 0000000..dac944f --- /dev/null +++ b/packages/cli/src/analyzers/analysis/importClause.ts @@ -0,0 +1,8 @@ +/** + * What may sit between `import`/`export` and `from`: names, braces, commas, `*`, + * `as` and comments, never a quote, `;`, `=`, `(`, `)` or `:`. Stopping there and + * capping the length keeps a quote-free file (generated types, no-semicolon + * style) from rescanning to the end of the text at every `export`; the unbounded + * `[^'"]+` it replaces made a file of 12,000 exported functions cost 13 seconds. + */ +export const IMPORT_CLAUSE = String.raw`[^'";:=()]{1,4000}`; diff --git a/packages/cli/src/analyzers/analysis/index.ts b/packages/cli/src/analyzers/analysis/index.ts index 16f4386..ff4a995 100644 --- a/packages/cli/src/analyzers/analysis/index.ts +++ b/packages/cli/src/analyzers/analysis/index.ts @@ -1,5 +1,6 @@ export { scanFiles, + MAX_READ_BYTES, } from "./fileScanner.js"; export type { ScannedFile, diff --git a/packages/cli/src/analyzers/analysis/lineIndex.ts b/packages/cli/src/analyzers/analysis/lineIndex.ts new file mode 100644 index 0000000..f0a2f7c --- /dev/null +++ b/packages/cli/src/analyzers/analysis/lineIndex.ts @@ -0,0 +1,26 @@ +/** + * Line numbers for many nodes of one file. ts-morph's `getStartLineNumber()` + * recounts newlines from the top of the text on every call, so a generated file + * with n declarations costs O(n * size) and a 4 MB file runs for minutes. + */ +export class LineIndex { + private readonly breaks: number[] = []; + + constructor(text: string) { + for (let at = text.indexOf("\n"); at !== -1; at = text.indexOf("\n", at + 1)) { + this.breaks.push(at); + } + } + + /** 1-based line of the character at `position`: one more than the newlines before it. */ + lineAt(position: number): number { + let low = 0; + let high = this.breaks.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (this.breaks[middle] < position) low = middle + 1; + else high = middle; + } + return low + 1; + } +} diff --git a/packages/cli/src/analyzers/analysis/tsMorphAnalyzer.ts b/packages/cli/src/analyzers/analysis/tsMorphAnalyzer.ts index 30238d1..0b932fc 100644 --- a/packages/cli/src/analyzers/analysis/tsMorphAnalyzer.ts +++ b/packages/cli/src/analyzers/analysis/tsMorphAnalyzer.ts @@ -14,8 +14,10 @@ import type { SymbolInfo } from "./fileAnalysis.js"; import type { ScannedFile } from "./fileScanner.js"; +import { LineIndex } from "./lineIndex.js"; import { AstroPreprocessor, SveltePreprocessor, VuePreprocessor } from "./preprocessors/index.js"; import type { LanguagePreprocessor } from "./preprocessors/index.js"; +import { compareText } from "../../utils/compare.js"; // Native extensions — ts-morph parses these directly, no preprocessing needed. const NATIVE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx"]); @@ -83,6 +85,8 @@ export class TsMorphAnalyzer implements FileAnalyzer { _originalPath: string ): FileAnalysis { const sourceFile = this.project.createSourceFile(parsePath, content, { overwrite: true }); + const lines = new LineIndex(sourceFile.getFullText()); + const lineOf = (node: Node) => lines.lineAt(node.getStart()); // --- Imports --- const imports = new Set(); @@ -122,7 +126,7 @@ export class TsMorphAnalyzer implements FileAnalyzer { // --- Exports --- // `module.exports = { a, b }` and `exports.c = ...` are what most Node // packages and older JS codebases export; the ESM view above sees none of it. - const commonJs = readCommonJsExports(sourceFile); + const commonJs = readCommonJsExports(sourceFile, lineOf); const exports = [...new Set([ ...sourceFile.getExportedDeclarations().keys(), ...commonJs.map((entry) => entry.name) @@ -138,7 +142,7 @@ export class TsMorphAnalyzer implements FileAnalyzer { symbols.push({ name: statement.getNameOrThrow(), kind: "function", - line: statement.getStartLineNumber(), + line: lineOf(statement), exported: statement.isExported() || statement.isDefaultExport(), async: statement.isAsync() }); @@ -150,7 +154,7 @@ export class TsMorphAnalyzer implements FileAnalyzer { symbols.push({ name: statement.getNameOrThrow(), kind: "class", - line: statement.getStartLineNumber(), + line: lineOf(statement), exported: classExported }); for (const method of statement.getMethods()) { @@ -158,7 +162,7 @@ export class TsMorphAnalyzer implements FileAnalyzer { symbols.push({ name: method.getName(), kind: "method", - line: method.getStartLineNumber(), + line: lineOf(method), exported: false, async: method.isAsync() }); @@ -167,17 +171,17 @@ export class TsMorphAnalyzer implements FileAnalyzer { } if (Node.isInterfaceDeclaration(statement)) { - symbols.push(createDeclarationSymbol(statement, "interface")); + symbols.push(createDeclarationSymbol(statement, "interface", lineOf(statement))); continue; } if (Node.isTypeAliasDeclaration(statement)) { - symbols.push(createDeclarationSymbol(statement, "type")); + symbols.push(createDeclarationSymbol(statement, "type", lineOf(statement))); continue; } if (Node.isEnumDeclaration(statement)) { - symbols.push(createDeclarationSymbol(statement, "enum")); + symbols.push(createDeclarationSymbol(statement, "enum", lineOf(statement))); continue; } @@ -197,7 +201,7 @@ export class TsMorphAnalyzer implements FileAnalyzer { symbols.push({ name, kind: "const", - line: declaration.getStartLineNumber(), + line: lineOf(declaration), exported: statement.isExported(), async: isAsyncFunction }); @@ -206,8 +210,12 @@ export class TsMorphAnalyzer implements FileAnalyzer { } } + const symbolsByName = new Map(); + for (const symbol of symbols) { + if (!symbolsByName.has(symbol.name)) symbolsByName.set(symbol.name, symbol); + } for (const entry of commonJs) { - const existing = symbols.find((symbol) => symbol.name === (entry.alias ?? entry.name)); + const existing = symbolsByName.get(entry.alias ?? entry.name); if (existing) { existing.exported = true; } else if (entry.name !== "default") { @@ -220,7 +228,7 @@ export class TsMorphAnalyzer implements FileAnalyzer { }); } } - symbols.sort((left, right) => left.line - right.line || left.name.localeCompare(right.name)); + symbols.sort((left, right) => left.line - right.line || compareText(left.name, right.name)); // --- Top functions --- const topFunctions = symbols @@ -276,7 +284,7 @@ interface CommonJsExport { * Top-level `module.exports = ...`, `module.exports.x = ...` and `exports.x = ...`. * An object literal contributes its keys; anything else is the default export. */ -function readCommonJsExports(sourceFile: SourceFile): CommonJsExport[] { +function readCommonJsExports(sourceFile: SourceFile, lineOf: (node: Node) => number): CommonJsExport[] { const found: CommonJsExport[] = []; const seen = new Set(); @@ -290,7 +298,7 @@ function readCommonJsExports(sourceFile: SourceFile): CommonJsExport[] { name, ...(name === "default" && value && Node.isIdentifier(value) ? { alias: value.getText() } : {}), kind: callable || (value && Node.isClassExpression(value)) ? "function" : "const", - line: node.getStartLineNumber(), + line: lineOf(node), async: Boolean(callable && (value as { isAsync(): boolean }).isAsync()) }); }; @@ -368,16 +376,16 @@ function emptyAnalysis( function createDeclarationSymbol( declaration: { getName(): string; - getStartLineNumber(): number; isExported(): boolean; isDefaultExport(): boolean; }, - kind: "interface" | "type" | "enum" + kind: "interface" | "type" | "enum", + line: number ): SymbolInfo { return { name: declaration.getName(), kind, - line: declaration.getStartLineNumber(), + line, exported: declaration.isExported() || declaration.isDefaultExport() }; } @@ -385,5 +393,5 @@ function createDeclarationSymbol( function compareFunctions(left: FunctionInfo, right: FunctionInfo): number { return Number(right.exported) - Number(left.exported) || left.line - right.line - || left.name.localeCompare(right.name); + || compareText(left.name, right.name); } diff --git a/packages/cli/src/analyzers/detectors/frameworkDetector.ts b/packages/cli/src/analyzers/detectors/frameworkDetector.ts index 0816b53..f0435cb 100644 --- a/packages/cli/src/analyzers/detectors/frameworkDetector.ts +++ b/packages/cli/src/analyzers/detectors/frameworkDetector.ts @@ -38,23 +38,30 @@ export function detectFramework(files: ScannedFile[]): Framework { */ export function detectFrameworks(files: ScannedFile[]): DetectedFramework[] { const detected = new Set(); + const sourceFiles = files.filter((file) => isArchitectureSource(file.path)); // Phase 1: dependency manifest — highest confidence signal. // package.json is the canonical source of truth for what frameworks are installed. - for (const dependencies of readManifestDependencies(files)) { + for (const manifest of readManifestDependencies(files)) { + const dependencies = { ...manifest.runtime, ...manifest.dev }; + // A server framework that is only a devDependency is usually test tooling + // (a mock server in a library's tests), so production code has to import it. + const serverDeclared = (...names: string[]) => names.some((name) => + name in manifest.runtime || (name in manifest.dev && importsPackage(sourceFiles, name)) + ); if ("next" in dependencies) detected.add("nextjs"); - if ("express" in dependencies) detected.add("express"); + if (serverDeclared("express")) detected.add("express"); if ("astro" in dependencies) detected.add("astro"); if ("nuxt" in dependencies) detected.add("nuxt"); - if ("fastify" in dependencies) detected.add("fastify"); - if ("@nestjs/core" in dependencies || "@nestjs/common" in dependencies) { + if (serverDeclared("fastify")) detected.add("fastify"); + if (serverDeclared("@nestjs/core", "@nestjs/common")) { detected.add("nestjs"); } if ("@sveltejs/kit" in dependencies) detected.add("sveltekit"); - if ("hono" in dependencies) detected.add("hono"); - if ("elysia" in dependencies) detected.add("elysia"); - if ("koa" in dependencies) detected.add("koa"); - if (ADONIS_PACKAGES.some((name) => name in dependencies)) detected.add("adonisjs"); + if (serverDeclared("hono")) detected.add("hono"); + if (serverDeclared("elysia")) detected.add("elysia"); + if (serverDeclared("koa")) detected.add("koa"); + if (serverDeclared(...ADONIS_PACKAGES)) detected.add("adonisjs"); // SolidStart, Qwik City and Angular are full frameworks with their own router; // plain `solid-js` / `@builder.io/qwik` are just component libraries. @@ -111,7 +118,6 @@ export function detectFrameworks(files: ScannedFile[]): DetectedFramework[] { // Phase 2: file structure heuristics — secondary signal. // Only used when package.json is absent or incomplete (e.g. monorepo root without deps). - const sourceFiles = files.filter((file) => isArchitectureSource(file.path)); // Next.js: next.config.* or App Router / Pages Router file conventions. // _app and _document are Next.js-unique; a bare src/pages/api/ folder is NOT @@ -259,7 +265,12 @@ function hasExpressCallSite(content: string): boolean { || /from\s+["']express["']/.test(content); } -function readManifestDependencies(files: ScannedFile[]): Array> { +type ManifestDependencies = { + runtime: Record; + dev: Record; +}; + +function readManifestDependencies(files: ScannedFile[]): ManifestDependencies[] { return files .filter((file) => file.path.endsWith("package.json") && isArchitectureSource(file.path) @@ -267,7 +278,7 @@ function readManifestDependencies(files: ScannedFile[]): Array readDependencies(file.content)); } -function readDependencies(content: string): Record { +function readDependencies(content: string): ManifestDependencies { try { const parsed = parseJsonText(content) as { dependencies?: Record; @@ -275,10 +286,18 @@ function readDependencies(content: string): Record { }; return { - ...parsed.dependencies, - ...parsed.devDependencies + runtime: { ...parsed.dependencies }, + dev: { ...parsed.devDependencies } }; } catch { - return {}; + return { runtime: {}, dev: {} }; } } + +const CODE_FILE = /\.[cm]?[jt]sx?$|\.vue$|\.svelte$|\.astro$/; + +function importsPackage(sourceFiles: ScannedFile[], name: string): boolean { + const escaped = name.replace(/\W/g, "\\$&"); + const pattern = new RegExp(`(?:from\\s*|require\\(\\s*|import\\s*\\(\\s*|import\\s+)["']${escaped}["'/]`); + return sourceFiles.some((file) => CODE_FILE.test(file.path) && pattern.test(file.content)); +} diff --git a/packages/cli/src/analyzers/detectors/frontendFeatureDetector.ts b/packages/cli/src/analyzers/detectors/frontendFeatureDetector.ts index db2bdca..0b4cac5 100644 --- a/packages/cli/src/analyzers/detectors/frontendFeatureDetector.ts +++ b/packages/cli/src/analyzers/detectors/frontendFeatureDetector.ts @@ -5,6 +5,7 @@ import { buildReverseGraph } from "../graph/index.js"; import { singularize } from "../analysis/extractors/fallbackExtractor.js"; import type { FeatureInfo } from "../features/featureDetector.js"; import { isDynamicSegment } from "./nonResourceSegments.js"; +import { compareText } from "../../utils/compare.js"; // --------------------------------------------------------------------------- // Constants @@ -369,7 +370,7 @@ export function detectClientRouteFeatures( /** The page closest to the segment root first: `room/[id]/page.tsx` before `room/[id]/info/page.tsx`. */ function byShallowPath(files: string[]): string[] { return [...files].sort((left, right) => - left.split("/").length - right.split("/").length || left.localeCompare(right) + left.split("/").length - right.split("/").length || compareText(left, right) ); } diff --git a/packages/cli/src/analyzers/detectors/nestRouteDetector.ts b/packages/cli/src/analyzers/detectors/nestRouteDetector.ts index 3e45644..d7137ff 100644 --- a/packages/cli/src/analyzers/detectors/nestRouteDetector.ts +++ b/packages/cli/src/analyzers/detectors/nestRouteDetector.ts @@ -1,6 +1,7 @@ import { Project, SyntaxKind } from "ts-morph"; import type { ScannedFile } from "../analysis/index.js"; import type { RouteInfo } from "./routeDetector.js"; +import { compareText } from "../../utils/compare.js"; const ROUTE_DECORATORS = new Set([ "Get", @@ -82,6 +83,6 @@ export function detectNestRoutes(files: ScannedFile[]): RouteInfo[] { } return routes.sort((left, right) => - left.path.localeCompare(right.path) || left.file.localeCompare(right.file) + compareText(left.path, right.path) || compareText(left.file, right.file) ); } diff --git a/packages/cli/src/analyzers/detectors/routeDetector.ts b/packages/cli/src/analyzers/detectors/routeDetector.ts index d70e56f..4754f2b 100644 --- a/packages/cli/src/analyzers/detectors/routeDetector.ts +++ b/packages/cli/src/analyzers/detectors/routeDetector.ts @@ -23,6 +23,7 @@ import { sortRoutes } from "./routeUtils.js"; import { usesTrpc, detectTrpcRoutes } from "./trpcRouteDetector.js"; +import { compareText } from "../../utils/compare.js"; export type RouteInfo = { path: string; @@ -485,7 +486,7 @@ function collectFastifyRoutes(content: string): Array<[string, string[]]> { addRouteMethods(methodsByPath, content, FASTIFY_OBJECT_ROUTE_PATTERN); return [...methodsByPath] .map(([path, methods]) => [path, [...methods].sort()] as [string, string[]]) - .sort(([left], [right]) => left.localeCompare(right)); + .sort(([left], [right]) => compareText(left, right)); } function addRouteMethods( @@ -635,7 +636,7 @@ function collectRouteMethods( return [...methodsByPath] .map(([path, methods]) => [path, [...methods].sort()] as [string, string[]]) - .sort(([left], [right]) => left.localeCompare(right)); + .sort(([left], [right]) => compareText(left, right)); } /** diff --git a/packages/cli/src/analyzers/detectors/routeUtils.ts b/packages/cli/src/analyzers/detectors/routeUtils.ts index fba52bb..1c9ca6a 100644 --- a/packages/cli/src/analyzers/detectors/routeUtils.ts +++ b/packages/cli/src/analyzers/detectors/routeUtils.ts @@ -2,6 +2,7 @@ import { posix } from "node:path"; import { parseJsonText, type ScannedFile } from "../analysis/index.js"; import { isArchitectureSource } from "../graph/sourceScope.js"; import { matchLocalPath, normalizePath } from "../graph/localPaths.js"; +import { compareText } from "../../utils/compare.js"; // Helpers shared by the per-framework route detectors. @@ -87,7 +88,7 @@ export function findHttpMethods(content: string): string[] { export function sortRoutes(routes: T[]): T[] { return routes.sort((left, right) => - left.path.localeCompare(right.path) || left.file.localeCompare(right.file) + compareText(left.path, right.path) || compareText(left.file, right.file) ); } diff --git a/packages/cli/src/analyzers/detectors/serviceDetector.ts b/packages/cli/src/analyzers/detectors/serviceDetector.ts index adfaff2..d830397 100644 --- a/packages/cli/src/analyzers/detectors/serviceDetector.ts +++ b/packages/cli/src/analyzers/detectors/serviceDetector.ts @@ -1,4 +1,5 @@ import { parseJsonText, type ScannedFile } from "../analysis/index.js"; +import { IMPORT_CLAUSE } from "../analysis/importClause.js"; import { isArchitectureSource } from "../graph/index.js"; import { SERVICES, SOURCE_SERVICE_SIGNALS } from "../registry/index.js"; @@ -50,7 +51,10 @@ function readPackageDependencyNames(files: ScannedFile[]): string[] { function readImportedPackageNames(files: ScannedFile[]): string[] { const names = new Set(); - const importPattern = /(?:import\s+(?:[^'"]+\s+from\s+)?|export\s+[^'"]+\s+from\s+|require\()\s*['"]([^'"]+)['"]/g; + const importPattern = new RegExp( + String.raw`(?:import\s+(?:${IMPORT_CLAUSE}\s+from\s+)?|export\s+${IMPORT_CLAUSE}\s+from\s+|require\()\s*['"]([^'"]+)['"]`, + "g" + ); for (const file of files.filter((item) => [".ts", ".tsx", ".js", ".jsx"].includes(item.extension) diff --git a/packages/cli/src/analyzers/detectors/trpcRouteDetector.ts b/packages/cli/src/analyzers/detectors/trpcRouteDetector.ts index 0643ee1..f4b479f 100644 --- a/packages/cli/src/analyzers/detectors/trpcRouteDetector.ts +++ b/packages/cli/src/analyzers/detectors/trpcRouteDetector.ts @@ -1,6 +1,7 @@ import type { ScannedFile } from "../analysis/index.js"; import type { RouteInfo } from "./routeDetector.js"; import { isArchitectureSource } from "../graph/index.js"; +import { compareText } from "../../utils/compare.js"; type ProcedureKind = "QUERY" | "MUTATION"; @@ -83,7 +84,7 @@ export function detectTrpcRoutes( } return routes.sort((left, right) => - left.path.localeCompare(right.path) || left.file.localeCompare(right.file) + compareText(left.path, right.path) || compareText(left.file, right.file) ); } diff --git a/packages/cli/src/analyzers/features/featureCandidates.ts b/packages/cli/src/analyzers/features/featureCandidates.ts index c53e8df..6c2c1df 100644 --- a/packages/cli/src/analyzers/features/featureCandidates.ts +++ b/packages/cli/src/analyzers/features/featureCandidates.ts @@ -1,5 +1,6 @@ import type { FeatureInfo } from "./featureDetector.js"; import { isDynamicSegment } from "../detectors/nonResourceSegments.js"; +import { compareText } from "../../utils/compare.js"; export type FeatureCandidateSource = | "registry" @@ -102,7 +103,7 @@ export function reconcileFeatureCandidates( const deterministicCandidates = candidates .filter((candidate) => candidate.source !== "ai") .map(normalizeCandidate) - .sort((left, right) => left.id.localeCompare(right.id)); + .sort((left, right) => compareText(left.id, right.id)); const rejectedCandidateIds = candidates .filter((candidate) => candidate.source === "ai") .map((candidate) => candidate.id) @@ -215,14 +216,14 @@ export function reconcileFeatureCandidates( const clusters = [...componentIndexes.values()] .map((indexes) => createCluster(indexes, deterministicCandidates, anchorsByPair)) - .sort((left, right) => left.id.localeCompare(right.id)); + .sort((left, right) => compareText(left.id, right.id)); return { clusters, rejectedCandidateIds, mergeDecisions }; } /** Project only deterministic clusters to the established public snapshot shape. */ export function projectFeatureCandidates(clusters: FeatureCluster[]): FeatureInfo[] { - return clusters + const features = clusters .filter((cluster) => cluster.candidates.some((candidate) => candidate.source !== "ai")) .map((cluster) => { const projection = projectCompatibilityFeature(cluster); @@ -247,8 +248,47 @@ export function projectFeatureCandidates(clusters: FeatureCluster[]): FeatureInf confidence: clusterConfidence(cluster.candidates), evidence, }; - }) - .sort((left, right) => left.name.localeCompare(right.name)); + }); + + return uniqueFeatureNames(features).sort((left, right) => compareText(left.name, right.name)); +} + +const GENERIC_DIRECTORIES = new Set([ + "src", "app", "apps", "pages", "routes", "components", "lib", "features", "modules", "views", "screens" +]); + +/** + * Two clusters can end up with the same canonical label: a root "Home" page and a + * `pages/homes/` demo folder both singularize to "Home". Under one name, lookups by + * feature name and the per-feature map files become ambiguous, so the later cluster + * is renamed after the folder that tells the two apart. + */ +function uniqueFeatureNames(features: FeatureInfo[]): FeatureInfo[] { + const used = new Set(); + return features.map((feature) => { + let name = feature.name; + if (used.has(name.toLowerCase())) { + const hint = directoryHint(feature); + const base = hint ? `${feature.name} (${hint})` : feature.name; + name = base; + for (let suffix = 2; used.has(name.toLowerCase()); suffix += 1) name = `${base} ${suffix}`; + } + used.add(name.toLowerCase()); + return name === feature.name ? feature : { ...feature, name }; + }); +} + +function directoryHint(feature: FeatureInfo): string | undefined { + const paths = feature.entryPoints.length > 0 ? feature.entryPoints : feature.files; + const directories = paths.map((path) => path.split("/").slice(0, -1)); + if (directories.length === 0) return undefined; + + const common = directories.reduce((shared, current) => { + let length = 0; + while (length < shared.length && length < current.length && shared[length] === current[length]) length += 1; + return shared.slice(0, length); + }); + return [...common].reverse().find((segment) => !GENERIC_DIRECTORIES.has(segment.toLowerCase())); } function projectCompatibilityFeature(cluster: FeatureCluster): FeatureInfo | undefined { @@ -285,7 +325,7 @@ function createCluster( candidates: FeatureCandidate[], anchorsByPair: Map ): FeatureCluster { - const members = indexes.map((index) => candidates[index]).sort((left, right) => left.id.localeCompare(right.id)); + const members = indexes.map((index) => candidates[index]).sort((left, right) => compareText(left.id, right.id)); const canonical = [...members].sort(compareCanonicalCandidates)[0]; const anchors = uniqueSorted(indexes.flatMap((left) => indexes.flatMap((right) => left < right ? anchorsByPair.get(`${left}:${right}`) ?? [] : [] @@ -315,7 +355,7 @@ function compareCanonicalCandidates(left: FeatureCandidate, right: FeatureCandid const evidence = right.evidence.length - left.evidence.length; if (evidence !== 0) return evidence; - return left.id.localeCompare(right.id); + return compareText(left.id, right.id); } function findHardAnchors(left: FeatureCandidate, right: FeatureCandidate): string[] { @@ -431,7 +471,7 @@ function normalizeCandidate(candidate: FeatureCandidate): FeatureCandidate { files: uniqueSorted(item.files), ...(item.routePaths ? { routePaths: uniqueSorted(item.routePaths) } : {}), ...(item.entityNames ? { entityNames: uniqueSorted(item.entityNames) } : {}), - })).sort((left, right) => left.ruleId.localeCompare(right.ruleId)), + })).sort((left, right) => compareText(left.ruleId, right.ruleId)), files: uniqueSorted(candidate.files), routePaths: uniqueSorted(candidate.routePaths), entityNames: uniqueSorted(candidate.entityNames), @@ -462,7 +502,7 @@ function normalizeAnchor(value: string): string { } function uniqueSorted(values: string[]): string[] { - return [...new Set(values)].sort((left, right) => left.localeCompare(right)); + return [...new Set(values)].sort((left, right) => compareText(left, right)); } class UnionFind { diff --git a/packages/cli/src/analyzers/features/featureDetector.ts b/packages/cli/src/analyzers/features/featureDetector.ts index a752135..43017d1 100644 --- a/packages/cli/src/analyzers/features/featureDetector.ts +++ b/packages/cli/src/analyzers/features/featureDetector.ts @@ -5,6 +5,7 @@ import type { RelationInfo, } from "../analysis/index.js"; import { classifyFileRole, isTechnicalFeatureSource, isDocumentationMeta, type FileRole } from "../analysis/index.js"; +import { IMPORT_CLAUSE } from "../analysis/importClause.js"; import type { DatabaseInfo, RouteInfo, @@ -22,6 +23,7 @@ import { } from "./featureCandidates.js"; import { HOME_FEATURE_NAME } from "../detectors/frontendFeatureDetector.js"; import { FEATURE_SIGNALS, hasAiProviderUrl, isAiProviderImport, type FeatureSignal } from "../registry/index.js"; +import { compareText } from "../../utils/compare.js"; export type FeatureInfo = { name: string; @@ -354,7 +356,7 @@ export function detectFeatures( .map((file) => file.path) .sort((left, right) => featureFilePriority(definition.name, left) - featureFilePriority(definition.name, right) - || left.localeCompare(right) + || compareText(left, right) ) .slice(0, 12); @@ -409,7 +411,7 @@ export function detectFeatures( .map((file) => file.path) .sort((left, right) => featureFilePriority(signal.name, left) - featureFilePriority(signal.name, right) - || left.localeCompare(right) + || compareText(left, right) ) .slice(0, 5); @@ -425,7 +427,7 @@ export function detectFeatures( candidates.push(toFeatureCandidate("registry", `signal-${signal.name}`, createFeatureInfo( signal.name, evidence, - signal.terms, + groundedSignalTerms(signal, evidence, analyses), signal.purpose, analyses ))); @@ -466,7 +468,7 @@ export function detectFeatures( const reconciliation = reconcileFeatureCandidates(candidates, fileReferenceCounts); const features = projectFeatureCandidates(reconciliation.clusters); const enriched = enrichAuthenticationFeature(features, reconciliation.clusters, scopedFiles, analyses) - .sort((left, right) => left.name.localeCompare(right.name)); + .sort((left, right) => compareText(left.name, right.name)); return { features: enriched, mergeDecisions: reconciliation.mergeDecisions ?? [], @@ -702,7 +704,7 @@ function findEntityFiles(entityName: string, files: ScannedFile[]): string[] { .sort((a, b) => { const tierDiff = entityFileTierScore(a) - entityFileTierScore(b); if (tierDiff !== 0) return tierDiff; - return a.localeCompare(b); + return compareText(a, b); }) .slice(0, 5); } @@ -710,7 +712,7 @@ function findEntityFiles(entityName: string, files: ScannedFile[]): string[] { function splitNameToSegments(name: string): string[] { return name .replace(/([a-z0-9])([A-Z])/g, "$1 $2") - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .replace(/(?<=[A-Z])(?=[A-Z][a-z])/g, " ") .toLowerCase() .split(/[^a-z0-9]+/) .filter(Boolean); @@ -925,6 +927,28 @@ function matchesSignal( ); } +/** + * The registry lists every vendor and keyword a signal knows about. Only the + * ones this project's evidence really contains belong in its search terms, or an + * agent reading "clerk, auth0, kinde" assumes the project uses them. + */ +function groundedSignalTerms( + signal: Pick, + evidence: string[], + analyses: Record +): string[] { + if (signal.importOnly) return signal.terms; + + const paths = evidence.map((path) => path.toLowerCase()); + const grounded = [...signal.terms, ...(signal.importTerms ?? [])].filter((term) => + paths.some((path) => matchesPathTerm(path, term)) + || evidence.some((path) => + analyses[path]?.imports.some((specifier) => !specifier.startsWith(".") && matchesImportTerm(specifier, term)) + ) + ); + return grounded.length > 0 ? [...new Set(grounded)] : [signal.name.toLowerCase()]; +} + /** * matchesImportTerm — segment-aware import specifier matching. * @@ -1130,7 +1154,7 @@ export function featureFilePriority(featureName: string, path: string): number { export function orderAuthenticationFiles(files: string[]): string[] { return [...new Set(files)].sort((left, right) => authenticationFilePriority(left) - authenticationFilePriority(right) - || left.localeCompare(right) + || compareText(left, right) ); } @@ -1153,7 +1177,10 @@ export function authenticationFilePriority(path: string): number { // --------------------------------------------------------------------------- function extractImportsFallback(content: string): string[] { const imports: string[] = []; - const pattern = /(?:import\s+(?:[^'"]+\s+from\s+)?|require\()\s*['"]([^'"]+)['"]/g; + const pattern = new RegExp( + String.raw`(?:import\s+(?:${IMPORT_CLAUSE}\s+from\s+)?|require\()\s*['"]([^'"]+)['"]`, + "g" + ); let match = pattern.exec(content); while (match) { imports.push(match[1].toLowerCase()); diff --git a/packages/cli/src/analyzers/graph/aliasResolver.ts b/packages/cli/src/analyzers/graph/aliasResolver.ts index c6c2f65..326c88b 100644 --- a/packages/cli/src/analyzers/graph/aliasResolver.ts +++ b/packages/cli/src/analyzers/graph/aliasResolver.ts @@ -4,6 +4,7 @@ import { dirname, join, relative, resolve, sep } from "node:path"; import { ts } from "ts-morph"; import { loadBundlerAliases } from "./bundlerAliases.js"; import { pickNuxtSrcDir } from "./nuxtAutoImports.js"; +import { compareText } from "../../utils/compare.js"; export interface AliasMapping { /** Specifier prefix (`@/`), or the whole specifier when `exact` is set. */ @@ -139,7 +140,7 @@ async function findConfigDirectories(root: string): Promise { // A frontend/ or client/ next to the root config is as common as apps/*. try { const children = await readdir(root, { withFileTypes: true }); - for (const entry of children.sort((a, b) => a.name.localeCompare(b.name))) { + for (const entry of children.sort((a, b) => compareText(a.name, b.name))) { if ( entry.isDirectory() && !entry.name.startsWith(".") @@ -156,7 +157,7 @@ async function findConfigDirectories(root: string): Promise { for (const parent of WORKSPACE_PARENTS) { try { const entries = await readdir(join(root, parent), { withFileTypes: true }); - for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + for (const entry of entries.sort((a, b) => compareText(a.name, b.name))) { if (entry.isDirectory() && !entry.name.startsWith(".") && entry.name !== "node_modules") { directories.push(join(root, parent, entry.name)); } diff --git a/packages/cli/src/analyzers/graph/dependencyGraph.ts b/packages/cli/src/analyzers/graph/dependencyGraph.ts index aaa66b3..1c04909 100644 --- a/packages/cli/src/analyzers/graph/dependencyGraph.ts +++ b/packages/cli/src/analyzers/graph/dependencyGraph.ts @@ -1,4 +1,5 @@ import type { FileAnalysis, ScannedFile } from "../analysis/index.js"; +import { IMPORT_CLAUSE } from "../analysis/importClause.js"; import type { AliasMapping } from "./aliasResolver.js"; import { isDeclaredAlias, resolveAliasCandidates } from "./aliasResolver.js"; import { matchLocalPath, normalizePath } from "./localPaths.js"; @@ -18,7 +19,10 @@ export interface DependencyGraphDiagnostics { } // Static imports, re-exports, require() and dynamic import() with a literal. -const IMPORT_RE = /(?:\bimport\s+(?:[^'"]+\s+from\s+)?|\bexport\s+[^'"]+\s+from\s+|\b(?:require|import)\s*\(\s*)['"]([^'"]+)['"]/g; +const IMPORT_RE = new RegExp( + String.raw`(?:\bimport\s+(?:${IMPORT_CLAUSE}\s+from\s+)?|\bexport\s+${IMPORT_CLAUSE}\s+from\s+|\b(?:require|import)\s*\(\s*)['"]([^'"]+)['"]`, + "g" +); export function buildDependencyGraph( files: ScannedFile[], diff --git a/packages/cli/src/analyzers/graph/entryPoints.ts b/packages/cli/src/analyzers/graph/entryPoints.ts index e913ee2..9e26e28 100644 --- a/packages/cli/src/analyzers/graph/entryPoints.ts +++ b/packages/cli/src/analyzers/graph/entryPoints.ts @@ -1,6 +1,7 @@ import type { FileGraph } from "./dependencyGraph.js"; import type { ManifestEntry } from "./packageManifests.js"; import { isArchitectureSource } from "./sourceScope.js"; +import { compareText } from "../../utils/compare.js"; const ENTRY_PATTERNS = [ /(^|\/)page\.[jt]sx?$/, @@ -114,7 +115,7 @@ export function detectEntryPoints(graph: FileGraph, manifestEntries: ManifestEnt const scoreOf = (path: string) => declared.get(path) ?? entryPointScore(path); return Object.keys(graph) - .filter((path) => isSourceFile(path)) + .filter((path) => isSourceFile(path) && !isDeclarationFile(path)) .filter((path) => isArchitectureSource(path) && !isGeneratedFile(path)) .filter((path) => { if (declared.has(path)) return true; @@ -129,7 +130,7 @@ export function detectEntryPoints(graph: FileGraph, manifestEntries: ManifestEnt || (!imported.has(path) && FRAMEWORK_LOADED_PATTERNS.some((pattern) => pattern.test(path))) || unimportedWithImports; }) - .sort((a, b) => scoreOf(a) - scoreOf(b) || a.localeCompare(b)) + .sort((a, b) => scoreOf(a) - scoreOf(b) || compareText(a, b)) .slice(0, 20); } @@ -140,3 +141,7 @@ function isGeneratedFile(path: string): boolean { function isSourceFile(path: string): boolean { return /\.[cm]?[jt]sx?$|\.vue$|\.svelte$|\.astro$/.test(path); } + +function isDeclarationFile(path: string): boolean { + return /\.d\.[cm]?ts$/.test(path); +} diff --git a/packages/cli/src/analyzers/graph/nuxtAutoImports.ts b/packages/cli/src/analyzers/graph/nuxtAutoImports.ts index c015301..fd87a2a 100644 --- a/packages/cli/src/analyzers/graph/nuxtAutoImports.ts +++ b/packages/cli/src/analyzers/graph/nuxtAutoImports.ts @@ -1,6 +1,7 @@ import type { FileAnalysis, ScannedFile } from "../analysis/index.js"; import type { FileGraph } from "./dependencyGraph.js"; import { isArchitectureSource } from "./sourceScope.js"; +import { compareText } from "../../utils/compare.js"; /** * Nuxt wires files together without import statements: components, composables @@ -63,7 +64,7 @@ export function detectNuxtApps(files: ScannedFile[]): NuxtApp[] { }); } - return apps.sort((a, b) => a.dir.localeCompare(b.dir)); + return apps.sort((a, b) => compareText(a.dir, b.dir)); } /** @@ -226,7 +227,7 @@ function renderedComponents(content: string): Set { function splitByCase(value: string): string[] { return value .replace(/([a-z\d])([A-Z])/g, "$1 $2") - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .replace(/(?<=[A-Z])(?=[A-Z][a-z])/g, " ") .split(/[\s\-_./]+/) .filter(Boolean); } diff --git a/packages/cli/src/analyzers/graph/packageManifests.ts b/packages/cli/src/analyzers/graph/packageManifests.ts index 78f523b..b263b42 100644 --- a/packages/cli/src/analyzers/graph/packageManifests.ts +++ b/packages/cli/src/analyzers/graph/packageManifests.ts @@ -1,6 +1,7 @@ import { parseJsonText, type ScannedFile } from "../analysis/index.js"; import { matchLocalPath, normalizePath } from "./localPaths.js"; import { isArchitectureSource } from "./sourceScope.js"; +import { compareText } from "../../utils/compare.js"; /** * package.json knowledge the dependency graph needs: which workspace package a @@ -80,7 +81,7 @@ export function readPackageManifests(files: ScannedFile[]): PackageManifest[] { }); } - return manifests.sort((a, b) => a.dir.localeCompare(b.dir)); + return manifests.sort((a, b) => compareText(a.dir, b.dir)); } export function indexPackages(manifests: PackageManifest[]): PackageIndex { diff --git a/packages/cli/src/analyzers/graph/sourceScope.ts b/packages/cli/src/analyzers/graph/sourceScope.ts index 3b166ab..15c12fc 100644 --- a/packages/cli/src/analyzers/graph/sourceScope.ts +++ b/packages/cli/src/analyzers/graph/sourceScope.ts @@ -9,6 +9,7 @@ const NON_PRODUCTION_SEGMENTS = new Set([ "e2e", "fixtures", "test", + "test-d", "tests" ]); diff --git a/packages/cli/src/analyzers/inference/domainInference.ts b/packages/cli/src/analyzers/inference/domainInference.ts index 30bebd9..720ceec 100644 --- a/packages/cli/src/analyzers/inference/domainInference.ts +++ b/packages/cli/src/analyzers/inference/domainInference.ts @@ -5,6 +5,7 @@ import { classifyOwnershipTopology, type OwnershipPattern } from "./ownershipTop import { createHash } from "node:crypto"; import { readFile, writeFile, mkdir } from "node:fs/promises"; import { join } from "node:path"; +import { z } from "zod"; // --------------------------------------------------------------------------- // Types @@ -63,7 +64,7 @@ type DomainInferenceCache = { function hashDomainInput(input: DomainInferenceInput): string { const stable = JSON.stringify({ - v: 2, + v: 3, entityNames: [...input.entityNames].sort(), relations: [...input.relations] .map((r) => `${r.from}:${r.kind}:${r.to}`) @@ -81,15 +82,17 @@ function hashDomainInput(input: DomainInferenceInput): string { async function readDomainCache( projectRoot: string, - inputHash: string + inputHash: string, + input: DomainInferenceInput ): Promise { try { const cachePath = join(projectRoot, ".devmap", "domain-cache.json"); const raw = await readFile(cachePath, "utf-8"); - const cache = JSON.parse(raw) as DomainInferenceCache; + const cache = JSON.parse(raw) as Partial; if (cache.inputHash !== inputHash) return null; // input berubah → stale - return cache.result; + // A cached result that no longer passes validation is a miss, not a crash. + return sanitizeDomainResult(cache.result, input); } catch { return null; // file tidak ada atau corrupt → miss } @@ -151,7 +154,7 @@ export async function inferDomain( // Cache hit → return langsung, no LLM call if (projectRoot) { - const cached = await readDomainCache(projectRoot, inputHash); + const cached = await readDomainCache(projectRoot, inputHash, input); if (cached) return cached; } @@ -220,7 +223,8 @@ Respond with ONLY a JSON object (no markdown, no explanation): } Rules: -- domainFeatures: 3-6 features specific to this domain, not generic technical features +- domainFeatures: up to 6 features specific to this domain, not generic technical features. Use an empty array when the metadata does not support any +- Every relatedEntities value must be copied from the entities list above - Do not include Authentication, Database, or other infrastructure as domainFeatures - Base everything on the metadata provided, not assumptions - Entity names alone (e.g. 'Message', 'Room', 'User') are NOT reliable signals of application domain. The same entity name can represent a chat message, an activity log, a personal note, or a comment — depending on ownership pattern, not naming. Use ownershipPattern and absentCapabilities as primary evidence: single_user_isolated strongly suggests a personal/private tool (notes, journal, tracker), not a multi-user communication platform. Only conclude 'chat' or 'messaging' domain if ownershipPattern is shared_access or direct_messaging, or detectedCapabilities includes collaboration/social/real-time.`; @@ -230,6 +234,67 @@ Rules: // Response parser // --------------------------------------------------------------------------- +const domainFeatureSchema = z.object({ + name: z.string().trim().min(1).max(80), + purpose: z.string().trim().min(1).max(300), + relatedEntities: z.union([ + z.array(z.string()), + z.string().transform((value) => [value]) + ]) +}); + +const domainResultSchema = z.object({ + domain: z.string().trim().min(1).max(80), + summary: z.string().trim().min(1).max(500), + confidence: z.number().optional(), + domainFeatures: z.array(z.unknown()).optional() +}); + +/** + * Single gate for everything the model (or a stale cache file) hands back. + * Wrong types are rejected instead of crashing later, and a domain feature only + * survives with entities that really exist in the static analysis, because an + * AI feature has no files of its own to be grounded in. + */ +function sanitizeDomainResult( + value: unknown, + input: DomainInferenceInput +): DomainInferenceResult | null { + const parsed = domainResultSchema.safeParse(value); + if (!parsed.success) return null; + + const knownEntities = new Map(input.entityNames.map((name) => [name.toLowerCase(), name])); + const domainFeatures: DomainFeature[] = []; + + for (const candidate of parsed.data.domainFeatures ?? []) { + const feature = domainFeatureSchema.safeParse(candidate); + if (!feature.success) continue; + + const relatedEntities = [...new Set( + feature.data.relatedEntities + .map((name) => knownEntities.get(name.trim().toLowerCase())) + .filter((name): name is string => name !== undefined) + )]; + if (relatedEntities.length === 0) continue; + + domainFeatures.push({ + name: feature.data.name, + purpose: feature.data.purpose, + relatedEntities + }); + if (domainFeatures.length === 6) break; + } + + return { + domain: parsed.data.domain, + summary: parsed.data.summary, + domainFeatures, + confidence: Math.min(1, Math.max(0, parsed.data.confidence ?? 0)), + model: "unknown", + tokensUsed: 0 + }; +} + function parseDomainInferenceResponse( raw: string, input: DomainInferenceInput @@ -242,36 +307,7 @@ function parseDomainInferenceResponse( .replace(/```\s*$/i, "") .trim(); - const parsed = JSON.parse(cleaned) as { - domain?: string; - summary?: string; - confidence?: number; - domainFeatures?: Array<{ - name?: string; - purpose?: string; - relatedEntities?: string[]; - }>; - }; - - if (!parsed.domain || !parsed.summary) return null; - - const domainFeatures: DomainFeature[] = (parsed.domainFeatures ?? []) - .filter((f) => f.name && f.purpose) - .map((f) => ({ - name: f.name!, - purpose: f.purpose!, - relatedEntities: f.relatedEntities ?? [] - })) - .slice(0, 6); - - return { - domain: parsed.domain, - summary: parsed.summary, - domainFeatures, - confidence: Math.min(1, Math.max(0, parsed.confidence ?? 0.7)), - model: "unknown", // diisi caller - tokensUsed: 0 // diisi caller - }; + return sanitizeDomainResult(JSON.parse(cleaned), input); } catch { return null; } diff --git a/packages/cli/src/analyzers/pipeline/heuristicAnalyzer.ts b/packages/cli/src/analyzers/pipeline/heuristicAnalyzer.ts index dd85083..2ab3a3e 100644 --- a/packages/cli/src/analyzers/pipeline/heuristicAnalyzer.ts +++ b/packages/cli/src/analyzers/pipeline/heuristicAnalyzer.ts @@ -7,6 +7,8 @@ import type { ScannedFile, } from "../analysis/index.js"; import { readLanguage, supportsLanguageReader } from "./languageReaders.js"; +import { IMPORT_CLAUSE } from "../analysis/importClause.js"; +import { compareText } from "../../utils/compare.js"; // TS/JS extensions are intentionally excluded here. // TsMorphAnalyzer handles .ts/.tsx/.js/.jsx with full AST accuracy (confidence: "high"). @@ -95,7 +97,10 @@ export class FallbackAnalyzer implements FileAnalyzer { function readImportSpecifiers(content: string): string[] { const imports = new Set(); - const pattern = /(?:\bimport\s+(?:[^'"]+\s+from\s+)?|\bexport\s+[^'"]+\s+from\s+|\b(?:require|import)\s*\(\s*)['"]([^'"]+)['"]/g; + const pattern = new RegExp( + String.raw`(?:\bimport\s+(?:${IMPORT_CLAUSE}\s+from\s+)?|\bexport\s+${IMPORT_CLAUSE}\s+from\s+|\b(?:require|import)\s*\(\s*)['"]([^'"]+)['"]`, + "g" + ); let match = pattern.exec(content); while (match) { @@ -174,7 +179,7 @@ function readSymbols(content: string): SymbolInfo[] { } } - return [...symbols.values()].sort((left, right) => left.line - right.line || left.name.localeCompare(right.name)); + return [...symbols.values()].sort((left, right) => left.line - right.line || compareText(left.name, right.name)); } function lineAt(content: string, index: number): number { @@ -184,5 +189,5 @@ function lineAt(content: string, index: number): number { function compareFunctions(left: FunctionInfo, right: FunctionInfo): number { return Number(right.exported) - Number(left.exported) || left.line - right.line - || left.name.localeCompare(right.name); + || compareText(left.name, right.name); } diff --git a/packages/cli/src/analyzers/pipeline/languageReaders.ts b/packages/cli/src/analyzers/pipeline/languageReaders.ts index 6cbca3b..60ee3fb 100644 --- a/packages/cli/src/analyzers/pipeline/languageReaders.ts +++ b/packages/cli/src/analyzers/pipeline/languageReaders.ts @@ -1,4 +1,5 @@ import type { SymbolInfo } from "../analysis/index.js"; +import { compareText } from "../../utils/compare.js"; export interface LanguageReading { imports: string[]; @@ -350,5 +351,5 @@ function readJava(code: string): LanguageReading { } function byLine(left: SymbolInfo, right: SymbolInfo): number { - return left.line - right.line || left.name.localeCompare(right.name); + return left.line - right.line || compareText(left.name, right.name); } diff --git a/packages/cli/src/analyzers/pipeline/projectMap.ts b/packages/cli/src/analyzers/pipeline/projectMap.ts index 8d4e2f6..1828d2c 100644 --- a/packages/cli/src/analyzers/pipeline/projectMap.ts +++ b/packages/cli/src/analyzers/pipeline/projectMap.ts @@ -1,5 +1,6 @@ import { output, type Spinner } from "../../utils/output.js"; import { hashContent } from "../../cache/fileHash.js"; +import { DEVMAP_VERSION } from "../../utils/packageMetadata.js"; import { REASON_TAGS } from "./reasonTags.js"; import { analyzeFiles } from "./analyzerRegistry.js"; import { detectProjectMetadata, type ProjectMetadata } from "./projectMetadata.js"; @@ -32,6 +33,7 @@ import { } from "../features/index.js"; import { scanFiles, + MAX_READ_BYTES, parseJsonText, extractEntities, type ScannedFile, @@ -45,6 +47,7 @@ import { domainFeaturesToFeatureInfo, type DomainInferenceResult, } from "../inference/index.js"; +import { compareText } from "../../utils/compare.js"; export const SNAPSHOT_SCHEMA_VERSION = "1"; @@ -238,14 +241,21 @@ export async function createProjectMap( ); const result = await inferDomain(inferenceInput, callAI, projectRoot); if (result) { - domain = result; // Merge domain-specific features ke features list. // Pakai similarity engine — bukan name equality — sehingga // "Customizable Plans" tidak duplicate "Plan Management" yang sudah ada. // Canonical name (first-seen) dipertahankan oleh mergeDomainFeatures. - const domainFeatures = domainFeaturesToFeatureInfo(result.domainFeatures); - mergeDomainFeatures(features, domainFeatures); - features.sort((a, b) => a.name.localeCompare(b.name)); + // The merge runs on a copy so AI output can never abort the analysis or + // leave the static feature list half-updated. + try { + const merged = [...features]; + mergeDomainFeatures(merged, domainFeaturesToFeatureInfo(result.domainFeatures)); + merged.sort((a, b) => compareText(a.name, b.name)); + features.splice(0, features.length, ...merged); + domain = result; + } catch { + domain = undefined; + } } activeSpinner.succeed(domain ? `Domain inferred: ${domain.domain}` : "Domain inference skipped"); } @@ -325,16 +335,22 @@ function createAgentInstructions(): ProjectMap["agentInstructions"] { }; } +// An unread file still has to change the fingerprint when it grows or shrinks. +function contentHash(file: ScannedFile): string { + return hashContent(file.oversize ? `oversize:${file.size}` : file.content); +} + export function createProjectFingerprint(files: ScannedFile[]): string { - const content = files - .map((file) => [file.path, hashContent(file.content)] as const) + const entries = files + .map((file) => [file.path, contentHash(file)] as const) // Plain code-unit order: localeCompare would change the fingerprint of an // unchanged project when the machine locale differs. .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([path, hash]) => `${path}:${hash}`) - .join("\n"); + .map(([path, hash]) => `${path}:${hash}`); - return hashContent(content); + // The analyzer version is part of the fingerprint, so a snapshot written by an + // older DevMap is regenerated instead of being reused as "unchanged". + return hashContent([`devmap@${DEVMAP_VERSION}`, ...entries].join("\n")); } function rankCriticalFiles( @@ -393,7 +409,7 @@ function rankCriticalFiles( return { path: file.path, referencedBy, score, reasons }; }) .filter((file) => file.score > 0) - .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path)) + .sort((left, right) => right.score - left.score || compareText(left.path, right.path)) .slice(0, 10); } @@ -469,7 +485,7 @@ function createFileIndexEntry( return { analyzer: analysis.analyzer, analysisConfidence: analysis.confidence, - hash: hashContent(file.content), + hash: contentHash(file), imports, exportedSymbols, symbols: analysis.symbols, @@ -685,7 +701,7 @@ function attachFeatureEntryPoints( ])].sort((left, right) => Number(isToolingPath(left)) - Number(isToolingPath(right)) || left.split("/").length - right.split("/").length - || left.localeCompare(right) + || compareText(left, right) ); const orderedFiles = feature.name === "Authentication" ? orderAuthenticationFiles(feature.files) @@ -744,7 +760,7 @@ function chooseFeatureEntryPoint( if (featureName === "Documentation" && files.length > 1) { const sorted = [...files].sort((a, b) => featureFilePriority(featureName, a) - featureFilePriority(featureName, b) - || a.localeCompare(b) + || compareText(a, b) ); return sorted[0]; } @@ -1184,6 +1200,14 @@ function detectAnalysisWarnings( warnings.push("No features detected. Feature-based navigation may be unavailable."); } + const unread = files.filter((file) => file.oversize); + if (unread.length > 0) { + const names = unread.slice(0, 5).map((file) => file.path).join(", "); + warnings.push( + `${unread.length} file(s) over ${MAX_READ_BYTES / (1024 * 1024)} MB were listed but not read: ${names}${unread.length > 5 ? ", ..." : ""}.` + ); + } + const packageJson = files.find((file) => file.path === "package.json"); if (!packageJson) { return warnings; @@ -1198,10 +1222,12 @@ function detectAnalysisWarnings( return warnings; } +// Lookarounds, not `([A-Z]+)([A-Z][a-z])`: that pattern is quadratic on a long run +// of capitals, and a 100,000-letter class name made the index step take 7 seconds. function splitSearchTerms(value: string): string[] { const spaced = value .replace(/([a-z0-9])([A-Z])/g, "$1 $2") - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2"); + .replace(/(?<=[A-Z])(?=[A-Z][a-z])/g, " "); return spaced .toLowerCase() diff --git a/packages/cli/src/analyzers/pipeline/projectMetadata.ts b/packages/cli/src/analyzers/pipeline/projectMetadata.ts index 2825fbe..768319c 100644 --- a/packages/cli/src/analyzers/pipeline/projectMetadata.ts +++ b/packages/cli/src/analyzers/pipeline/projectMetadata.ts @@ -35,10 +35,16 @@ export function detectProjectMetadata( frameworks: DetectedFramework[] = framework === "unknown" ? [] : [framework] ): ProjectMetadata { const manifests = readPackageManifests(files); - const projectTypes = detectProjectTypes(frameworks, manifests); + const workspacePatterns = readWorkspacePatterns(files, manifests); + const projectManifests = manifests.filter((manifest) => + isProjectManifest(manifest, workspacePatterns, frameworks) + ); + const projectTypes = detectProjectTypes(frameworks, manifests, projectManifests); const primaryManifest = selectPrimaryManifest(manifests, projectTypes); - const workspaceType = detectWorkspaceType(projectRoot, manifests); - const primaryFramework = isRootCliOrLibrary(manifests, framework) ? "unknown" : framework; + const workspaceType: WorkspaceType = workspacePatterns.length > 0 || projectManifests.length > 1 + ? "monorepo" + : "single-package"; + const primaryFramework = isRootCliOrLibrary(manifests, framework, frameworks) ? "unknown" : framework; return { name: readProjectName(manifests) ?? basename(projectRoot), @@ -105,7 +111,11 @@ function readPackageManifests(files: ScannedFile[]): PackageManifest[] { * itself a CLI, and shouldn't have its correctly-detected framework (e.g. * an Astro app living in apps/web) blanked out to "unknown" because of it. */ -function isRootCliOrLibrary(manifests: PackageManifest[], framework: Framework): boolean { +function isRootCliOrLibrary( + manifests: PackageManifest[], + framework: Framework, + detected: DetectedFramework[] +): boolean { const rootManifest = manifests.find((manifest) => manifest.path === "package.json"); if (!rootManifest) return false; @@ -114,7 +124,7 @@ function isRootCliOrLibrary(manifests: PackageManifest[], framework: Framework): // `npm init` writes "main": "index.js" into every package, including plain // Express/Next apps. A root that itself declares the detected framework is // that app, not a library, whatever its main/exports say. - if (framework !== "unknown" && declaresFramework(rootManifest, framework)) return false; + if (framework !== "unknown" && declaresFramework(rootManifest, framework, detected)) return false; return Boolean(rootManifest.exports || rootManifest.main); } @@ -145,14 +155,24 @@ const FRAMEWORK_DEPENDENCIES: Record = { koa: ["koa"] }; -function declaresFramework(manifest: PackageManifest, framework: Framework): boolean { - return (FRAMEWORK_DEPENDENCIES[framework] ?? []).some((dependency) => - dependency in manifest.dependencies || dependency in manifest.devDependencies - ); +function declaresFramework( + manifest: PackageManifest, + framework: Framework, + detected: DetectedFramework[] +): boolean { + // A server framework that is only a devDependency is test tooling (a mock + // server) unless production code imports it, which detectFrameworks() already + // checked: only a detected one may count through devDependencies. + const declared = BACKEND_FRAMEWORK_SET.has(framework) && !detected.includes(framework as DetectedFramework) + ? manifest.dependencies + : { ...manifest.dependencies, ...manifest.devDependencies }; + return (FRAMEWORK_DEPENDENCIES[framework] ?? []).some((dependency) => dependency in declared); } -function declaresAnyFramework(manifest: PackageManifest): boolean { - return Object.keys(FRAMEWORK_DEPENDENCIES).some((framework) => declaresFramework(manifest, framework as Framework)); +function declaresAnyFramework(manifest: PackageManifest, detected: DetectedFramework[]): boolean { + return Object.keys(FRAMEWORK_DEPENDENCIES).some((framework) => + declaresFramework(manifest, framework as Framework, detected) + ); } function readProjectName(manifests: PackageManifest[]): string | null { @@ -162,7 +182,8 @@ function readProjectName(manifests: PackageManifest[]): string | null { function detectProjectTypes( frameworks: DetectedFramework[], - manifests: PackageManifest[] + manifests: PackageManifest[], + projectManifests: PackageManifest[] ): ProjectType[] { const types = new Set(); @@ -180,10 +201,13 @@ function detectProjectTypes( } // A package that ships an app framework with its default "main" is an app, // not a library. An editor extension's "main" is its activation entry, not a public API. - if (manifests.some((manifest) => + // Only the root and real workspace members can make the repo a library: a + // nested helper package (a template's init script, say) is not something the + // repo publishes. + if (projectManifests.some((manifest) => (manifest.exports || manifest.main) && !manifest.bin - && !declaresAnyFramework(manifest) + && !declaresAnyFramework(manifest, frameworks) && !isVscodeExtension(manifest) )) { types.add("library"); @@ -211,16 +235,91 @@ function selectPrimaryManifest( return manifests.find((manifest) => manifest.path === "package.json") ?? manifests[0]; } -function detectWorkspaceType( - projectRoot: string, - manifests: PackageManifest[] -): WorkspaceType { - const rootManifest = manifests.find((manifest) => manifest.path === "package.json"); - return existsSync(join(projectRoot, "pnpm-workspace.yaml")) - || Boolean(rootManifest?.workspaces) - || manifests.length > 1 - ? "monorepo" - : "single-package"; +const CONVENTIONAL_MEMBER_DIRECTORY = /^(?:apps|packages|libs|services|projects|modules)\//; + +function manifestDirectory(path: string): string { + return path.slice(0, Math.max(0, path.length - "package.json".length - 1)); +} + +/** + * Workspace globs the root declares: package.json `workspaces` and the + * `packages:` list of pnpm-workspace.yaml. pnpm 10 keeps plain settings + * (`onlyBuiltDependencies`, ...) in that file too, so its mere presence says + * nothing about a monorepo. + */ +function readWorkspacePatterns(files: ScannedFile[], manifests: PackageManifest[]): string[] { + const rootWorkspaces = manifests.find((manifest) => manifest.path === "package.json")?.workspaces; + const fromManifest = Array.isArray(rootWorkspaces) + ? rootWorkspaces + : typeof rootWorkspaces === "object" && rootWorkspaces !== null + && Array.isArray((rootWorkspaces as { packages?: unknown }).packages) + ? (rootWorkspaces as { packages: unknown[] }).packages + : []; + const pnpmFile = files.find((file) => file.path === "pnpm-workspace.yaml"); + const fromPnpm = pnpmFile ? readPnpmWorkspacePackages(pnpmFile.content) : []; + + return [...fromManifest.filter((item): item is string => typeof item === "string"), ...fromPnpm] + .map((pattern) => pattern.trim()) + .filter((pattern) => pattern.length > 0 && !pattern.startsWith("!")); +} + +function readPnpmWorkspacePackages(content: string): string[] { + const patterns: string[] = []; + let inPackages = false; + + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.replace(/(^|\s+)#.*$/, ""); + const header = line.match(/^packages\s*:\s*(.*)$/); + if (header) { + const inline = header[1].trim(); + inPackages = inline === ""; + if (inline.startsWith("[")) { + patterns.push(...inline.replace(/^\[|\]$/g, "").split(",").map(unquote)); + } + continue; + } + if (!inPackages) continue; + + const item = line.match(/^\s*-\s*(.+?)\s*$/); + if (item) patterns.push(unquote(item[1])); + else if (/^\S/.test(line)) inPackages = false; + } + + return patterns; +} + +function unquote(value: string): string { + return value.trim().replace(/^(["'])(.*)\1$/, "$2"); +} + +function matchesWorkspacePattern(directory: string, pattern: string): boolean { + const source = pattern + .replace(/^\.\//, "") + .replace(/\/+$/, "") + .replace(/\/package\.json$/, "") + .split("/") + .map((segment) => segment === "**" + ? ".*" + : segment.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*")) + .join("/"); + return new RegExp(`^${source}$`).test(directory); +} + +/** + * The root, a declared or conventionally placed workspace member, or a nested + * package that is itself an app. A nested package.json that is none of these + * (a template's init script, a docs helper) is not a package of this project. + */ +function isProjectManifest( + manifest: PackageManifest, + patterns: string[], + detected: DetectedFramework[] +): boolean { + if (manifest.path === "package.json") return true; + const directory = manifestDirectory(manifest.path); + return patterns.some((pattern) => matchesWorkspacePattern(directory, pattern)) + || CONVENTIONAL_MEMBER_DIRECTORY.test(directory) + || declaresAnyFramework(manifest, detected); } function hasDependency(manifests: PackageManifest[], dependency: string): boolean { @@ -241,6 +340,7 @@ function detectLanguage(files: ScannedFile[]): ProjectLanguage { [".js", ".jsx", ".mjs", ".cjs"].includes(file.extension) ).length; + if (typeScriptFiles === 0 && javaScriptFiles === 0) return "unknown"; if (typeScriptFiles > 0 && javaScriptFiles === 0) return "typescript"; if (javaScriptFiles > 0 && typeScriptFiles === 0) return "javascript"; if (typeScriptFiles >= javaScriptFiles * 2) return "typescript"; diff --git a/packages/cli/src/cache/agentNavigation.ts b/packages/cli/src/cache/agentNavigation.ts index 6347e8a..bf987fe 100644 --- a/packages/cli/src/cache/agentNavigation.ts +++ b/packages/cli/src/cache/agentNavigation.ts @@ -2,6 +2,7 @@ import { mkdir, readdir, unlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { FeatureInfo } from "../analyzers/features/index.js"; import type { ProjectMap } from "../analyzers/pipeline/index.js"; +import { compareText } from "../utils/compare.js"; export type AgentNavigationWriteResult = { indexPath: string; @@ -26,24 +27,17 @@ export async function writeAgentNavigationFiles( await mkdir(featureDirectory, { recursive: true }); await removeStaleFeatureMaps(featureDirectory); - // Dedup by id — safety net for features that generate the same id, - // e.g. "Checklist Item Management" and "checklist-item Management" - // both resolve to "checklist-item-management". First-seen wins. - const seenIds = new Set(); - const dedupedFeatures = snapshot.features.filter((feature) => { - const id = featureId(feature.name); - if (seenIds.has(id)) return false; - seenIds.add(id); - return true; - }); - - const featureIndexes = dedupedFeatures.map((feature) => - createFeatureIndex(feature, snapshot) + // Two features can slug to the same id ("Checklist Item Management" and + // "checklist-item Management"). Both stay reachable: the later one gets a + // numeric suffix instead of silently disappearing from the index. + const ids = assignFeatureIds(snapshot.features); + const featureIndexes = snapshot.features.map((feature, index) => + createFeatureIndex(feature, ids[index], snapshot) ); const featurePaths: string[] = []; - for (const feature of dedupedFeatures) { - const id = featureId(feature.name); + for (const [index, feature] of snapshot.features.entries()) { + const id = ids[index]; const path = join(featureDirectory, `${id}.json`); await writeJson(path, createFeatureMap(id, feature, snapshot)); featurePaths.push(path); @@ -75,8 +69,7 @@ export async function writeAgentNavigationFiles( return { indexPath, featurePaths }; } -function createFeatureIndex(feature: FeatureInfo, snapshot: ProjectMap): AgentFeatureIndex { - const id = featureId(feature.name); +function createFeatureIndex(feature: FeatureInfo, id: string, snapshot: ProjectMap): AgentFeatureIndex { return { id, name: feature.name, @@ -108,7 +101,7 @@ function createFeatureMap(id: string, feature: FeatureInfo, snapshot: ProjectMap - (featureOrder.get(right.path) ?? Number.MAX_SAFE_INTEGER) || (snapshot.fileIndex[right.path]?.importance ?? 0) - (snapshot.fileIndex[left.path]?.importance ?? 0) - || left.path.localeCompare(right.path); + || compareText(left.path, right.path); }) .map((file) => file.path) .slice(0, 8); @@ -164,7 +157,7 @@ function selectIndexCriticalFiles(snapshot: ProjectMap): string[] { }) .sort((left, right) => calculateStartHereScore(right, snapshot, entryDistance) - calculateStartHereScore(left, snapshot, entryDistance) - || left.localeCompare(right) + || compareText(left, right) ) .slice(0, 8); } @@ -241,7 +234,7 @@ function selectCriticalFiles(feature: FeatureInfo, snapshot: ProjectMap): string .sort((left, right) => (snapshot.fileIndex[right]?.importance ?? 0) - (snapshot.fileIndex[left]?.importance ?? 0) - || left.localeCompare(right) + || compareText(left, right) ) .slice(0, 5); } @@ -298,6 +291,17 @@ function featureId(name: string): string { .replace(/^-|-$/g, "") || "feature"; } +function assignFeatureIds(features: FeatureInfo[]): string[] { + const taken = new Set(); + return features.map((feature) => { + const base = featureId(feature.name); + let id = base; + for (let suffix = 2; taken.has(id); suffix += 1) id = `${base}-${suffix}`; + taken.add(id); + return id; + }); +} + async function removeStaleFeatureMaps(directory: string): Promise { const entries = await readdir(directory, { withFileTypes: true }); await Promise.all(entries diff --git a/packages/cli/src/utils/compare.ts b/packages/cli/src/utils/compare.ts new file mode 100644 index 0000000..2ad99e4 --- /dev/null +++ b/packages/cli/src/utils/compare.ts @@ -0,0 +1,10 @@ +const collator = new Intl.Collator("en"); + +/** + * Text order that does not depend on the machine. `String#localeCompare` follows + * LC_ALL/LANG, so `schema.prisma` and `seed.ts` swap places under cs_CZ ("ch" is + * a letter there) and a snapshot of the same project differs per developer. + */ +export function compareText(left: string, right: string): number { + return collator.compare(left, right); +} diff --git a/packages/cli/test/analyzer-grounding.test.ts b/packages/cli/test/analyzer-grounding.test.ts new file mode 100644 index 0000000..9073ba0 --- /dev/null +++ b/packages/cli/test/analyzer-grounding.test.ts @@ -0,0 +1,326 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { detectFrameworks } from "../src/analyzers/detectors/frameworkDetector.js"; +import { detectEntryPoints } from "../src/analyzers/graph/entryPoints.js"; +import { isArchitectureSource } from "../src/analyzers/graph/sourceScope.js"; +import { inferDomain, type DomainInferenceInput } from "../src/analyzers/inference/domainInference.js"; +import { detectProjectMetadata } from "../src/analyzers/pipeline/projectMetadata.js"; +import { createProjectMap } from "../src/analyzers/pipeline/projectMap.js"; +import { writeAgentNavigationFiles } from "../src/cache/agentNavigation.js"; + +function scanned(path: string, content: string) { + return { + path, + absolutePath: `/fixture/${path}`, + extension: path.slice(path.lastIndexOf(".")), + size: Buffer.byteLength(content), + lines: content.split(/\r?\n/).length, + content + }; +} + +async function withProject(files: Record, run: (root: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), "devmap-grounding-")); + try { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), content); + } + await run(root); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +const PACKAGE = JSON.stringify({ name: "fixture", dependencies: {} }); + +// --------------------------------------------------------------------------- +// Project metadata: nothing is reported that the files do not show +// --------------------------------------------------------------------------- + +test("a project without TypeScript or JavaScript files has an unknown language, not typescript", () => { + const files = [ + scanned("manage.py", "import os\n"), + scanned("app/models.py", "class Article:\n pass\n"), + scanned("requirements.txt", "django\n") + ]; + + assert.equal(detectProjectMetadata("/fixture", "unknown", files).language, "unknown"); +}); + +test("a TypeScript project is still reported as typescript", () => { + const files = [scanned("package.json", PACKAGE), scanned("src/index.ts", "export const a = 1;\n")]; + + assert.equal(detectProjectMetadata("/fixture", "unknown", files).language, "typescript"); +}); + +test("a server framework that is only a devDependency is not the project's framework", () => { + const manifest = JSON.stringify({ + name: "http-client", + main: "./dist/index.js", + types: "./dist/index.d.ts", + devDependencies: { express: "^4.0.0" } + }); + const testServer = scanned("test/helpers/server.ts", 'import express from "express";\nexport const app = express();\n'); + const files = [scanned("package.json", manifest), scanned("src/index.ts", "export const get = () => 1;\n"), testServer]; + + const frameworks = detectFrameworks(files); + const project = detectProjectMetadata("/fixture", "unknown", files, frameworks); + + assert.deepEqual(frameworks, []); + assert.equal(project.framework, "unknown"); + assert.deepEqual(project.projectTypes, ["library"]); +}); + +test("a devDependency server framework counts once production code imports it", () => { + const manifest = JSON.stringify({ name: "bundled-api", devDependencies: { express: "^4.0.0" } }); + const files = [ + scanned("package.json", manifest), + scanned("src/server.ts", 'import express from "express";\nexport const app = express();\n') + ]; + + assert.deepEqual(detectFrameworks(files), ["express"]); +}); + +test("a runtime dependency on a server framework is enough", () => { + const manifest = JSON.stringify({ name: "api", dependencies: { fastify: "^5.0.0" } }); + + assert.deepEqual(detectFrameworks([scanned("package.json", manifest)]), ["fastify"]); +}); + +// The settings-only pnpm-workspace.yaml (onlyBuiltDependencies, overrides) is what pnpm 10 +// projects commonly have; only a `packages` list declares workspace members. +async function metadataOf(files: Record, frameworks: Parameters[3]) { + let project: ReturnType | undefined; + await withProject(files, async (root) => { + project = detectProjectMetadata(root, frameworks?.[0] ?? "unknown", Object.entries(files).map(([path, content]) => scanned(path, content)), frameworks); + }); + return project as ReturnType; +} + +test("pnpm-workspace.yaml without a packages list does not make a monorepo", async () => { + const project = await metadataOf({ + "package.json": JSON.stringify({ name: "app", dependencies: { next: "^15.0.0" } }), + "pnpm-workspace.yaml": "onlyBuiltDependencies:\n - esbuild\n - sharp\n" + }, ["nextjs"]); + + assert.equal(project.workspaceType, "single-package"); +}); + +test("pnpm-workspace.yaml packages make a monorepo, including the inline list form", async () => { + const members = { + "apps/web/package.json": JSON.stringify({ name: "web", dependencies: { next: "^15.0.0" } }), + "packages/ui/package.json": JSON.stringify({ name: "ui" }), + "package.json": JSON.stringify({ name: "root", private: true }) + }; + const block = await metadataOf({ ...members, "pnpm-workspace.yaml": "packages:\n - 'apps/*'\n - \"packages/*\" # shared\n" }, ["nextjs"]); + const inline = await metadataOf({ ...members, "pnpm-workspace.yaml": "packages: [apps/*, packages/*]\n" }, ["nextjs"]); + + assert.equal(block.workspaceType, "monorepo"); + assert.equal(inline.workspaceType, "monorepo"); +}); + +test("a one-off package.json inside an app does not make it a monorepo or a library", async () => { + const project = await metadataOf({ + "package.json": JSON.stringify({ name: "app", dependencies: { "@remix-run/react": "^2.0.0" } }), + "remix.init/package.json": JSON.stringify({ name: "remix.init", private: true, main: "index.js" }) + }, ["remix"]); + + assert.equal(project.workspaceType, "single-package"); + assert.equal(project.projectTypes.includes("library"), false); +}); + +test("declaration files are never entry points, and type-test folders are not production source", () => { + const entries = detectEntryPoints({ + "src/app/routes/auth/user-request.d.ts": ["src/app/models/user.ts"], + "src/app/models/user.ts": [], + "src/index.ts": ["src/app/models/user.ts"] + }); + + assert.deepEqual(entries, ["src/index.ts"]); + assert.equal(isArchitectureSource("test-d/search-params.ts"), false); + assert.equal(isArchitectureSource("src/search/query.ts"), true); +}); + +// --------------------------------------------------------------------------- +// Features: names, ids and search terms stay tied to the project's own evidence +// --------------------------------------------------------------------------- + +test("search terms list only the vocabulary the project's files contain", async () => { + await withProject({ + "package.json": PACKAGE, + "src/auth/login.ts": "export function login() { return 1; }\n", + "src/auth/session.ts": "export function session() { return 2; }\n", + "src/auth/guard.ts": "export function guard() { return 3; }\n" + }, async (root) => { + const authentication = (await createProjectMap(root)).features.find((feature) => feature.name === "Authentication"); + + assert.ok(authentication); + for (const vendor of ["clerk", "auth0", "kinde", "firebase/auth", "@supabase/auth", "better-auth"]) { + assert.equal(authentication.searchTerms.includes(vendor), false, `${vendor} is not in this project`); + } + assert.ok(authentication.searchTerms.includes("login")); + }); +}); + +test("an imported vendor package does appear in the search terms", async () => { + await withProject({ + "package.json": PACKAGE, + "src/auth/login.ts": "export function login() { return 1; }\n", + "src/auth/session.ts": "export function session() { return 2; }\n", + "src/auth/config.ts": 'import NextAuth from "next-auth";\nexport const handler = NextAuth({});\n' + }, async (root) => { + const authentication = (await createProjectMap(root)).features.find((feature) => feature.name === "Authentication"); + + assert.ok(authentication?.searchTerms.includes("next-auth")); + assert.equal(authentication.searchTerms.includes("clerk"), false); + }); +}); + +test("a root page and a demo folder that both singularize to Home get distinct feature names", async () => { + const page = "---\nconst title = 'x';\n---\n

{title}

\n"; + await withProject({ + "package.json": PACKAGE, + "src/pages/index.astro": page, + "src/pages/homes/saas.astro": page, + "src/pages/homes/startup.astro": page + }, async (root) => { + const names = (await createProjectMap(root)).features.map((feature) => feature.name); + + assert.equal(new Set(names).size, names.length, `duplicate names in ${JSON.stringify(names)}`); + assert.ok(names.includes("Home")); + assert.ok(names.includes("Home (homes)")); + }); +}); + +test("agent navigation keeps both features when two names slug to the same id", async () => { + await withProject({ + "package.json": PACKAGE, + "src/auth/login.ts": "export function login() { return 1; }\n", + "src/auth/session.ts": "export function session() { return 2; }\n" + }, async (root) => { + const snapshot = await createProjectMap(root); + const [first] = snapshot.features; + assert.ok(first); + snapshot.features = [ + { ...first, name: "Checklist Item Management" }, + { ...first, name: "checklist-item Management" } + ]; + + const result = await writeAgentNavigationFiles(root, snapshot); + const index = JSON.parse(await readFile(result.indexPath, "utf8")) as { features: Array<{ id: string; map: string }> }; + + assert.deepEqual(index.features.map((feature) => feature.id), ["checklist-item-management", "checklist-item-management-2"]); + assert.equal(result.featurePaths.length, 2); + for (const feature of index.features) { + assert.equal(JSON.parse(await readFile(join(root, feature.map), "utf8")).id, feature.id); + } + }); +}); + +// --------------------------------------------------------------------------- +// Domain inference: a bad model answer or a bad cache never breaks analysis +// --------------------------------------------------------------------------- + +const DOMAIN_INPUT: DomainInferenceInput = { + entityNames: ["Order", "Customer"], + relations: [{ from: "Order", to: "Customer", kind: "belongsTo" }], + capabilities: ["payments"], + technicalFeatures: ["Payments"], + routeCount: 4, + framework: "nextjs", + ownershipPattern: "single_user_isolated", + crossUserFields: [], + absentCapabilities: [] +}; + +function answer(payload: unknown): () => Promise { + return async () => typeof payload === "string" ? payload : JSON.stringify(payload); +} + +test("domain inference accepts relatedEntities written as one string", async () => { + const result = await inferDomain(DOMAIN_INPUT, answer({ + domain: "Order Management", + summary: "Tracks customer orders.", + domainFeatures: [{ name: "Order Tracking", purpose: "Follow an order.", relatedEntities: "Order" }], + confidence: 0.8 + })); + + assert.deepEqual(result?.domainFeatures.map((feature) => feature.relatedEntities), [["Order"]]); +}); + +test("domain inference drops features that name entities the project does not have", async () => { + const result = await inferDomain(DOMAIN_INPUT, answer({ + domain: "Order Management", + summary: "Tracks customer orders.", + domainFeatures: [ + { name: "Loyalty Points", purpose: "Rewards.", relatedEntities: ["LoyaltyAccount"] }, + { name: "Customer Orders", purpose: "Orders per customer.", relatedEntities: ["customer", "Invoice"] } + ], + confidence: 7 + })); + + assert.deepEqual(result?.domainFeatures, [ + { name: "Customer Orders", purpose: "Orders per customer.", relatedEntities: ["Customer"] } + ]); + assert.equal(result?.confidence, 1); +}); + +test("domain inference returns null instead of throwing on unusable answers", async () => { + assert.equal(await inferDomain(DOMAIN_INPUT, answer("not json at all")), null); + assert.equal(await inferDomain(DOMAIN_INPUT, answer({ summary: "no domain field" })), null); + assert.equal(await inferDomain(DOMAIN_INPUT, async () => { throw new Error("offline"); }), null); +}); + +async function cachedDomainRun(poison: (cache: { result: unknown }) => void, fresh: () => Promise) { + let result: Awaited> = null; + await withProject({ "package.json": PACKAGE }, async (root) => { + await inferDomain(DOMAIN_INPUT, answer({ + domain: "Order Management", + summary: "Tracks customer orders.", + domainFeatures: [{ name: "Order Tracking", purpose: "Follow an order.", relatedEntities: ["Order"] }], + confidence: 0.5 + }), root); + + const cachePath = join(root, ".devmap", "domain-cache.json"); + const cache = JSON.parse(await readFile(cachePath, "utf8")); + poison(cache); + await writeFile(cachePath, JSON.stringify(cache)); + + result = await inferDomain(DOMAIN_INPUT, fresh, root); + }); + return result as Awaited>; +} + +test("a cached feature with a malformed entity list is dropped without a new model call", async () => { + let calls = 0; + const result = await cachedDomainRun( + (cache) => { + (cache.result as { domainFeatures: unknown[] }).domainFeatures = [ + { name: "Order Tracking", purpose: "Follow an order.", relatedEntities: 42 } + ]; + }, + async () => { calls += 1; return "{}"; } + ); + + assert.equal(calls, 0); + assert.equal(result?.domain, "Order Management"); + assert.deepEqual(result?.domainFeatures, []); +}); + +test("a cached result that is not an object is a miss and the model is asked again", async () => { + let calls = 0; + const result = await cachedDomainRun( + (cache) => { cache.result = "garbage"; }, + async () => { + calls += 1; + return JSON.stringify({ domain: "Fresh", summary: "Fresh answer.", domainFeatures: [], confidence: 0.4 }); + } + ); + + assert.equal(calls, 1); + assert.equal(result?.domain, "Fresh"); +}); diff --git a/packages/cli/test/analyzer-stability.test.ts b/packages/cli/test/analyzer-stability.test.ts new file mode 100644 index 0000000..95a4f1c --- /dev/null +++ b/packages/cli/test/analyzer-stability.test.ts @@ -0,0 +1,235 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readdir, readFile, rm, symlink, truncate, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { LineIndex } from "../src/analyzers/analysis/lineIndex.js"; +import { MAX_READ_BYTES, scanFiles, type ScannedFile } from "../src/analyzers/analysis/fileScanner.js"; +import { TsMorphAnalyzer } from "../src/analyzers/analysis/tsMorphAnalyzer.js"; +import { detectExternalServices } from "../src/analyzers/detectors/serviceDetector.js"; +import { buildDependencyGraph } from "../src/analyzers/graph/dependencyGraph.js"; +import { createProjectFingerprint, createProjectMap } from "../src/analyzers/pipeline/projectMap.js"; +import { compareText } from "../src/utils/compare.js"; + +function scanned(path: string, content: string): ScannedFile { + return { + path, + absolutePath: `/fixture/${path}`, + extension: path.slice(path.lastIndexOf(".")), + size: Buffer.byteLength(content), + lines: content.split(/\r?\n/).length, + content + }; +} + +async function withTempProject(run: (root: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), "devmap-stability-")); + try { + await run(root); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +async function timed(run: () => Promise | T): Promise<{ value: T; ms: number }> { + const started = performance.now(); + const value = await run(); + return { value, ms: performance.now() - started }; +} + +// The budgets below are an order of magnitude above the measured time and far +// below what the quadratic implementations needed (minutes), so they only trip +// on a real regression and not on a slow CI machine. + +test("LineIndex returns the same line as counting newlines before the position", () => { + const text = "first\nsecond\r\nthird\n\n fifth line\nlast"; + const index = new LineIndex(text); + + for (let position = 0; position <= text.length; position += 1) { + const expected = text.slice(0, position).split("\n").length; + assert.equal(index.lineAt(position), expected, `position ${position}`); + } +}); + +test("a generated file with tens of thousands of declarations is analyzed in linear time", async () => { + const count = 40_000; + const content = Array.from( + { length: count }, + (_, index) => `export function f${index}(a: number, b: string): number { return a + b.length + ${index}; }` + ).join("\n"); + const analyzer = new TsMorphAnalyzer(); + + const { value: analysis, ms } = await timed(() => analyzer.analyze(scanned("src/generated.ts", content), { files: [] })); + + assert.equal(analysis.symbols.length, count); + assert.equal(analysis.symbols[0]?.line, 1); + assert.equal(analysis.symbols[count - 1]?.line, count); + assert.ok(ms < 20_000, `analysis took ${Math.round(ms)}ms`); +}); + +test("symbol lines are right for CRLF sources and for CommonJS exports", async () => { + const content = [ + "const a = 1;", + "", + "function load() {}", + "class Box {", + " open() {}", + "}", + "module.exports = { load, extra: () => 1 };" + ].join("\r\n"); + + const analysis = await new TsMorphAnalyzer().analyze(scanned("src/box.js", content), { files: [] }); + const lineOf = (name: string) => analysis.symbols.find((symbol) => symbol.name === name)?.line; + + assert.equal(lineOf("a"), 1); + assert.equal(lineOf("load"), 3); + assert.equal(lineOf("Box"), 4); + assert.equal(lineOf("open"), 5); + assert.equal(lineOf("extra"), 7); + assert.equal(analysis.symbols.find((symbol) => symbol.name === "load")?.exported, true); +}); + +test("import specifiers are still read from every import form without parser output", () => { + const files = [ + scanned("src/main.ts", [ + 'import a from "./a";', + 'import { b1, b2 } from "./b";', + 'import * as c from "./c";', + 'import type { D } from "./d";', + "import {", + " e1,", + " e2,", + '} from "./e";', + 'export * from "./f";', + 'export { g } from "./g";', + 'const h = require("./h");', + 'const i = await import("./i");', + 'import "./j";' + ].join("\n")), + ...["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"].map((name) => scanned(`src/${name}.ts`, `export const ${name} = 1;\n`)) + ]; + + const { graph } = buildDependencyGraph(files, {}); + + assert.deepEqual( + [...(graph["src/main.ts"] ?? [])].sort(), + ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"].map((name) => `src/${name}.ts`) + ); +}); + +test("import scanning does not rescan a quote-free file from every export", async () => { + const count = 40_000; + const content = Array.from({ length: count }, (_, index) => `export function f${index}() { return ${index}; }`).join("\n"); + const files = [scanned("src/generated.ts", content), scanned("package.json", '{"name":"x"}')]; + + const graph = await timed(() => buildDependencyGraph(files, {})); + const services = await timed(() => detectExternalServices(files)); + + assert.deepEqual(graph.value.graph["src/generated.ts"], []); + assert.deepEqual(services.value, []); + assert.ok(graph.ms < 10_000, `dependency graph took ${Math.round(graph.ms)}ms`); + assert.ok(services.ms < 10_000, `service detection took ${Math.round(services.ms)}ms`); +}); + +test("a very long identifier does not stall the search-term index", async () => { + await withTempProject(async (root) => { + await writeFile(join(root, "package.json"), '{"name":"long-name"}'); + await mkdir(join(root, "src")); + await writeFile(join(root, "src", "index.ts"), `export class ${"A".repeat(300_000)} {}\n`); + + const { value: map, ms } = await timed(() => createProjectMap(root)); + + assert.equal(map.stats.totalFiles, 2); + assert.ok(ms < 20_000, `createProjectMap took ${Math.round(ms)}ms`); + }); +}); + +test("scanFiles lists a file over the size limit without reading it", async () => { + await withTempProject(async (root) => { + await writeFile(join(root, "real.ts"), "export const x = 1;\n"); + await writeFile(join(root, "dataset.csv"), ""); + await truncate(join(root, "dataset.csv"), MAX_READ_BYTES + 1); + + const files = await scanFiles(root); + const dataset = files.find((file) => file.path === "dataset.csv"); + + assert.ok(dataset, "the oversize file is still listed"); + assert.equal(dataset.oversize, true); + assert.equal(dataset.content, ""); + assert.equal(dataset.lines, 0); + assert.equal(dataset.size, MAX_READ_BYTES + 1); + assert.equal(files.find((file) => file.path === "real.ts")?.oversize, undefined); + assert.equal(files.find((file) => file.path === "real.ts")?.lines, 2); + }); +}); + +test("an unread file still changes the fingerprint when its size changes, and is reported", async () => { + await withTempProject(async (root) => { + await writeFile(join(root, "package.json"), '{"name":"with-dump"}'); + await writeFile(join(root, "dump.sql"), ""); + await truncate(join(root, "dump.sql"), MAX_READ_BYTES + 10); + + const before = await scanFiles(root); + const map = await createProjectMap(root); + await truncate(join(root, "dump.sql"), MAX_READ_BYTES + 20); + const after = await scanFiles(root); + + assert.notEqual(createProjectFingerprint(before), createProjectFingerprint(after)); + assert.ok( + map.warnings?.some((warning) => warning.includes("dump.sql") && warning.includes("not read")), + `warnings: ${JSON.stringify(map.warnings)}` + ); + }); +}); + +test("scanFiles counts lines the way split on line breaks does", async () => { + await withTempProject(async (root) => { + await writeFile(join(root, "crlf.ts"), "a\r\nb\nc"); + await writeFile(join(root, "trailing.ts"), "a\n"); + await writeFile(join(root, "empty.ts"), ""); + + const lines = Object.fromEntries((await scanFiles(root)).map((file) => [file.path, file.lines])); + + assert.deepEqual(lines, { "crlf.ts": 3, "empty.ts": 0, "trailing.ts": 2 }); + }); +}); + +test("scanFiles skips a symlinked file that points outside the project and keeps one that stays inside", async () => { + const outside = await mkdtemp(join(tmpdir(), "devmap-outside-")); + try { + await writeFile(join(outside, "secret.ts"), "export const secret = 'not part of the project';\n"); + + await withTempProject(async (root) => { + await writeFile(join(root, "real.ts"), "export const x = 1;\n"); + await symlink(join(root, "real.ts"), join(root, "inside-link.ts")); + await symlink(join(outside, "secret.ts"), join(root, "outside-link.ts")); + + const paths = (await scanFiles(root)).map((file) => file.path); + + assert.deepEqual(paths, ["inside-link.ts", "real.ts"]); + }); + } finally { + await rm(outside, { recursive: true, force: true }); + } +}); + +test("compareText orders names the same way under every machine locale", () => { + assert.ok(compareText("schema.prisma", "seed.ts") < 0, "cs_CZ would sort seed.ts first"); + assert.ok(compareText("index.ts", "Input.module.css") < 0, "tr_TR would treat the dotted i as a different letter"); + assert.deepEqual(["seed.ts", "schema.prisma", "Alpha", "beta"].sort(compareText), ["Alpha", "beta", "schema.prisma", "seed.ts"]); + assert.equal(compareText("same", "same"), 0); +}); + +test("no source file orders text with String#localeCompare", async () => { + const sourceRoot = join(fileURLToPath(new URL("../src", import.meta.url))); + const offenders: string[] = []; + + for (const entry of await readdir(sourceRoot, { recursive: true, withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(".ts")) continue; + const path = join(entry.parentPath, entry.name); + if (/\.localeCompare\(/.test(await readFile(path, "utf8"))) offenders.push(path); + } + + assert.deepEqual(offenders, [], "use compareText from utils/compare.ts so output does not depend on LANG"); +});