diff --git a/CHANGELOG.md b/CHANGELOG.md index 954df86a..b40dda8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,55 @@ Lachesis is pre-1.0. Until 1.0 the graph schema, the query surface and the MCP t may change between minor versions; those changes are called out here explicitly rather than left for you to discover. +## [0.5.2] + +Code-understanding release. 0.5.1 gave the 2.0 Explorer bundle a comprehension +layer; this release makes that layer read like something a newcomer could follow and +scopes it to the code that actually ships. A project can now declare what is and is +not its own source, and the guided request walks, hop captions and concept areas are +reworked so the projection reads as prose over the real call graph rather than a wall +of symbols. Version, projection and build-scoping only: no graph-schema, query-engine +or candidate-surface changes, and the schema 1.0 security bundle is unchanged. + +### Added + +- **`lachesis.yml` project configuration.** A repository can declare its own build and + export knobs in a single `lachesis.yml` at the tree root, resolved from the analysis + root upward. Vendored, generated, build-config and test/example scaffolding are + excluded from the build and from the export projection by default, so the graph and + the comprehension surface describe the product code rather than its dependencies and + fixtures; the defaults are overridable per project. +- **Human-readable hop captions.** Each hop in a guided request walk keeps its exact + symbol (still greppable) and gains a `reads_as` phrase derived from the symbol's own + morphology — a leading action verb rendered over its object tokens — so a reader + follows the lifecycle in plain language without losing the identifier. +- **Module-centric concepts.** `graph.concepts` are now the module areas a newcomer + would name — one per product file, ranked by how much it defines and labelled by its + own module stem — replacing the single coarse call-community that a flat package + collapsed into. +- **Curated tour manifests and recorded symbol documentation** are consumed by the + export when present, and a bounded, source-backed core spine is exported for + exploration. + +### Changed + +- **Request lifecycles root at the real dispatcher and reach their result.** A guided + request now begins at the function that actually dispatches the work rather than the + thin entry trampoline, and the spine is routed forward — over edges recovered from + the call graph, not just the story tree — to the hop that constructs the returned + result. Lifecycle roots are ranked by their control cone so the genuine dispatcher + leads. Entrypoints, request roots and concepts are gated to the repository's primary + language so the projection stays in the repo's own idiom. This changes 2.0 + comprehension output by design; the strict source-backing on every emitted hop is + unchanged. + +### Fixed + +- Node locations are normalized (a null file, line or end-line reads as `""`/`0` + rather than a missing key), and the code-understanding contract is enforced — a + code-understanding projection with no openable entrypoints is rejected rather than + emitted empty. + ## [0.5.1] Explorer-bundle comprehension release. The graph-first 2.0 Explorer bundle already diff --git a/lachesis/cli/analyze.py b/lachesis/cli/analyze.py index 0df8b8b1..6d1162df 100644 --- a/lachesis/cli/analyze.py +++ b/lachesis/cli/analyze.py @@ -255,6 +255,20 @@ def _run(argv: list[str] | None = None) -> None: "exists to reach is never scoped out. An explicitly named file is always " "kept; a directory is walked with the same ignore rules as source_dir.", ) + parser.add_argument( + "--config", metavar="FILE", default=None, + help="path to a lachesis.yml config file. When omitted, the tree is searched " + "upward from source_dir for lachesis.yml (or .yaml/.lachesis.* variants). " + "The config sets what a build ingests, size caps, and runtime knobs; its " + "built-in default excludes tests, examples, docs, fixtures, benchmarks and " + "vendored trees from the graph.", + ) + parser.add_argument( + "--all-sources", action="store_true", + help="disable the default non-product exclusion and compile the whole tree -- " + "tests, examples, docs and vendored code included. Equivalent to a config " + "with `build.exclude: []`, and wins over any config file for this run.", + ) args = parser.parse_args(argv) # Validate the source tree up front. Without this the streaming build path # happily runs against a nonexistent path or a single file, finds no frontend @@ -300,6 +314,27 @@ def _run(argv: list[str] | None = None) -> None: for included in include_paths: if not os.path.exists(included): parser.error(f"--include path does not exist: {included}") + # Resolve the project config (lachesis.yml). Its build.paths filter carries the + # non-product exclusion default -- tests, examples, docs, fixtures, benchmarks and + # vendored trees are dropped from the graph unless the tree opts back in with + # `build.exclude: []`/an allow-list, or this run passes --all-sources. The filter is + # threaded into every build variant *and* into source_content_hash, so a filtered + # build and its cache-validity key describe the very same file set. Any config knob + # that mirrors an env var (the `runtime:` block, atropos root) is applied to the + # environment here, before the first pipeline call reads it; setdefault keeps an + # inherited env var winning over the file, matching the documented precedence. + from lachesis import config as _config + try: + cfg = _config.load(start=args.source_dir, explicit=args.config) + except _config.ConfigError as error: + parser.error(str(error)) + for warning in cfg.warnings: + print(f"lachesis config: {warning}", file=sys.stderr) + if cfg.source: + print(f"lachesis: using config {cfg.source}", file=sys.stderr) + _config.apply_runtime_env(cfg) + # --all-sources wins over the file: complete coverage, no exclusion. + path_filter = None if args.all_sources else cfg.build.paths # --prune deletes pure-lexical/proof records at the store boundary, so apply the # same output defaults before the streaming branch as the ordinary path below. # Previously the early return skipped this block and made --stream-shards run @@ -314,11 +349,13 @@ def _run(argv: list[str] | None = None) -> None: args.source_dir, args.stream_shards, frontend_out, timeout_seconds=args.timeout, max_files_per_package=args.shard_large_packages, + path_filter=path_filter, ) else: readers, snapshots = run_project_streaming( args.source_dir, args.stream_shards, frontend_out, timeout_seconds=args.timeout, include_paths=include_paths, + path_filter=path_filter, ) if not snapshots: parser.error( @@ -374,6 +411,7 @@ def _run(argv: list[str] | None = None) -> None: readers, snapshots = run_project_streaming( args.source_dir, stream_root, frontend_out, timeout_seconds=args.timeout, include_paths=include_paths, + path_filter=path_filter, ) if not snapshots: parser.error( @@ -402,17 +440,20 @@ def _run(argv: list[str] | None = None) -> None: args.source_dir, frontend_out, enrich=compile_enrich, max_workers=args.max_workers, timeout_seconds=args.timeout, max_files_per_package=args.shard_large_packages, + path_filter=path_filter, ) elif args.incremental: graph, snapshots = run_project_incremental(args.source_dir, frontend_out, enrich=compile_enrich, timeout_seconds=args.timeout, - include_paths=include_paths) + include_paths=include_paths, + path_filter=path_filter) else: graph, snapshots = run_project(args.source_dir, frontend_out, enrich=compile_enrich, timeout_seconds=args.timeout, - include_paths=include_paths) + include_paths=include_paths, + path_filter=path_filter) build_fingerprint = None if args.incremental and frontend_out: manifest_path = default_manifest_path(frontend_out) @@ -444,7 +485,8 @@ def _run(argv: list[str] | None = None) -> None: source_dir=args.source_dir if args.reduced else None, # Hashed rather than assumed: the store records what the tree was at build time, # so a load can tell whether an already-joined cache still describes it. - source_content_hash=(source_content_hash(args.source_dir, include_paths=include_paths) + source_content_hash=(source_content_hash(args.source_dir, include_paths=include_paths, + path_filter=path_filter) if args.reduced else None), build_fingerprint=build_fingerprint, ) diff --git a/lachesis/cli/main.py b/lachesis/cli/main.py index d01bd05d..5f55436e 100644 --- a/lachesis/cli/main.py +++ b/lachesis/cli/main.py @@ -358,6 +358,17 @@ def git(*a: str) -> str | None: return repo, commit +def _load_curated_tour(path: str) -> dict: + """Load an OSS tour fragment without accepting a verified owner claim.""" + config = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) + curated_tour = config.get("meta", {}).get("curated_tour", config.get("curated_tour")) + if not isinstance(curated_tour, dict): + raise ValueError("curated tour file must contain meta.curated_tour") + curated_tour = dict(curated_tour) + curated_tour.pop("maintainer", None) + return curated_tour + + def command_trace(args: argparse.Namespace) -> int: """Build (or reuse) a graph and export a lachesis-explorer bundle.json.""" from lachesis.cli.indexer import (EnvironmentProblem, NoSourceFound, @@ -389,6 +400,13 @@ def command_trace(args: argparse.Namespace) -> int: repo, commit = _repo_meta(source) progress.phase("exporting bundle") + curated_tour = None + if args.curated_tour: + try: + curated_tour = _load_curated_tour(args.curated_tour) + except (OSError, UnicodeError, json.JSONDecodeError, AttributeError, ValueError) as error: + _stderr(f"lachesis trace: curated tour: {error}") + return EXIT_USAGE try: bundle = bundle_mod.build_bundle( str(graph_path), @@ -401,6 +419,7 @@ def command_trace(args: argparse.Namespace) -> int: schema_version=args.schema_version, source_url_template=args.source_url_template, description=args.description, + curated_tour=curated_tour, ) except Exception as error: # noqa: BLE001 - CLI turns export errors into one line _stderr(f"lachesis trace: {error}") @@ -657,6 +676,10 @@ def command_build(args: argparse.Namespace) -> int: forwarded.extend(["--stream-shards", args.stream_shards]) for included in getattr(args, "include_paths", None) or []: forwarded.extend(["--include", included]) + if getattr(args, "config", None): + forwarded.extend(["--config", args.config]) + if getattr(args, "all_sources", False): + forwarded.append("--all-sources") return analyze.main(forwarded) @@ -947,6 +970,13 @@ def build_parser() -> argparse.ArgumentParser: help="also analyse this file or directory even if it is outside " "source_dir (repeatable); point it at an advisory's file so a " "narrowed scope never excludes the file the run must reach") + build.add_argument("--config", metavar="FILE", default=None, + help="lachesis.yml to control this build (default: search upward " + "from source_dir). Its built-in default excludes tests, " + "examples, docs, fixtures, benchmarks and vendored trees.") + build.add_argument("--all-sources", action="store_true", + help="compile the whole tree, including tests/examples/docs/vendor " + "(disables the non-product exclusion; wins over any config)") build.set_defaults(handler=command_build, no_prune=False) trace = subcommands.add_parser( @@ -971,6 +1001,8 @@ def build_parser() -> argparse.ArgumentParser: help="explicit HTTP(S) source template using {file}, {line}, {end_line}, {revision}") trace.add_argument("--description", metavar="TEXT", help="one-line projection description recorded in bundle meta (2.0)") + trace.add_argument("--curated-tour", metavar="JSON", + help="read a meta.curated_tour fragment and validate it against the exported paths") trace.add_argument("--per-family", type=_positive_int, default=6, metavar="N", help="max leads to draw from each sink family (default: 6)") trace.add_argument("--max-flows", type=_positive_int, default=40, metavar="N", diff --git a/lachesis/cli/test_cli_args.py b/lachesis/cli/test_cli_args.py index 2c359229..b95306be 100644 --- a/lachesis/cli/test_cli_args.py +++ b/lachesis/cli/test_cli_args.py @@ -1,7 +1,10 @@ +import json +import tempfile import unittest +from pathlib import Path -from lachesis.cli.main import build_parser +from lachesis.cli.main import _load_curated_tour, build_parser class CliArgumentTests(unittest.TestCase): @@ -28,7 +31,21 @@ def test_scan_rejects_invalid_limits_and_ranks(self): with self.subTest(option=option, value=value): with self.assertRaises(SystemExit) as raised: parser.parse_args(["scan", option, value]) - self.assertEqual(raised.exception.code, 2) + self.assertEqual(raised.exception.code, 2) + + def test_curated_tour_argument_and_loader_strip_unverified_identity(self): + args = build_parser().parse_args(["trace", "--curated-tour", "lachesis-tour.json"]) + self.assertEqual(args.curated_tour, "lachesis-tour.json") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "tour.json" + path.write_text(json.dumps({"meta": {"curated_tour": { + "id": "tour.start", "title": "Start here", + "maintainer": {"name": "Untrusted", "verified": True}, + "steps": [{"flow_id": "request.main"}], + }}}), encoding="utf-8") + loaded = _load_curated_tour(str(path)) + self.assertNotIn("maintainer", loaded) + self.assertEqual("tour.start", loaded["id"]) if __name__ == "__main__": diff --git a/lachesis/config.py b/lachesis/config.py new file mode 100644 index 00000000..d8d26859 --- /dev/null +++ b/lachesis/config.py @@ -0,0 +1,473 @@ +"""Project configuration for lachesis (``lachesis.yml``). + +A single optional file that controls what a build ingests, how large a graph or +an export may grow, where the atropos catalog lives, and the runtime knobs that +are otherwise reachable only through ``LACHESIS_*`` environment variables. It is +discovered by walking up from the analysed source tree (and the current working +directory), or named explicitly with ``--config``. + +Precedence, highest first: an explicit CLI flag, then the matching environment +variable, then this file, then the built-in default. A repository that ships a +``lachesis.yml`` therefore changes the defaults for everyone who builds it, while +a one-off flag or env var still wins for a single run. + +Two deliberate departures from "no file means no change": + +* Non-product code — tests, examples, docs, fixtures, benchmarks, vendored + trees, and their common synonyms — is excluded from a build **by default**, + with or without a config file. A tree is understood by its product source; the + scaffolding around it drowns the projection and the architecture ranking. Set + ``build.exclude: []`` (or list an ``build.include`` allow-list) to bring any of + it back. + +* The parser is loaded lazily. PyYAML is imported only when a config file is + actually found, so the stdlib-only core is unaffected for anyone who never + writes one. A file that is present but unparseable because PyYAML is missing is + a hard, explained error rather than a silent skip. + +The schema is intentionally broad and forward-looking: every knob that today is +a CLI flag or an ``LACHESIS_*`` variable has a home here, grouped by the stage it +controls (``build``, ``export``, ``runtime``, ``atropos``). Unknown keys are +reported, not fatal, so a newer config file stays loadable by an older reader. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional, Sequence + +# The names looked for when walking up the tree, in order of preference. The +# dotfile form is accepted so a repo can keep the file out of the way. +CONFIG_FILENAMES = ("lachesis.yml", "lachesis.yaml", ".lachesis.yml", ".lachesis.yaml") + +# The built-in "non-product" exclusion set. A path is non-product when any of its +# directory segments is one of these scaffolding names, or when its basename is a +# test module, a vendored dependency, a generated/minified artifact, or a build +# config. Matching is on path *segments* and basenames, never on substrings, so +# product files whose name merely contains a keyword — ``testing.py``, +# ``templating.py``, ``documentation.py`` — are kept. Validated against the pallets +# /flask tree: of 83 .py files it drops exactly the 41 tests + 17 examples + 1 doc +# and keeps all 24 ``src/flask`` modules. +# +# The set is deliberately language-agnostic — it classifies *paths*, so one rule +# covers Python, TypeScript, JavaScript and C at once. It is the same default a +# review confirmed the graph itself must honor (not only the bundle export): a +# code-property graph over an application has no business modelling a vendored +# dependency, a build output tree, or a generated bundle — a bug found inside one +# is not actionable in the analysed repo, and the noise drowns the product signal +# for comprehension and for security triage alike. Dependencies (``node_modules``, +# ``vendor``, ``third_party``, ``site-packages``), build/CI tooling (``scripts``), +# generated output (``dist``), minified/declaration/build-config artifacts +# (``*.min.js``, ``*.d.ts``, ``*.config.js``) are therefore all dropped by default. +# Any of it is recoverable with an explicit ``build.include`` allow-list or by +# clearing ``build.exclude`` — the opt-in escape hatch for deliberately auditing a +# dependency. +# +# Kept as a single compiled regex (rather than a glob list) because it is the +# default applied on every build and both the builder and the exporter consult +# it; the user-facing ``build.exclude`` list is expressed as globs and compiled +# on top of this. +_NONPRODUCT_SEGMENTS = ( + r"tests?", r"testing", r"__tests__", r"specs?", + r"examples?", r"samples?", r"demos?", + r"docs?", r"documentation", + r"benchmarks?", r"bench", r"perf", + r"fixtures?", r"testdata", r"test[_-]?data", r"__mocks__", r"mocks", + r"vendor", r"vendored", r"third[_-]?party", r"node_modules", r"site-packages", + r"\.tox", r"\.nox", r"\.venv", r"venv", + r"dist", r"scripts", +) +_NONPRODUCT_BASENAMES = ( + r"conftest\.py", + r"test_[^/]*\.py", r"[^/]*_test\.py", + r"[^/]*\.spec\.[a-z]+", r"[^/]*\.test\.[a-z]+", + # Generated / vendored / build-config artifacts: a minified bundle, a + # TypeScript declaration file, and the common ``*.config.js`` build configs + # (rollup/webpack/vite/babel/jest/…) are outputs and scaffolding, not source. + r"[^/]*\.min\.[a-z0-9]+", + r"[^/]*\.d\.ts", + r"[^/]*\.config\.(?:js|cjs|mjs|ts)", +) +NONPRODUCT_RE = re.compile( + r"(?:^|/)(?:" + "|".join(_NONPRODUCT_SEGMENTS) + r")(?:/|$)" + r"|(?:^|/)(?:" + "|".join(_NONPRODUCT_BASENAMES) + r")$", + re.IGNORECASE, +) + + +# Every ``LACHESIS_*`` variable the codebase reads, so a ``runtime:`` block can set +# any of them from the file with the same precedence a shell export would have. A key +# outside this set is still applied (a newer reader may know a variable this one does +# not), but it is reported as a warning so a typo is visible rather than silent. Kept +# here, next to the parser, deliberately: it is the single list a reviewer checks when +# a new env var is introduced. ``ATROPOS_ROOT`` is included though it lacks the prefix +# because the atropos resolver reads it and the ``atropos.root`` knob maps onto it. +KNOWN_RUNTIME_ENV = frozenset({ + "ATROPOS_ROOT", + "LACHESIS_ATROPOS_TIMINGS", "LACHESIS_BIN", "LACHESIS_BIND_SIDECAR", + "LACHESIS_BIND_SIDECAR_MAX_MB", "LACHESIS_BLESS", "LACHESIS_C_CHUNK_FILES", + "LACHESIS_C_JOBS", "LACHESIS_CACHE_DIR", "LACHESIS_CFLAGS", "LACHESIS_COLUMNAR", + "LACHESIS_COMPILE_COMMANDS", "LACHESIS_CONCEPT_CACHE", "LACHESIS_CORPUS_ROOT", + "LACHESIS_DEFER_TRANSLATION_FACTS", "LACHESIS_EMIT_PROOFS", "LACHESIS_EMIT_TOKENS", + "LACHESIS_ENRICH_AT_BUILD", "LACHESIS_ENRICH_SHARDS", "LACHESIS_EQUALITY_HARNESS", + "LACHESIS_EQUALITY_TIER", "LACHESIS_FORMAT", "LACHESIS_FRONTEND_JOBS", + "LACHESIS_GRAPH", "LACHESIS_HARD_STOP", "LACHESIS_HOME", + "LACHESIS_INCLUDE_DEP_TYPES", "LACHESIS_INCLUDE_DIRS_FILE", "LACHESIS_INPROCESS", + "LACHESIS_ISOLATE_NATIVE", "LACHESIS_KUZU_BATCH", "LACHESIS_KUZU_BPS", + "LACHESIS_KUZU_BUFFER_POOL_SIZE", "LACHESIS_KUZU_CHECKPOINT_THRESHOLD", + "LACHESIS_KUZU_LOW_MEMORY", "LACHESIS_KUZU_MAX_DB_SIZE", + "LACHESIS_KUZU_QUERY_THREADS", "LACHESIS_MAX_DEPENDENCY_FILES", + "LACHESIS_MCP_PROFILE", "LACHESIS_MEMORY_BUDGET_MB", "LACHESIS_NATIVE_ATROPOS_LIB", + "LACHESIS_NATIVE_LIFETIME_LIB", "LACHESIS_NO_PROGRESS", "LACHESIS_PASS2_TIMINGS", + "LACHESIS_PROFILE", "LACHESIS_ROOTS_FILE", "LACHESIS_SEMANTIC_SHARDS", + "LACHESIS_SHARD_DIR", "LACHESIS_SHARD_ID", "LACHESIS_SHARD_ROOT", + "LACHESIS_SOURCE_MAP", "LACHESIS_SOURCE_ROOT", "LACHESIS_STREAM_BATCH_ROWS", + "LACHESIS_TIER_VALIDATION", "LACHESIS_TIMEIT", "LACHESIS_TIMEIT_REPORT", + "LACHESIS_TIMINGS", "LACHESIS_TRACEBACK", "LACHESIS_TS_MAX_OLD_SPACE_MB", + "LACHESIS_TS_STACK_KB", +}) + + +def is_nonproduct(relpath: str) -> bool: + """Whether a repo-relative path is scaffolding rather than product source. + + The default build- and export-time filter. Operates on the forward-slashed + repo-relative form (``display_path`` in the frontend), so it is independent of + where the tree lives on disk. + """ + return NONPRODUCT_RE.search(relpath.replace(os.sep, "/")) is not None + + +def _glob_to_regex(pattern: str) -> re.Pattern[str]: + """Translate a path glob to an anchored regex. + + Supports ``**`` (any run of characters including ``/``), ``*`` (any run within + a single path segment), and ``?`` (one non-separator character). A bare name + like ``tests`` matches that segment anywhere in the path, so ``tests`` and + ``tests/**`` and ``**/tests/**`` all do the intuitive thing. + """ + pattern = pattern.strip().replace(os.sep, "/") + out: list[str] = [] + i, n = 0, len(pattern) + while i < n: + c = pattern[i] + if c == "*": + if i + 1 < n and pattern[i + 1] == "*": + out.append(".*") + i += 2 + # A trailing ``/`` after ``**`` may match nothing. + if i < n and pattern[i] == "/": + out.append("/?") + i += 1 + continue + out.append("[^/]*") + i += 1 + continue + if c == "?": + out.append("[^/]") + elif c == "/": + out.append("/") + else: + out.append(re.escape(c)) + i += 1 + body = "".join(out) + # A bare segment name (no separators, no wildcards) matches that segment + # wherever it appears in the path. + if "/" not in pattern and "*" not in pattern and "?" not in pattern: + return re.compile(r"(?:^|/)" + body + r"(?:/|$)") + return re.compile(r"^" + body + r"$") + + +@dataclass(frozen=True) +class PathFilter: + """A compiled exclude/include decision over repo-relative paths. + + ``include`` is an allow-list that wins over ``exclude`` (and over the built-in + non-product default), so a specific example can be kept without disabling the + whole default. ``use_nonproduct_default`` folds the built-in scaffolding set + into the exclusion; the loader clears it when the user sets ``exclude`` to an + explicit list so that ``exclude: []`` genuinely means "compile everything". + """ + + exclude: tuple[re.Pattern[str], ...] = () + include: tuple[re.Pattern[str], ...] = () + use_nonproduct_default: bool = True + + def excluded(self, relpath: str) -> bool: + rel = relpath.replace(os.sep, "/") + if any(p.search(rel) for p in self.include): + return False + if any(p.search(rel) for p in self.exclude): + return True + if self.use_nonproduct_default and is_nonproduct(rel): + return True + return False + + def keep(self, relpath: str) -> bool: + return not self.excluded(relpath) + + +@dataclass(frozen=True) +class BuildConfig: + """What a build ingests and how large it may grow.""" + + paths: PathFilter = field(default_factory=PathFilter) + max_files: Optional[int] = None # cap on files compiled; None = no cap + max_nodes: Optional[int] = None # cap on graph node count; None = no cap + prune_tokens: Optional[bool] = None # None defers to the CLI default (--prune) + timeout_seconds: Optional[int] = None + frontend_jobs: Optional[int] = None + memory_budget_mb: Optional[int] = None + + +@dataclass(frozen=True) +class ExportConfig: + """Bounds and toggles on the comprehension bundle projection.""" + + max_nodes: Optional[int] = None # projection budget + max_entrypoints: Optional[int] = None + include_tests: bool = False # Dense-mode opt-in; default off + paths: PathFilter = field(default_factory=PathFilter) + + +@dataclass(frozen=True) +class AtroposConfig: + """Where the atropos catalog lives and which of it is active.""" + + root: Optional[str] = None # replaces $ATROPOS_ROOT / resolver default + enabled: bool = True + native_lib: Optional[str] = None # $LACHESIS_NATIVE_ATROPOS_LIB + timings: Optional[bool] = None # $LACHESIS_ATROPOS_TIMINGS + languages: Optional[tuple[str, ...]] = None # None = all present + kinds: Optional[tuple[str, ...]] = None # enable-list of sink kinds + flow_patterns: Optional[tuple[str, ...]] = None # enable-list of pattern ids + + +@dataclass(frozen=True) +class Config: + """A resolved configuration. ``source`` is the file it came from, if any.""" + + build: BuildConfig = field(default_factory=BuildConfig) + export: ExportConfig = field(default_factory=ExportConfig) + atropos: AtroposConfig = field(default_factory=AtroposConfig) + runtime: Mapping[str, Any] = field(default_factory=dict) + source: Optional[str] = None + warnings: tuple[str, ...] = () + + +# --------------------------------------------------------------------------- +# Discovery +# --------------------------------------------------------------------------- + +def find_config(start: Optional[str] = None) -> Optional[Path]: + """The nearest config file at or above ``start`` (default: cwd). + + Walks upward to the filesystem root, returning the first match. A build points + ``start`` at the analysed source directory so the tree's own config is found + even when the build is launched from elsewhere. + """ + base = Path(start or os.getcwd()).resolve() + for directory in (base, *base.parents): + for name in CONFIG_FILENAMES: + candidate = directory / name + if candidate.is_file(): + return candidate + return None + + +def _load_yaml(path: Path) -> dict[str, Any]: + """Parse a config file, importing PyYAML lazily. + + A present-but-unparseable file is an error a user needs to see, so a missing + parser is raised with an actionable message rather than swallowed. + """ + try: + import yaml # type: ignore + except ImportError as exc: # pragma: no cover - environment-dependent + raise ConfigError( + f"{path} is present but PyYAML is not installed. Install it with " + f"`pip install lachesis-cpg[config]` (or `pip install pyyaml`), or " + f"remove the file to fall back to defaults." + ) from exc + try: + data = yaml.safe_load(path.read_text()) or {} + except yaml.YAMLError as exc: + raise ConfigError(f"{path} is not valid YAML: {exc}") from exc + if not isinstance(data, dict): + raise ConfigError(f"{path} must be a mapping at the top level, got {type(data).__name__}.") + return data + + +class ConfigError(Exception): + """A config file exists but cannot be honored.""" + + +# --------------------------------------------------------------------------- +# Parsing / resolution +# --------------------------------------------------------------------------- + +def _compile_globs(patterns: Iterable[str]) -> tuple[re.Pattern[str], ...]: + return tuple(_glob_to_regex(str(p)) for p in patterns) + + +def _path_filter(section: Mapping[str, Any], warnings: list[str], where: str) -> PathFilter: + exclude = section.get("exclude") + include = section.get("include", []) or [] + if exclude is None: + # Key absent: keep the non-product default, no explicit patterns. + return PathFilter( + include=_compile_globs(include), + use_nonproduct_default=True, + ) + if not isinstance(exclude, list) or not isinstance(include, list): + warnings.append(f"{where}: `exclude`/`include` must be lists; ignoring.") + return PathFilter(use_nonproduct_default=True) + # An explicit exclude list replaces the default set. `exclude: []` therefore + # means "compile everything", which is the intuitive reading. + return PathFilter( + exclude=_compile_globs(exclude), + include=_compile_globs(include), + use_nonproduct_default=False, + ) + + +def _as_int(value: Any, where: str, warnings: list[str]) -> Optional[int]: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + warnings.append(f"{where}: expected an integer, got {value!r}; ignoring.") + return None + + +def _as_bool(value: Any, where: str, warnings: list[str]) -> Optional[bool]: + if value is None: + return None + if isinstance(value, bool): + return value + warnings.append(f"{where}: expected true/false, got {value!r}; ignoring.") + return None + + +def _as_str_tuple(value: Any) -> Optional[tuple[str, ...]]: + if value is None: + return None + if isinstance(value, str): + return (value,) + if isinstance(value, list): + return tuple(str(v) for v in value) + return None + + +def parse(data: Mapping[str, Any], source: Optional[str] = None) -> Config: + """Turn a raw config mapping into a resolved, typed ``Config``. + + Tolerant by design: a malformed field is dropped with a warning rather than + failing the build, and unknown top-level sections are reported so a config + written for a newer reader still loads. + """ + warnings: list[str] = [] + known = {"build", "export", "atropos", "runtime"} + for key in data: + if key not in known: + warnings.append(f"unknown top-level section {key!r}; ignoring.") + + build_raw = data.get("build") or {} + export_raw = data.get("export") or {} + atropos_raw = data.get("atropos") or {} + runtime_raw = data.get("runtime") or {} + + build = BuildConfig( + paths=_path_filter(build_raw, warnings, "build"), + max_files=_as_int(build_raw.get("max_files"), "build.max_files", warnings), + max_nodes=_as_int(build_raw.get("max_nodes"), "build.max_nodes", warnings), + prune_tokens=_as_bool(build_raw.get("prune_tokens"), "build.prune_tokens", warnings), + timeout_seconds=_as_int(build_raw.get("timeout_seconds"), "build.timeout_seconds", warnings), + frontend_jobs=_as_int(build_raw.get("frontend_jobs"), "build.frontend_jobs", warnings), + memory_budget_mb=_as_int(build_raw.get("memory_budget_mb"), "build.memory_budget_mb", warnings), + ) + export = ExportConfig( + max_nodes=_as_int(export_raw.get("max_nodes"), "export.max_nodes", warnings), + max_entrypoints=_as_int(export_raw.get("max_entrypoints"), "export.max_entrypoints", warnings), + include_tests=bool(_as_bool(export_raw.get("include_tests"), "export.include_tests", warnings) or False), + paths=_path_filter(export_raw, warnings, "export") if ("exclude" in export_raw or "include" in export_raw) else build.paths, + ) + atropos = AtroposConfig( + root=atropos_raw.get("root"), + enabled=bool(_as_bool(atropos_raw.get("enabled"), "atropos.enabled", warnings) if atropos_raw.get("enabled") is not None else True), + native_lib=atropos_raw.get("native_lib"), + timings=_as_bool(atropos_raw.get("timings"), "atropos.timings", warnings), + languages=_as_str_tuple(atropos_raw.get("languages")), + kinds=_as_str_tuple(atropos_raw.get("kinds")), + flow_patterns=_as_str_tuple(atropos_raw.get("flow_patterns")), + ) + if runtime_raw and not isinstance(runtime_raw, dict): + warnings.append("runtime: expected a mapping of LACHESIS_* variables; ignoring.") + runtime_raw = {} + runtime: dict[str, Any] = {} + for key, value in (runtime_raw or {}).items(): + name = str(key) + if name not in KNOWN_RUNTIME_ENV: + warnings.append( + f"runtime.{name}: not a known LACHESIS_* variable; applying anyway." + ) + runtime[name] = value + + return Config( + build=build, + export=export, + atropos=atropos, + runtime=runtime, + source=source, + warnings=tuple(warnings), + ) + + +def _env_str(value: Any) -> str: + """Render a config value the way a shell export would carry it.""" + if isinstance(value, bool): + return "1" if value else "0" + return str(value) + + +def apply_runtime_env(config: Config) -> None: + """Fold config-declared runtime knobs into ``os.environ``. + + Every key of the ``runtime:`` block is a ``LACHESIS_*`` variable, and the atropos + root/native-lib/timings map onto the environment variables their resolver reads. + ``setdefault`` is deliberate: an inherited environment variable still wins over the + file, which is the documented precedence (flag > env > file > default). Idempotent, + and safe to call before any pipeline import — nothing here imports the pipeline. + """ + for name, value in (config.runtime or {}).items(): + os.environ.setdefault(str(name), _env_str(value)) + atropos = config.atropos + if atropos.root: + os.environ.setdefault("ATROPOS_ROOT", str(atropos.root)) + if atropos.native_lib: + os.environ.setdefault("LACHESIS_NATIVE_ATROPOS_LIB", str(atropos.native_lib)) + if atropos.timings is not None: + os.environ.setdefault("LACHESIS_ATROPOS_TIMINGS", _env_str(atropos.timings)) + + +def load(start: Optional[str] = None, explicit: Optional[str] = None) -> Config: + """Discover and parse the config, or return the all-default ``Config``. + + ``explicit`` (a ``--config`` path) is honored verbatim and must exist; + otherwise the tree is searched from ``start``. When no file is found the + returned ``Config`` is all-defaults — which still excludes non-product code, + because that default lives in ``PathFilter`` itself, not in the file. + """ + if explicit: + path = Path(explicit).resolve() + if not path.is_file(): + raise ConfigError(f"--config {explicit} does not exist.") + else: + path = find_config(start) + if path is None: + return Config() + return parse(_load_yaml(path), source=str(path)) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index c0e4afb0..df1ffc3c 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -30,12 +30,18 @@ import hashlib import json import os +import re import subprocess from datetime import datetime, timezone from typing import Any, Optional from lachesis.nav import mcp_server as M +try: + from lachesis.config import is_nonproduct as _is_nonproduct +except Exception: # config is pure-stdlib and same-package, so this should not fail; + _is_nonproduct = None # if it ever does, the gate fails open (keeps everything). + BUNDLE_VERSION = "1.0" FINDING_SCHEMA_VERSION = "0.1" _HEX64 = 64 @@ -45,6 +51,26 @@ def _call(name: str, args: dict) -> Any: return json.loads(M.call_tool(name, args, "json")) +def _is_nonproduct_path(path: Optional[str]) -> bool: + """True when a source path is test/example/docs/benchmark scaffolding. + + The featured comprehension surfaces (entrypoints, request roots, the core spine) + describe what the *product* does, so scaffolding must never seed them. Build-time + exclusion normally keeps such files out of the graph entirely, but the exporter + must not rely on that -- run against a graph built without exclusion, an uncalled + ``test_*`` function is an in-degree-0 callable and would otherwise rank as a + top-of-stack driver, refeaturing exactly the tests the classifier is meant to + drop. Reuses the same classifier the build filter uses, so the two agree; fails + open (keeps the node) only if the classifier is somehow unavailable. + """ + if not path or _is_nonproduct is None: + return False + try: + return bool(_is_nonproduct(path)) + except Exception: + return False + + # --------------------------------------------------------------------- identity def _basename(path: Optional[str]) -> str: @@ -523,38 +549,719 @@ def _norm_node(gl, node: dict) -> dict: "kind": gl.kind(node.get("id")), "file": file, "line": line} -def _call_chain(index, gl, start_id: str, depth: int) -> list[str]: - """A single deterministic CALLS chain out of ``start_id`` (source order). +# The request lifecycle a reader wants is the *success* path; error, teardown and +# logging branches are real but secondary, so we only derank them when choosing the +# primary hop -- never drop them. Word-token match (not raw substring) over the +# identifier keeps this generic and framework-agnostic: it is a vocabulary of +# English failure/teardown verbs, never a hardcoded symbol from one library. +_LIFECYCLE_ERROR_TOKENS = frozenset({ + "exception", "error", "err", "teardown", "cleanup", "abort", "raise", + "rollback", "fail", "reject", "panic", "warn", "log", "logging", +}) +_CALL_EDGE_KINDS = ("CALLS", "INVOKES", "MAY_INVOKE") + +# A special-case/fallback branch is real but is not the lifecycle a reader opens the +# bundle to follow: an auto-generated default reply, a not-found placeholder, an +# unsupported-method stub. Deranked (never dropped) below error branches when picking +# the primary hop, so the spine stays on the ordinary request rather than diving into +# a corner case. Generic English morphology -- matches ``make_default_options_response`` +# or ``handle_not_found`` in any codebase, not a symbol from one framework. +_LIFECYCLE_FALLBACK_TOKENS = frozenset({ + "default", "fallback", "options", "notfound", "missing", "unsupported", + "unavailable", "placeholder", "noop", "stub", "unknown", +}) + +# A request lifecycle culminates in *constructing the thing it returns* -- a response, +# a rendered page, a serialized result. We recognise that terminus by morphology so the +# spine ends there rather than in a routing corner: a construction verb applied to a +# result noun. Generic across codebases (``make_response``, ``build_result``, +# ``render_page``, ``serialize_output``), never a hardcoded framework symbol. +_RESULT_CONSTRUCTION_VERBS = frozenset({ + "make", "build", "create", "construct", "render", "format", "compose", + "produce", "generate", "new", "serialize", "encode", "write", "emit", +}) +_RESULT_NOUNS = frozenset({ + "response", "reply", "result", "output", "answer", "payload", "body", + "page", "document", "content", "view", "html", "json", "template", +}) + + +def _identifier_tokens(name: Optional[str]) -> list[str]: + """Lowercased word tokens of an identifier, splitting snake_case and camelCase. + + ``full_dispatch_request`` -> ``[full, dispatch, request]``; ``makeResponse`` -> + ``[make, response]``; ``__call__`` -> ``[call]``; ``HTTPServer`` -> ``[http, + server]``. The atom every generic morphology check below reasons over, so a rule + keys off whole words rather than raw substrings (no ``err`` inside ``inherit``). + """ + return [t.lower() for t in re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]+|[a-z]+|\d+", + str(name or ""))] + + +def _is_fallback_name(name: Optional[str]) -> bool: + return bool(_LIFECYCLE_FALLBACK_TOKENS.intersection(_identifier_tokens(name))) - At each hop we descend into the callee that itself calls the most -- the branch - most likely to keep telling the request's story -- breaking ties by label so the - walk is reproducible. Cycles are cut by the visited set; a leaf ends the chain. - This invents no ordering: every consecutive pair is a real ``CALLS`` edge. + +def _is_result_construction(name: Optional[str]) -> bool: + """True when an identifier reads as 'construct the returned result'. + + Requires both a construction verb and a result noun as whole tokens, and is not a + fallback/error name -- so ``make_response`` and ``render_page`` qualify while a + special-case ``make_default_options_response`` (fallback) and a plain + ``process_response`` (no construction verb) do not. + """ + toks = set(_identifier_tokens(name)) + if _LIFECYCLE_ERROR_TOKENS.intersection(toks) or _LIFECYCLE_FALLBACK_TOKENS.intersection(toks): + return False + return bool(_RESULT_CONSTRUCTION_VERBS.intersection(toks) and _RESULT_NOUNS.intersection(toks)) + + +# Generic leading-verb -> third-person phrase, so a hop caption reads as what the +# step *does* rather than as a bare symbol. Keyed off the identifier's action token, +# it renders any codebase's ``make_*``/``parse_*``/``dispatch_*`` the same way -- a +# vocabulary of English verbs, never a per-framework symbol table. +_VERB_READS_AS = { + "make": "builds", "build": "builds", "create": "creates", "construct": "constructs", + "new": "creates", "render": "renders", "format": "formats", "compose": "assembles", + "produce": "produces", "generate": "generates", "prepare": "prepares", "wrap": "wraps", + "get": "reads", "fetch": "fetches", "load": "loads", "read": "reads", "find": "finds", + "lookup": "looks up", "resolve": "resolves", "select": "selects", "match": "matches", + "search": "searches", "query": "queries", "collect": "collects", "gather": "gathers", + "dispatch": "dispatches", "route": "routes", "handle": "handles", "process": "processes", + "run": "runs", "execute": "runs", "exec": "runs", "invoke": "invokes", "call": "calls", + "apply": "applies", "perform": "performs", "iter": "iterates over", + "parse": "parses", "decode": "decodes", "deserialize": "deserializes", "unpack": "unpacks", + "encode": "encodes", "serialize": "serializes", "dump": "serializes", "pack": "packs", + "write": "writes", "save": "saves", "store": "stores", "persist": "persists", + "send": "sends", "emit": "emits", "flush": "flushes", "commit": "commits", + "validate": "validates", "check": "checks", "verify": "verifies", "ensure": "ensures", + "sign": "signs", "unsign": "verifies the signature on", "hash": "hashes", + "init": "initializes", "initialize": "initializes", "setup": "sets up", + "configure": "configures", "register": "registers", "bind": "binds", "connect": "connects", + "open": "opens", "close": "closes", "push": "pushes", "pop": "pops", + "add": "adds", "append": "appends", "remove": "removes", "delete": "deletes", + "update": "updates", "set": "sets", "reset": "resets", "clear": "clears", + "preprocess": "preprocesses", "postprocess": "post-processes", "finalize": "finalizes", + "convert": "converts", "transform": "transforms", "normalize": "normalizes", +} +# Modifier/adjective tokens that decorate an identifier without naming its action or +# object; dropped from a caption so ``full_dispatch_request`` reads "dispatches the +# request", not "dispatches the full request". +_CAPTION_FILLER = frozenset({ + "full", "do", "self", "the", "internal", "impl", "inner", "raw", "safe", + "unsafe", "sync", "async", "maybe", "try", "helper", "default", "real", +}) + + +def _readable_caption(name: Optional[str], *, is_entry: bool = False) -> str: + """A short human phrase for a hop: what the step does, from its name's morphology. + + Finds the leading action verb (past any modifier like ``full``/``do``) and renders + it in the third person over the remaining object tokens: ``make_response`` -> + "builds the response", ``full_dispatch_request`` -> "dispatches the request", + ``parse_args`` -> "parses the args". A name with no recognised verb reads as its + humanized noun phrase (an entry as the place to "start"). Never a framework table -- + the same rule renders any codebase, and it degrades to the bare symbol on anything + it cannot parse, so it only ever adds a hint, never hides the identifier. """ - chain = [start_id] - seen = {start_id} - cur = start_id - for _ in range(max(0, depth - 1)): - nxt: list[dict] = [] + toks = _identifier_tokens(name) + if not toks: + return str(name or "step") + verb_i = None + for i, tok in enumerate(toks): + if tok in _VERB_READS_AS: + verb_i = i + break + if tok not in _CAPTION_FILLER: + break # a leading noun-style token: not a verb-first name + if verb_i is not None: + phrase = _VERB_READS_AS[toks[verb_i]] + obj = [t for t in toks[verb_i + 1:] if t not in _CAPTION_FILLER] + return f"{phrase} the {' '.join(obj)}" if obj else phrase + human = " ".join(t for t in toks if t not in _CAPTION_FILLER) or " ".join(toks) + return f"starts at {human}" if is_entry else human + +# Modules that are real product code but *peripheral* to the request lifecycle a +# reader wants first: the command-line front door, generic string/util helpers, the +# in-tree test harness (``testing.py`` -- kept in the graph, but never the headline +# lifecycle). A framework's CLI command has a long, valid execution story, so pure +# spine length floats it above the web path; demoting these modules as lifecycle +# *roots* keeps them in the bundle while letting the dispatch spine lead. Matched on +# the file's basename stem so it stays language-agnostic (cli.py, cli.js, cli.ts). +_PERIPHERAL_MODULE_STEMS = frozenset({ + "cli", "__main__", "__main", "cmd", "cmdline", "commands", "command", + "utils", "util", "helpers", "helper", "testing", "compat", "_compat", +}) + + +def _is_peripheral_module_path(path: Optional[str]) -> bool: + """True when a file is product code but off the primary request lifecycle. + + A soft signal for *ranking* only -- never for inclusion. The stem set is generic + (a CLI front-door, string/util helpers, the test harness); a segment named + ``commands`` catches a management-command package regardless of file name. + """ + if not path: + return False + p = str(path).replace("\\", "/") + stem = p.rsplit("/", 1)[-1].rsplit(".", 1)[0].lower() + if stem in _PERIPHERAL_MODULE_STEMS: + return True + segments = p.lower().split("/") + return "commands" in segments[:-1] + + +def _is_error_name(name: Optional[str]) -> bool: + return bool(_LIFECYCLE_ERROR_TOKENS.intersection(_identifier_tokens(name))) + + +# How many module areas the concept list surfaces. Concepts are the "areas" a reader +# would name (the request lifecycle, templates, sessions, the CLI); we bound them so a +# large tree stays legible while a small one is not padded. +_MAX_CONCEPTS = 12 + + +def _module_stem(path: Optional[str]) -> str: + """The bare module name of a source path: ``pkg/sessions.py`` -> ``sessions``.""" + base = str(path or "").replace("\\", "/").rsplit("/", 1)[-1] + return base.rsplit(".", 1)[0] or base + + +def _concept_label(path: Optional[str], stem_counts) -> str: + """A concept's display label from its module path. + + The module stem alone (``sessions``, ``templating``, ``cli``) is the area name a + reader recognises. Widen to ``parent · stem`` only to break a genuine collision -- + ``app.py`` and ``sansio/app.py`` both stem to ``app`` -- so labels stay short but + never ambiguous. Generic over any layout; never a per-framework name table. + """ + p = str(path or "").replace("\\", "/") + if p.startswith("src/"): + p = p[4:] + stem = _module_stem(p) + if stem_counts.get(stem, 0) > 1 and "/" in p: + parent = p.rsplit("/", 2)[-2] + if parent: + return f"{parent} · {stem}" + return stem + + +def _story_fn_openable(fn: dict) -> bool: + """A story step a reader can open: a real product file and a positive line. + + ``execution_story`` reports unresolved/external callees with a null file; those + are honest frontier markers, never places to root or continue a lifecycle spine. + """ + line = fn.get("line") + return bool(fn.get("file")) and isinstance(line, int) and line > 0 + + +def _reach2(index, node_id: str) -> int: + """Distinct callees within two CALLS hops -- a cheap 'does this drive control?'. + + An orchestration root (a WSGI ``__call__``, a CLI ``main``) fans out into a + broad two-hop cone; a leaf utility barely moves. Ranking candidate roots by this + before paying for a full execution story elevates the real lifecycles without + naming any framework. Bounded by the graph's own fan-out, so it stays cheap. + """ + try: + one = {t.get("id") for t in index.targets(node_id, *_CALL_EDGE_KINDS) + if t.get("id")} + except Exception: + return 0 + total = set(one) + for mid in one: try: - nxt = [n for n in index.targets(cur, "CALLS") - if n.get("id") and n["id"] not in seen] + total.update(t.get("id") for t in index.targets(mid, *_CALL_EDGE_KINDS) + if t.get("id")) except Exception: + continue + total.discard(node_id) + return len(total) + + +# Source-extension -> coarse language family. JavaScript and TypeScript are one +# family (the same web frontend, the same event/handler surface); the C headers +# and sources are one family. Used only to answer "what language is this repo +# primarily", so the grouping is deliberately coarse. +_LANG_BY_EXT = { + "py": "python", "pyi": "python", "pyx": "python", + "js": "web", "jsx": "web", "mjs": "web", "cjs": "web", + "ts": "web", "tsx": "web", "mts": "web", "cts": "web", + "c": "c", "h": "c", "cc": "c", "cpp": "c", "cxx": "c", + "hpp": "c", "hh": "c", "hxx": "c", +} + + +def _language_family(path: Optional[str]) -> Optional[str]: + """The coarse language family of a source path, or None if unrecognised.""" + if not isinstance(path, str): + return None + base = path.replace("\\", "/").rsplit("/", 1)[-1] + if "." not in base: + return None + return _LANG_BY_EXT.get(base.rsplit(".", 1)[-1].lower()) + + +def _primary_language_family(index, gl) -> Optional[str]: + """The dominant product-source language family of the graph, or None. + + Counts product (non-scaffolding) source files by family and returns the family + that is a strict majority. A repo with no clear majority — a genuinely polyglot + tree — returns None, which disables the language gate so nothing is dropped. + + This is the notion the entrypoint and request-lifecycle selection was missing: + on a multi-language repository (a Python framework that ships bundled JavaScript + admin widgets under ``static/``) the JS event handlers otherwise fill every + featured slot, so a newcomer sees a JS widget toolkit instead of the Python + request path. The gate keeps the projection in the language the repo actually is. + """ + counts: dict[str, int] = {} + try: + for node in index.nodes_of_kind("file"): + f = gl.loc(node)[0] or gl.prop(node, "file") + if not f or _is_nonproduct_path(f): + continue + fam = _language_family(f) + if fam: + counts[fam] = counts.get(fam, 0) + 1 + except Exception: + return None + if not counts: + return None + top = max(counts, key=lambda k: counts[k]) + if counts[top] * 2 <= sum(counts.values()): + return None # no strict majority -> polyglot -> do not gate + return top + + +def _descend_trampoline(index, gl, nid: str, *, + primary_family: Optional[str] = None, limit: int = 4) -> str: + """Skip thin forwarders so a lifecycle root is the real orchestrator. + + A WSGI ``Flask.__call__`` is a one-line trampoline: ``return self.wsgi_app(...)``. + Rooting the story at it prepends a meaningless hop and, worse, makes the *entry* + of the request the trampoline rather than the dispatcher a reader wants named. + While the current node forwards to exactly one product callee of the repo's + dominant language (a single direct CALLS target), descend to it. Bounded by + ``limit`` and a ``seen`` set so a mutually-recursive pair can never loop. + """ + seen = {nid} + for _ in range(limit): + try: + callees = [t.get("id") for t in index.targets(nid, "CALLS") if t.get("id")] + except Exception: + return nid + openable = [] + for cid in callees: + if cid in seen: + continue + node = gl.nodes.get(cid) + if node is None: + continue + f, l = gl.loc(node)[0], gl.loc(node)[1] + if not f or not isinstance(l, int) or l <= 0 or _is_nonproduct_path(f): + continue + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + continue + openable.append(cid) + if len(openable) != 1: + return nid + nid = openable[0] + seen.add(nid) + return nid + + +def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int, + primary_family: Optional[str] = None) -> list[str]: + """Candidate roots for request-lifecycle stories, best driver first. + + Two sources, deduped in priority order: the planner's entry handlers (already + ranked upstream), then every product callable that nothing else in the product + calls -- an in-degree-0 top-of-stack (a WSGI ``__call__``, an event loop, a + public API orchestrator). The in-degree-0 set is ordered by two-hop reach so the + orchestration roots precede the many leaf helpers that also happen to be + uncalled once tests are excluded. Truncated to ``cap`` so the story pass is + bounded regardless of codebase size. + """ + # Both sources feed one ranked candidate pool. A planner handler is *not* an + # automatic front-of-line: a framework like Click emits dozens of thin decorator + # handlers (``version_option``, ``argument``) that each spin a valid but peripheral + # story, and if they were kept ahead of the drivers they would fill ``cap`` and + # starve the real dispatcher (``Command.main``, in-degree-0, widest cone) out of the + # pass entirely. Ranking every candidate by two-hop reach means the widest-cone + # lifecycle always survives the cap; the downstream story pass re-ranks the + # survivors, so this ordering governs only *which* candidates it gets to see. + candidates: list[tuple[int, str]] = [] + seen: set[str] = set() + + def _consider(nid: str, node: Optional[dict]) -> None: + if not nid or nid in seen: + return + f = gl.loc(node)[0] if node is not None else None + if node is None or _is_nonproduct_path(f): + return # a test/example handler is not a product lifecycle root + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + return # a non-primary-language handler (bundled JS in a Python repo) + seen.add(nid) + candidates.append((_reach2(index, nid), nid)) + + for hid in handler_ids: + _consider(hid, gl.nodes.get(hid) if hid else None) + + try: + callable_nodes = list(index.nodes_of_kind("function", "method", "constructor")) + except Exception: + callable_nodes = [] + for node in callable_nodes: + nid = node.get("id") + if not nid or nid in seen: + continue + f, l = gl.loc(node)[0], gl.loc(node)[1] + if not f or not isinstance(l, int) or l <= 0: + continue + # An uncalled test_* function is in-degree-0; exclude scaffolding so it never + # ranks as a top-of-stack driver on a graph built without build-time exclusion. + if _is_nonproduct_path(f): + continue + # ...and never a non-primary-language driver: a bundled JS handler in a + # Python repo is in-degree-0 too, but it is not this repo's request path. + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + continue + try: + out = sum(1 for _ in index.targets(nid, *_CALL_EDGE_KINDS)) + if out < 1: + continue + inn = sum(1 for _ in index.sources(nid, *_CALL_EDGE_KINDS)) + except Exception: + continue + if inn == 0: + # An in-degree-0 root is often a thin WSGI/entry trampoline (Flask.__call__, + # Click's BaseCommand.__call__); descend to the real orchestrator it forwards + # to *before* ranking, so the driver is ordered by the dispatcher's own + # control cone (Click's main reaches far more than its one-line __call__) and + # the lifecycle is named at the dispatcher rather than the forwarder. + driver = _descend_trampoline(index, gl, nid, primary_family=primary_family) + _consider(driver, gl.nodes.get(driver)) + + candidates.sort(key=lambda pair: (-pair[0], pair[1])) + return [nid for _, nid in candidates][:cap] + + +def _hop_semantics(via: str, branch: dict) -> dict: + """Per-hop reader facts from an execution-story step: how it is reached and + whether it decides. ``reached_via`` names the call-seam boundary (a direct call + vs a dynamic dispatch the graph resolved), and the branch summary flags a hop + that forks control -- the decision points a newcomer traces. All derived from + real story structure; absent facts are simply omitted so hops stay compact. + """ + out: dict = {} + v = via or "" + if v == "entry": + out["reached_via"] = "entry" + elif v == "direct": + out["reached_via"] = "direct call" + elif v.startswith("indirect:"): + out["reached_via"] = f"dynamic dispatch ({v.split(':', 1)[1] or 'resolved'})" + elif v: + out["reached_via"] = v + count = branch.get("count") or 0 + if count: + out["decides"] = True + out["branch_count"] = count + kinds = branch.get("kinds") or [] + if kinds: + out["decision_kinds"] = kinds + return out + + +def _story_spine(story: dict, index, gl, *, max_hops: int) -> tuple[list[str], list[str], dict]: + """Linearize an execution story into (primary success spine, all functions, meta). + + The story is a call tree keyed by (caller -> function). The spine walks from the + entry always choosing the direct-edge, deepest-subtree callee, deranking obvious + error/teardown branches, so it follows the happy path (a WSGI entry down through + dispatch to the response) rather than wandering into a handler. Every consecutive + pair on the spine is a real edge the story observed; cycles are cut by ``seen``. + Returns the ordered spine node ids, the flat set of every function id the story + touched (the raw material for the architecture core), and a per-spine-node + semantics map (how each hop is reached, whether it branches). + + ``index``/``gl`` let the walk recover call edges the story *tree* attached to a + different parent (see ``_candidate_children``): the story visits each function + once, so a genuine callee can hang off an earlier caller than the one whose body + actually makes the call, and a tree-only walk could never reach it. + """ + steps = story.get("steps") or [] + entry = (story.get("entry") or {}).get("node_id") + if not entry: + return [], [], {} + children: dict[str, list[tuple[int, dict, str]]] = {} + functions: dict[str, dict] = {} + branches: dict[str, dict] = {} + for step in steps: + fn = step.get("function") or {} + fid = fn.get("node_id") + if not fid: + continue + functions[fid] = fn + # Per-function control facts: how many decision points the body has and which + # control kinds -- surfaced on the hop as its decision signal. + rows = step.get("branches") or [] + kinds = sorted({r.get("control") for r in rows if r.get("control")}) + branches[fid] = {"count": step.get("branch_count") or 0, "kinds": kinds} + caller = (step.get("caller") or {}).get("node_id") + if caller: + children.setdefault(caller, []).append( + (step.get("sequence", 0), fn, step.get("via") or "")) + + def _candidate_children(node_id: str) -> list[tuple[dict, str]]: + """Callees to consider when extending the spine from ``node_id``. + + The execution story is a *tree*: each function is attached under its + first-discovered caller, so a real callee can hang off a different parent + than the one whose body makes the call. Flask's ``finalize_request`` (which + builds the response) lands under ``handle_exception`` in the tree, not under + ``full_dispatch_request`` whose call actually reaches it -- so a walk over + story-children alone can never route the spine to the response terminus. + Recover the missing edges from the graph: every genuine callee of + ``node_id`` that the story itself visited becomes a candidate, carrying its + story fn record and a via classified from the edge kind. This invents no + nodes (only functions already in the story are admitted) and no edges the + graph does not hold; it merely lets the spine follow the real call an + earlier caller happened to be credited with in the tree. + """ + out: list[tuple[dict, str]] = [] + story_ids: set[str] = set() + for _seq, fn, via in sorted(children.get(node_id, []), key=lambda t: t[0]): + cid = fn.get("node_id") + if cid: + story_ids.add(cid) + out.append((fn, via)) + try: + direct = {t.get("id") for t in index.targets(node_id, _CALL_EDGE_KINDS[0]) + if t.get("id")} + callees = [t.get("id") for t in index.targets(node_id, *_CALL_EDGE_KINDS) + if t.get("id")] + except Exception: + return out + added: set[str] = set() + for t in callees: + if t in story_ids or t in added or t not in functions: + continue + added.add(t) + out.append((functions[t], + "direct" if t in direct else "indirect:may_invoke")) + return out + + memo: dict[str, int] = {} + + def subtree(nid: str, guard: frozenset) -> int: + if nid in memo: + return memo[nid] + if nid in guard: + return 0 + deeper = guard | {nid} + total = 0 + for _, fn, _via in children.get(nid, []): + cid = fn.get("node_id") + if cid: + total += 1 + subtree(cid, deeper) + # Only cache when no guard cycle influenced the count (guard was the path + # to nid); good enough as a heuristic ranker and keeps the walk bounded. + memo[nid] = total + return total + + reach_memo: dict[str, bool] = {} + + def reaches_result(nid: str, guard: frozenset) -> bool: + """Does this subtree build the value the request returns? A response, a + rendered page, a serialized result -- recognised by morphology (see + ``_is_result_construction``), so the spine can end at the response terminus + rather than in a routing corner. Bounded and cycle-guarded like ``subtree``. + """ + if nid in reach_memo: + return reach_memo[nid] + if nid in guard: + return False + if _is_result_construction((functions.get(nid) or {}).get("name")): + reach_memo[nid] = True + return True + deeper = guard | {nid} + found = any(cid and reaches_result(cid, deeper) + for _, fn, _via in children.get(nid, []) + for cid in (fn.get("node_id"),)) + reach_memo[nid] = found + return found + + spine = [entry] + seen = {entry} + cur = entry + meta: dict[str, dict] = {entry: _hop_semantics("entry", branches.get(entry) or {})} + while len(spine) < max_hops: + kids = [(fn, via) + for fn, via in _candidate_children(cur) + if fn.get("node_id") not in seen and _story_fn_openable(fn)] + if not kids: break - if not nxt: - break + # Rank each candidate hop, best first, by five generic signals: + # 1. a direct CALLS edge is the real control flow; ``indirect:may_invoke`` + # hops are duck-typed over-approximations (a session deserialize, a JSON + # dump that *might* run), so direct wins; + # 2. error/teardown branches derank (real, but not the success path); + # 3. special-case/fallback branches derank next (an auto OPTIONS reply, a + # not-found stub -- a corner, not the ordinary request); + # 4. a branch that reaches the response/result construction is preferred, so + # the spine ends where the request builds what it returns + # (full_dispatch_request -> finalize_request -> make_response) rather than + # tunnelling into the widest routing subtree and stopping at a corner; + # 5. the deepest subtree breaks any remaining tie. + # Every signal is morphology over the identifier, never a framework symbol. + pick = max(kids, key=lambda kv: ( + 1 if kv[1] == "direct" else 0, + 0 if _is_error_name(kv[0].get("name")) else 1, + 0 if _is_fallback_name(kv[0].get("name")) else 1, + 1 if reaches_result(kv[0].get("node_id"), frozenset()) else 0, + subtree(kv[0].get("node_id"), frozenset()))) + nid = pick[0].get("node_id") + meta[nid] = _hop_semantics(pick[1], branches.get(nid) or {}) + cur = nid + seen.add(nid) + spine.append(nid) + ordered_functions = [fid for fid in functions if _story_fn_openable(functions[fid])] + return spine, ordered_functions, meta + + +def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], *, + max_requests: int, max_core: int, + max_hops: int, + primary_family: Optional[str] = None) -> tuple[list[dict], list[dict]]: + """Request lifecycles and the architecture core, from bounded execution stories. + + Runs a bounded forward execution story from each candidate driver (see + ``_lifecycle_roots``), ranks them by how much real control each covers (spine + length, then breadth), and keeps the deepest few as guided request paths -- each + the success spine of one story, every consecutive hop a real observed edge. A + shallower story whose root already sits inside a kept spine is skipped, so we do + not emit both ``__call__ -> wsgi_app -> ...`` and its ``wsgi_app -> ...`` suffix. + The union of every kept story's functions, bounded, becomes the core spine a + newcomer reads first. Best-effort: any failure yields empty lists, never raises. + """ + requests: list[dict] = [] + core: list[dict] = [] + try: + roots = _lifecycle_roots(index, gl, handler_ids, cap=30, + primary_family=primary_family) + except Exception: + return requests, core - def out_degree(node: dict) -> int: + ranked: list[tuple[int, int, int, int, str, list[str], list[str], dict]] = [] + for root in roots: + try: + story = _call("execution_story", + {"entry": root, "max_depth": max_hops + 2, + "max_steps": 120, "format": "json"}) + except Exception: + continue + if not isinstance(story, dict): + continue + spine, functions, meta = _story_spine(story, index, gl, max_hops=max_hops) + if len(spine) < 2: + continue + # A peripheral root (a CLI command, a util helper) still yields a long, valid + # story, so ranking on spine length alone floats it above the web request path + # a reader opened the bundle to see. Demote by the root's module so the primary + # dispatch lifecycle leads; the peripheral path is kept, just not first. + root_file = gl.loc(gl.nodes.get(root))[0] if gl.nodes.get(root) else None + demote = 1 if _is_peripheral_module_path(root_file) else 0 + # The dispatcher a reader wants first is the top-of-stack that drives the most + # code, not whichever helper happens to keep the longest in-library spine. The + # true lifecycle (Flask.wsgi_app, Click's Command.main -> invoke) exits to + # external user code quickly, so its openable spine is *short* even though its + # control cone is the widest; a string helper (secho -> echo -> isatty) stays + # in-library and spins a longer spine. Ranking by 2-hop reach first puts the + # real driver on top; spine length only breaks ties between comparable drivers. + reach = _reach2(index, root) + ranked.append((demote, -reach, len(spine), len(functions), root, spine, functions, meta)) + + # Primary lifecycles first (widest control cone), then deepest, then broadest. + ranked.sort(key=lambda row: (row[0], row[1], -row[2], -row[3], row[4])) + + node_ids = set(asm.nodes) + covered: set[str] = set() + core_ids: set[str] = set() + used_ids: set[str] = set() + for _, _, _, _, root, spine, functions, meta in ranked: + if len(requests) >= max_requests: + break + if root in covered: # a redundant suffix of a spine already shown + continue + hops: list[dict] = [] + chain_ids: list[str] = [] + for nid in spine: + node = gl.nodes.get(nid) + if node is None: + continue + asm.add_node(_norm_node(gl, node), default_kind="function") + node_ids.add(nid) + label = gl.label(node) + # ``caption`` stays the exact symbol (a reader can grep it); ``reads_as`` + # adds a human phrase derived from the symbol's morphology, so the hop + # says what the step does ("dispatches the request") without hiding the + # identifier. First hop on the spine is the entry -- phrased as a start. + hop = {"node_id": nid, "caption": label, + "reads_as": _readable_caption(label, is_entry=not hops)} + hop.update(meta.get(nid) or {}) + hops.append(hop) + chain_ids.append(nid) + if len(hops) < 2: + continue + for a, b in zip(chain_ids, chain_ids[1:]): + asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, node_ids) + covered.update(chain_ids) + root_label = gl.label(gl.nodes.get(root)) or "entry" + rid = f"request.{_slug(root_label)}" + if rid in used_ids: + rid = f"{rid}.{_slug(root)}" + used_ids.add(rid) + requests.append({ + "id": rid, + "kind": "call-path", + "description": f"Request lifecycle from {root_label} through " + f"{len(hops) - 1} call(s).", + "entry_node": root, + "hops": hops, + }) + # The architecture core draws from every kept story's functions, spine first. + for nid in [*chain_ids, *(f for f in functions if f not in chain_ids)]: + if len(core_ids) >= max_core: + break + if nid in core_ids: + continue + node = gl.nodes.get(nid) + if node is None: + continue + f, l = gl.loc(node)[0], gl.loc(node)[1] + if not f or not isinstance(l, int) or l <= 0: + continue + if _is_nonproduct_path(f): + continue # keep scaffolding off the architecture core + asm.add_node(_norm_node(gl, node), default_kind="function") + node_ids.add(nid) try: - return len(index.targets(node["id"], "CALLS")) + degree = sum(1 for _ in index.targets(nid, *_CALL_EDGE_KINDS)) except Exception: - return 0 - - pick = min(nxt, key=lambda n: (-out_degree(n), gl.label(n), n["id"])) - cur = pick["id"] - seen.add(cur) - chain.append(cur) - return chain + degree = 0 + core.append({"node_id": nid, "label": gl.label(node), + "file": f, "line": l, "degree": degree}) + core_ids.add(nid) + return requests, core def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, @@ -567,7 +1274,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, never raises: a graph the comprehension layer cannot walk simply reads as a bare graph rather than failing the whole export. """ - empty = {"entrypoints": [], "requests": [], "files": [], "modules": []} + empty = {"entrypoints": [], "requests": [], "files": [], "modules": [], "concepts": [], "core": []} try: from lachesis.planner.entrypoints import EntryPoints, _anchor_strength ctx = M.ctx() @@ -576,8 +1283,15 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: return empty + # The repo's dominant product language. A multi-language tree (a Python framework + # that bundles JavaScript admin widgets) otherwise features the wrong language: + # its JS event handlers rank as entrypoints and fill every request flow. Gating to + # the primary family keeps the projection in the language the repo actually is. + primary_family = _primary_language_family(index, gl) + entrypoints: list[dict] = [] requests: list[dict] = [] + used_ids: set[str] = set() try: by_handler = EntryPoints(store).by_handler() # Strongest anchor per handler, then a stable global order over handlers. @@ -587,7 +1301,6 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, key=lambda kv: (_anchor_strength(kv[1]), kv[1].get("file") or "", kv[1].get("anchor_label") or "", kv[0])) - used_ids: set[str] = set() for handler_id, anchor in ordered: if len(entrypoints) >= max_entrypoints: break @@ -599,6 +1312,15 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, # file and line, or it is not a place a developer can actually begin. if not nfile or not isinstance(nline, int) or nline <= 0: continue + # ...and it must be product code -- never a test/example handler. + if _is_nonproduct_path(nfile): + continue + # ...and in the repo's primary language -- never a bundled JS admin + # widget standing in for the request path of a Python framework. + if primary_family is not None: + fam = _language_family(nfile) + if fam is not None and fam != primary_family: + continue asm.add_node(_norm_node(gl, node), default_kind="function") how = anchor.get("how") label = gl.label(node) @@ -618,33 +1340,66 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, "file": efile, "line": nline, }) - - # A guided path is only worth showing when it actually goes somewhere: - # the real CALLS chain out of the entry must have more than the entry. - chain = _call_chain(index, gl, handler_id, chain_depth) - if len(chain) < 2: - continue - hops = [] - for nid in chain: - cnode = gl.nodes.get(nid) - if cnode is None: - continue - asm.add_node(_norm_node(gl, cnode), default_kind="function") - hops.append({"node_id": nid, "caption": gl.label(cnode)}) - for a, b in zip(chain, chain[1:]): - asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, set(asm.nodes)) - if len(hops) >= 2: - requests.append({ - "id": f"request.{_slug(label)}", - "kind": "call-path", - "description": f"Follow control from {label} through " - f"{len(hops) - 1} call(s).", - "entry_node": handler_id, - "hops": hops, - }) except Exception: pass + # Guided request paths are no longer a greedy CALLS walk out of each exported + # symbol -- that surfaced leaf utilities (render_template) and never the request + # lifecycle. Instead root them at the real top-of-stack drivers and follow the + # success spine of each one's bounded execution story (wsgi __call__ -> wsgi_app + # -> full_dispatch_request -> dispatch_request -> ...). The same stories yield the + # architecture core, so both are built together below. + handler_ids = [entry["node_id"] for entry in entrypoints] + requests, core = _lifecycle_projection( + asm, index, gl, handler_ids, + max_requests=8, max_core=32, max_hops=max(2, chain_depth), + primary_family=primary_family) + + # Promote each lifecycle root to an entrypoint. ``by_handler`` only recognises + # module-level public helpers, so a framework's real request driver -- a WSGI + # ``Flask.wsgi_app``, an event loop -- is *never* an anchored handler and would + # otherwise be a request whose entry is nowhere in the entrypoint set. These + # drivers are the truest "begin reading here" nodes, so they lead the list. This + # also gives a library with no anchored handler at all (itsdangerous) a real, + # source-backed entrypoint, which the code-understanding contract requires. + entry_node_ids = {entry["node_id"] for entry in entrypoints} + promoted: list[dict] = [] + for req in requests: + root = req.get("entry_node") + if not root or root in entry_node_ids: + continue + node = gl.nodes.get(root) + if node is None: + continue + nfile, nline = gl.loc(node)[0], gl.loc(node)[1] + if not nfile or not isinstance(nline, int) or nline <= 0: + continue + if _is_nonproduct_path(nfile): + continue + if primary_family is not None: + fam = _language_family(nfile) + if fam is not None and fam != primary_family: + continue + entry_node_ids.add(root) + label = gl.label(node) + eid = f"entry.{_slug(label)}" + if eid in used_ids: + eid = f"{eid}.{_slug(root)}" + used_ids.add(eid) + try: + efile = comp._relative_path(nfile) or nfile + except Exception: + efile = nfile + promoted.append({ + "id": eid, + "label": label, + "kind": "request-lifecycle", + "node_id": root, + "file": efile, + "line": nline, + }) + entrypoints = promoted + entrypoints + files: list[dict] = [] try: seen_paths: set[str] = set() @@ -660,10 +1415,53 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: files = [] + # Concepts are the module *areas* a newcomer would name: the request lifecycle, + # routing, request context, templates, sessions, the CLI. Call-community + # clustering is too coarse here -- a flat single-package framework (every file in + # one directory) collapses into one giant community, so the whole request path, + # templating and session code read as a single undifferentiated blob. Derive areas + # from the *modules* instead: one concept per product file, ranked by how much it + # defines (definition count, then path for a stable order), capped at + # ``_MAX_CONCEPTS``. Fully generic -- the busiest modules of any codebase are its + # areas, named by their own path, never a framework symbol table -- and it degrades + # to an empty list, never raises. Ranking on the definition count alone keeps this a + # single cheap node scan (no per-node graph query), so it stays bounded on a large + # tree where an edge lookup per function would dominate the export. + concepts: list[dict] = [] + try: + import collections as _collections + defs: "_collections.Counter" = _collections.Counter() + for node in index.nodes_of_kind("function", "method", "constructor"): + f = gl.loc(node)[0] + if not f or _is_nonproduct_path(f): + continue + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + continue # keep the concept list in the repo's own language + try: + rel = comp._relative_path(f) or f + except Exception: + rel = f + defs[rel] += 1 + ranked_modules = sorted(defs.items(), key=lambda kv: (-kv[1], kv[0])) + top = ranked_modules[:_MAX_CONCEPTS] + stem_counts = _collections.Counter(_module_stem(rel) for rel, _ in top) + for rel, n in top: + concepts.append({ + "id": f"concept.{_slug(rel)}", + "label": _concept_label(rel, stem_counts), + "description": f"The {_module_stem(rel)} module ({n} definition(s)).", + "file_paths": [rel], + }) + except Exception: + concepts = [] + # Modules are not built here: they must partition the *final* included node # pool (one unambiguous module per node, keyed by that node's file), which is # only settled after candidate/capsule/entry nodes are all in and relativized. - return {"entrypoints": entrypoints, "requests": requests, "files": files} + return {"entrypoints": entrypoints, "requests": requests, "files": files, + "concepts": concepts, "core": core} # ------------------------------------------------------- source / node enrichment @@ -788,6 +1586,33 @@ def _enrich_graph_nodes(nodes: list[dict], gl) -> None: module = _dotted_module(node.get("file")) if module: node["qualified_name"] = f"{module}.{node.get('label')}" + # Scope: the enclosing callable every node lives in (problem #6 -- scope was + # absent on most nodes). owner_function returns the node itself when it is + # already a callable, so a function/method reports the module it belongs to, + # while an operand or a value reports the function that contains it. This is + # the container a frontend groups by, never an empty field. + try: + owner = gl.owner_function(twin) + except Exception: + owner = None + scope = None + if owner is not None and owner.get("id") != twin.get("id"): + owner_file, _os, _oe = gl.loc(owner) + owner_module = _dotted_module(owner_file) + owner_label = gl.label(owner) + scope = f"{owner_module}.{owner_label}" if owner_module else owner_label + if not scope: + scope = module + if scope: + node["scope"] = scope + for key in ("documentation", "docstring", "comment"): + try: + documentation = gl.prop(twin, key) + except Exception: + documentation = None + if isinstance(documentation, str) and documentation.strip(): + node["documentation"] = documentation.strip() + break try: excerpt = gl.source_excerpt(twin) except Exception: @@ -870,6 +1695,14 @@ def _finalize_requests(raw_requests: list[dict], node_map: dict, nid = hop.get("node_id") entry = {"id": f"{rid}:{i:02d}", "node_id": nid, "caption": hop.get("caption")} + if hop.get("reads_as"): + entry["reads_as"] = hop["reads_as"] + # Carry the story-derived hop semantics (how this hop is reached, whether + # it forks control) through decoration so a reader sees the call-seam and + # decision points, not just an ordered list of names. + for key in ("reached_via", "decides", "branch_count", "decision_kinds"): + if hop.get(key) is not None: + entry[key] = hop[key] if i > 1: entry["edge_label"] = _edge_label( edges_by_pair, hops[i - 2].get("node_id"), nid) @@ -906,6 +1739,11 @@ def _partition_modules(nodes: list[dict], entrypoints: list[dict]) -> list[dict] f = node.get("file") if not isinstance(f, str) or not f.strip(): continue + # Non-product files (tests, docs, examples, vendored deps, generated output) + # must not surface as modules a reader is invited to explore. The same gate the + # entrypoint/request selection uses, applied to the module partition. + if _is_nonproduct_path(f): + continue module_name = _dotted_module(f) or f node["module"] = module_name groups.setdefault(f, []).append(node["id"]) @@ -926,11 +1764,136 @@ def _partition_modules(nodes: list[dict], entrypoints: list[dict]) -> list[dict] return modules +def _project_concepts(raw_concepts: list[dict], nodes: list[dict]) -> list[dict]: + """Keep architecture concepts honest to the final included node pool. + + A concept is dropped entirely when every file it spans is non-product, and its + node set is restricted to product files, so a vendored dependency + (``node_modules · typescript · lib``), a build config (``rollup.config.js``), or a + docs/scripts tree never surfaces as an architecture concept — even on a graph + built without build-time exclusion. + """ + out: list[dict] = [] + for concept in raw_concepts or []: + paths = {str(path) for path in concept.get("file_paths") or [] if path + and not _is_nonproduct_path(str(path))} + if not paths: + continue + node_ids = [node["id"] for node in nodes + if isinstance(node.get("file"), str) and node.get("file") in paths + and not _is_nonproduct_path(node.get("file"))] + if not node_ids: + continue + out.append({ + "id": str(concept.get("id") or f"concept.{len(out)}"), + "label": str(concept.get("label") or "Code area"), + "description": str(concept.get("description") or "Connected code area."), + "node_ids": node_ids[:20], + }) + return out + + +def _project_curated_tour(raw: Optional[dict], values: list[dict], requests: list[dict]) -> Optional[dict]: + """Keep only tour steps that resolve in this exact exported projection. + + Tour files are user-authored convenience metadata, not evidence. A changed + repository can make an old flow or anchor disappear, so stale steps are + omitted instead of making the entire export fail. Maintainer identity is + deliberately not accepted from this unauthenticated file path. + """ + if not isinstance(raw, dict): + return None + title = str(raw.get("title") or "Start here").strip() + tour_id = str(raw.get("id") or "tour.start-here").strip() + if not title or not tour_id: + return None + paths = {str(path.get("id")): path for path in [*values, *requests] + if isinstance(path, dict) and path.get("id")} + steps: list[dict] = [] + for item in raw.get("steps") or []: + if not isinstance(item, dict): + continue + flow_id = str(item.get("flow_id") or item.get("flowId") or "").strip() + path = paths.get(flow_id) + if not path: + continue + raw_steps = path.get("steps") if isinstance(path.get("steps"), list) else path.get("hops") + node_ids = {str(step.get("node_id")) for step in raw_steps or [] + if isinstance(step, dict) and step.get("node_id")} + node_id = item.get("node_id") or item.get("nodeId") + if node_id is not None and str(node_id) not in node_ids: + continue + step = {"flow_id": flow_id} + if node_id is not None: + step["node_id"] = str(node_id) + for key in ("label", "note"): + if item.get(key) is not None and str(item[key]).strip(): + step[key] = str(item[key]).strip() + steps.append(step) + if not steps: + return None + result = {"id": tour_id, "title": title, "steps": steps} + description = str(raw.get("description") or "").strip() + if description: + result["description"] = description[:500] + overview = raw.get("overview") + if isinstance(overview, dict): + overview_description = str(overview.get("description") or "").strip() + overview_result = {"description": overview_description[:1000]} if overview_description else {} + concepts = overview.get("concepts") + if isinstance(concepts, list): + selected_concepts = [] + for item in concepts[:8]: + if not isinstance(item, dict) or not str(item.get("id") or "").strip() or not str(item.get("label") or "").strip(): + continue + concept = {"id": str(item["id"]).strip(), "label": str(item["label"]).strip()} + if str(item.get("description") or "").strip(): + concept["description"] = str(item["description"]).strip()[:300] + related = item.get("related_ids") + if isinstance(related, list) and related: + concept["related_ids"] = [str(value) for value in related if str(value).strip()][:8] + selected_concepts.append(concept) + if selected_concepts: + overview_result["concepts"] = selected_concepts + if overview_result: + result["overview"] = overview_result + selection = raw.get("selection") + if isinstance(selection, dict): + allowed = ("include_tests", "include_examples", "include_generated") + selected = {key: value for key, value in selection.items() if key in allowed and isinstance(value, bool) and value} + if selected: + result["selection"] = selected + return result + + +def _normalize_node_location(node: dict) -> None: + """Coerce a node's ``file``/``line``/``end_line`` to their 2.0 field types in place. + + Synthetic nodes (heap locations, summary objects) have no source and were + emitting ``file: null, line: null``; the 2.0 contract is ``file: ""`` and + ``line: 0`` -- a real absence, not a missing key of unknown type -- so a reader + can uniformly test ``line > 0`` for openability. ``end_line`` is made mandatory + and never less than ``line`` (a single-line span when no wider extent is known, + ``0`` for synthetics). Normalizing null to ""/0 does not change which nodes count + as source-backed: ``_has_source`` already rejects an empty file and a non-positive + line, so featured-path and entrypoint selection are unaffected. + """ + file = node.get("file") + node["file"] = file if isinstance(file, str) and file.strip() else "" + line = node.get("line") + line = line if isinstance(line, int) and not isinstance(line, bool) and line > 0 else 0 + node["line"] = line + end = node.get("end_line") + node["end_line"] = end if (isinstance(end, int) and not isinstance(end, bool) + and end >= line) else line + + def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[str], lang: Optional[str], indexed_nodes: int, source_url_template: Optional[str] = None, comprehension: Optional[dict] = None, - description: Optional[str] = None) -> dict: + description: Optional[str] = None, + curated_tour: Optional[dict] = None) -> dict: """Adapt the assembled evidence into Explorer's graph-first 2.0 contract. The security envelope remains available under ``security.findings``. The @@ -942,7 +1905,19 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s language = str(lang or meta.get("lang") or "unknown") revision = str(commit or meta.get("commit") or "unknown") findings = bundle.get("findings") or [] + # `security.findings` stays exhaustive (passed through untouched below); only the + # *featured* value paths are cleaned. Two hygiene rules (problems #7 and #8): + # #7 A value path that visits fewer than two distinct nodes has not moved -- + # it is a bare def-use artifact (a traceback local `tb`, a file handle `f`, + # `config_file`, `tb.tb_frame`), not a behavior. Featuring it as one is the + # reported defect. Genuine flows (`hashlib.sha1`, `send_file`, `re.split`, + # `Markup`) always traverse a source and a distinct sink, so the two-distinct + # -node floor drops exactly the artifacts and keeps every real flow, including + # the minimal two-step call-argument flows. + # #8 Identical paths (same endpoints and same ordered node ids) are collapsed to + # one; the graph often yields the same def-use twice from different findings. values = [] + seen_paths: set[tuple] = set() for finding in findings: witness = finding.get("witness") or {} steps = witness.get("steps") or [] @@ -951,14 +1926,23 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s finding_id = str(finding.get("finding_id") or "") if not finding_id: continue + step_ids = tuple(step.get("node_id") for step in steps) + if len({sid for sid in step_ids if sid}) < 2: + continue # #7: a path that never leaves one node is not a behavior + source_node = steps[0].get("node_id") + sink_node = steps[-1].get("node_id") + dedupe_key = (source_node, sink_node, step_ids) + if dedupe_key in seen_paths: + continue # #8: same endpoints and same ordered hops -- one is enough + seen_paths.add(dedupe_key) path_id = f"value:{finding_id}" values.append({ "id": path_id, "kind": "value-flow", "name": finding.get("display_name") or "value path", "description": finding.get("result_summary") or "Exporter-provided value path", - "source_node": steps[0].get("node_id"), - "sink_node": steps[-1].get("node_id"), + "source_node": source_node, + "sink_node": sink_node, "confidence": (finding.get("analysis") or {}).get("confidence"), "limitations": list((finding.get("analysis") or {}).get("limitations") or []), "steps": steps, @@ -966,6 +1950,8 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s graph = bundle.get("graph") or {} nodes = graph.get("nodes") or [] + for node in nodes: + _normalize_node_location(node) node_map = {n.get("id"): n for n in nodes} node_ids = set(node_map) edges = _canonical_edges(graph.get("edges") or [], node_ids) @@ -976,6 +1962,10 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s if e.get("node_id") in node_ids] requests = _finalize_requests(comp.get("requests") or [], node_map, edges_by_pair) modules = _partition_modules(nodes, entrypoints) + concepts = _project_concepts(comp.get("concepts") or [], nodes) + core = [item for item in (comp.get("core") or []) + if item.get("node_id") in node_ids] + tour = _project_curated_tour(curated_tour, values, requests) coverage = { "scope": "repository-projection", @@ -1011,12 +2001,16 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s "edges": edges, "files": comp.get("files") or [], "modules": modules, + "concepts": concepts, "entrypoints": entrypoints, + "core": core, "coverage": coverage, }, "paths": {"requests": requests, "values": values}, "security": {"findings": findings}, } + if tour is not None: + v2["meta"]["curated_tour"] = tour _validate_graph_first(v2) return v2 @@ -1043,6 +2037,22 @@ def _validate_graph_first(bundle: dict) -> None: if not nodes or None in node_ids: raise ValueError("graph-first bundle has invalid nodes") + # Every node carries the concrete 2.0 location types -- ``file`` a string + # (``""`` when absent), ``line`` and ``end_line`` non-negative ints with the + # span never inverted. Synthetic nodes (heap locations) legitimately report + # ``""``/``0``; what is rejected is the earlier ``null`` leak, which left the + # field's type undefined for consumers. + for node in nodes: + nid = node.get("id") + if not isinstance(node.get("file"), str): + raise ValueError(f"node {nid} file must be a string") + line = node.get("line") + if not isinstance(line, int) or isinstance(line, bool) or line < 0: + raise ValueError(f"node {nid} line must be an int >= 0") + end = node.get("end_line") + if not isinstance(end, int) or isinstance(end, bool) or end < line: + raise ValueError(f"node {nid} end_line must be an int >= line") + coverage = graph.get("coverage") or {} if coverage and coverage.get("included_nodes") != len(nodes): raise ValueError("graph-first coverage.included_nodes must equal node count") @@ -1058,6 +2068,21 @@ def _validate_graph_first(bundle: dict) -> None: if not _has_source(node_map.get(nid)): raise ValueError(f"entrypoint {entry.get('id')} node has no openable source") + # A comprehension-first projection is meaningless without a boundary to enter + # from and a path with enough hops to be a story. An empty entrypoint set (the + # ItsDangerous case) or paths that never exceed a bare def-use pair defeat the + # whole projection, so they are rejected here rather than shipped as a hollow + # bundle. Request hops are already proven source-backed above, so a >=3-hop + # request is a source-backed path of three or more hops by construction. + if bundle.get("analysis_projection") == "code-understanding": + if not (graph.get("entrypoints") or []): + raise ValueError( + "code-understanding projection requires at least one production entrypoint") + requests = (bundle.get("paths") or {}).get("requests") or [] + if not any(len(req.get("hops") or []) >= 3 for req in requests): + raise ValueError( + "code-understanding projection requires a source-backed path of >= 3 hops") + seen_module_nodes: set[str] = set() for module in graph.get("modules") or []: for nid in module.get("node_ids") or []: @@ -1092,6 +2117,7 @@ def build_bundle(graph_path: str, *, repo: Optional[str] = None, schema_version: str = "1.0", source_url_template: Optional[str] = None, description: Optional[str] = None, + curated_tour: Optional[dict] = None, max_entrypoints: int = 40, chain_depth: int = 6, max_files: int = 2000) -> dict: """Build an explorer bundle (schema 1.0) from a built+enriched graph.""" @@ -1198,7 +2224,8 @@ def build_bundle(graph_path: str, *, repo: Optional[str] = None, commit=commit or prov.get("commit_sha"), lang=lang, indexed_nodes=int(load.get("nodes") or 0), source_url_template=source_url_template, - comprehension=projection, description=description) + comprehension=projection, description=description, + curated_tour=curated_tour) if schema_version != "1.0": raise ValueError(f"unsupported Explorer schema version: {schema_version}") return bundle diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 2fad9c2a..523a1aff 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -224,6 +224,19 @@ def test_edge_referencing_unknown_node_rejected(self): class GraphFirstBundleTests(unittest.TestCase): + def _comprehension(self): + # A code-understanding bundle must carry a production entrypoint and a + # >= 3-hop source-backed path; the legacy nodes (source@3, sink@8) back both. + return { + "entrypoints": [{"id": "entry.source", "label": "input", "kind": "parameter", + "node_id": "source", "file": "src/a.c", "line": 3}], + "requests": [{"id": "request.flow", "kind": "call-path", "description": "d", + "entry_node": "source", + "hops": [{"node_id": "source", "caption": "receives"}, + {"node_id": "sink", "caption": "executes"}, + {"node_id": "source", "caption": "returns"}]}], + } + def _legacy_bundle(self): return { "meta": {"repo": "GNOME/libxml2", "lang": "c", "commit": "abc", "loc": 42}, @@ -251,6 +264,7 @@ def _legacy_bundle(self): def test_graph_first_uses_v2_contract_and_supported_source_placeholders(self): result = bundle._graph_first_bundle( self._legacy_bundle(), repo="GNOME/libxml2", commit="abc", lang="c", indexed_nodes=99, + comprehension=self._comprehension(), source_url_template="https://github.com/GNOME/libxml2/blob/{revision}/{file}#L{line}") self.assertEqual(result["schema_version"], "2.0") self.assertEqual(result["meta"]["indexed_nodes"], 99) @@ -261,7 +275,8 @@ def test_graph_first_uses_v2_contract_and_supported_source_placeholders(self): def test_graph_first_does_not_guess_source_host(self): result = bundle._graph_first_bundle( - self._legacy_bundle(), repo="group/project", commit="abc", lang="c", indexed_nodes=2) + self._legacy_bundle(), repo="group/project", commit="abc", lang="c", indexed_nodes=2, + comprehension=self._comprehension()) self.assertNotIn("source_url_template", result["meta"]) def test_graph_first_rejects_invalid_path_reference(self): @@ -279,6 +294,27 @@ def _sourced(self, nid, file, line, label): return {"id": nid, "kind": "function", "file": file, "line": line, "label": label, "snippet": f"def {label}(): ...", "end_line": line + 2} + # A valid code-understanding bundle always has a production entrypoint and a + # source-backed path of >= 3 hops (the NR2 contract). The helpers inject these + # defaults so a test focused on some other facet (concepts, tour, module dup) + # still builds a contract-valid base; a test provides its own to override. + def _default_entrypoints(self): + return [{"id": "entry.wsgi_app", "label": "wsgi_app", "kind": "http-handler", + "node_id": "n.a", "file": "src/flask/app.py", "line": 10}] + + def _default_requests(self): + return [{"id": "request.baseline", "kind": "call-path", "description": "baseline", + "entry_node": "n.a", + "hops": [{"node_id": "n.a", "caption": "receives"}, + {"node_id": "n.b", "caption": "dispatches"}, + {"node_id": "n.a", "caption": "returns"}]}] + + def _with_defaults(self, comprehension): + comp = dict(comprehension) + comp.setdefault("entrypoints", self._default_entrypoints()) + comp.setdefault("requests", self._default_requests()) + return comp + def _bundle_with(self, comprehension): legacy = { "meta": {"repo": "pallets/flask", "lang": "python", "commit": "abc", "loc": 100}, @@ -299,7 +335,24 @@ def _bundle_with(self, comprehension): } return bundle._graph_first_bundle( legacy, repo="pallets/flask", commit="abc", lang="python", - indexed_nodes=500, comprehension=comprehension) + indexed_nodes=500, comprehension=self._with_defaults(comprehension)) + + def _bundle_with_tour(self, comprehension, curated_tour): + legacy = { + "meta": {"repo": "pallets/flask", "lang": "python", "commit": "abc", "loc": 100}, + "graph": { + "nodes": [self._sourced("n.a", "src/flask/app.py", 10, "wsgi_app"), + self._sourced("n.b", "src/flask/app.py", 20, "dispatch_request")], + "edges": [{"source": "n.a", "target": "n.b", "kind": "CALLS"}], + }, + "findings": [{"finding_id": "a" * 64, "display_name": "x", "result_summary": "y", + "analysis": {"confidence": "high", "limitations": []}, + "witness": {"steps": [{"node_id": "n.a", "role": "origin"}, + {"node_id": "n.b", "role": "sink"}]}}], + } + return bundle._graph_first_bundle( + legacy, repo="pallets/flask", commit="abc", lang="python", indexed_nodes=500, + comprehension=self._with_defaults(comprehension), curated_tour=curated_tour) def test_full_projection_shapes_graph_and_paths(self): result = self._bundle_with({ @@ -309,8 +362,12 @@ def test_full_projection_shapes_graph_and_paths(self): "requests": [{"id": "request.lifecycle", "kind": "call-path", "description": "d", "entry_node": "n.a", "hops": [{"node_id": "n.a", "caption": "receives"}, - {"node_id": "n.b", "caption": "dispatches"}]}], + {"node_id": "n.b", "caption": "dispatches"}, + {"node_id": "n.c", "caption": "runs"}]}], "files": [{"id": "f1", "path": "src/flask/app.py"}], + "concepts": [{"id": "concept.flask", "label": "flask", "description": "d", + "file_paths": ["src/flask/app.py"]}], + "core": [{"node_id": "n.a", "label": "wsgi_app", "degree": 4}], }) self.assertEqual(result["meta"]["indexed_nodes"], 500) self.assertEqual(result["graph"]["coverage"]["included_nodes"], @@ -320,7 +377,7 @@ def test_full_projection_shapes_graph_and_paths(self): self.assertEqual(len(result["graph"]["entrypoints"]), 1) req = result["paths"]["requests"][0] self.assertEqual(req["source_node"], "n.a") - self.assertEqual(req["sink_node"], "n.b") + self.assertEqual(req["sink_node"], "n.c") self.assertEqual(req["hops"][0]["id"], "request.lifecycle:01") self.assertEqual(req["hops"][1]["edge_label"], "calls") # edges are first-class: id + canonical kind + relation alias. @@ -333,11 +390,47 @@ def test_full_projection_shapes_graph_and_paths(self): self.assertEqual(by_path["src/flask/app.py"]["anchor_node_id"], "n.a") seen = [nid for m in result["graph"]["modules"] for nid in m["node_ids"]] self.assertEqual(len(seen), len(set(seen))) + self.assertEqual(["n.a", "n.b"], result["graph"]["concepts"][0]["node_ids"]) + self.assertEqual(["n.a"], [item["node_id"] for item in result["graph"]["core"]]) + + def test_concept_without_included_nodes_is_dropped(self): + result = self._bundle_with({ + "concepts": [{"id": "concept.missing", "label": "missing", + "file_paths": ["src/other/missing.py"]}], + }) + self.assertEqual([], result["graph"]["concepts"]) + + def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): + result = self._bundle_with_tour( + {"concepts": [{"id": "concept.lifecycle", "label": "Lifecycle", "description": "d", "file_paths": ["src/flask/app.py"]}], "requests": [{"id": "request.lifecycle", "kind": "call-path", + "description": "d", "entry_node": "n.a", + "hops": [{"node_id": "n.a", "caption": "a"}, + {"node_id": "n.b", "caption": "b"}, + {"node_id": "n.a", "caption": "c"}]}]}, + {"id": "tour.start", "title": "Start here", "description": "Read this first.", + "maintainer": {"name": "Ignored"}, + "overview": {"description": "Read the request lifecycle first.", "concepts": [{"id": "concept.lifecycle", "label": "Request lifecycle", "description": "The main request route."}]}, + "selection": {"include_tests": True, "include_examples": False, "include_generated": True}, + "steps": [{"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, + {"flow_id": "request.missing"}]}, + ) + self.assertEqual({"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, + result["meta"]["curated_tour"]["steps"][0]) + self.assertNotIn("maintainer", result["meta"]["curated_tour"]) + self.assertEqual("Read the request lifecycle first.", result["meta"]["curated_tour"]["overview"]["description"]) + self.assertEqual("concept.lifecycle", result["meta"]["curated_tour"]["overview"]["concepts"][0]["id"]) + self.assertEqual({"include_tests": True, "include_generated": True}, result["meta"]["curated_tour"]["selection"]) + + def test_curated_tour_is_omitted_when_no_step_resolves(self): + result = self._bundle_with_tour( + {}, + {"id": "tour.start", "title": "Start here", "steps": [{"flow_id": "missing"}]}, + ) + self.assertNotIn("curated_tour", result["meta"]) def test_request_with_unsourced_hop_is_dropped(self): # n.ghost has no source; the guided path must not be emitted. result = self._bundle_with({ - "entrypoints": [], "requests": [{"id": "r", "kind": "call-path", "description": "d", "entry_node": "n.a", "hops": [{"node_id": "n.a", "caption": "a"}, @@ -348,22 +441,25 @@ def test_request_with_unsourced_hop_is_dropped(self): self.assertEqual(len(result["paths"]["requests"]), 1) def test_validator_rejects_coverage_mismatch(self): - result = self._bundle_with({"entrypoints": [], "requests": []}) + result = self._bundle_with({}) result["graph"]["coverage"]["included_nodes"] += 1 with self.assertRaises(ValueError): bundle._validate_graph_first(result) def test_validator_rejects_entrypoint_without_source(self): - result = self._bundle_with({"entrypoints": [], "requests": []}) + result = self._bundle_with({}) + # Valid 2.0 location types but no openable source ("" / 0): the entrypoint + # check must still reject it for lacking a real location. result["graph"]["nodes"].append({"id": "n.bare", "kind": "function", - "label": "bare"}) + "label": "bare", "file": "", "line": 0, + "end_line": 0}) result["graph"]["coverage"]["included_nodes"] = len(result["graph"]["nodes"]) result["graph"]["entrypoints"].append({"id": "entry.bare", "node_id": "n.bare"}) with self.assertRaises(ValueError): bundle._validate_graph_first(result) def test_validator_rejects_duplicate_module_node(self): - result = self._bundle_with({"entrypoints": [], "requests": []}) + result = self._bundle_with({}) result["graph"]["modules"].append( {"id": "module.dup", "name": "dup", "path": "x", "node_ids": ["n.a"]}) with self.assertRaises(ValueError): @@ -387,6 +483,30 @@ def test_has_source_requires_file_line_and_text(self): self.assertTrue(bundle._has_source( {"file": "a.py", "line": 3, "source_window": {"lines": ["x"]}})) + def test_enrich_graph_nodes_preserves_recorded_documentation(self): + class _Graph: + nodes = {"n": {"id": "n", "label": "serve", + "properties": {"file": "src/app.py", "start_line": 4, + "end_line": 8, "documentation": "Serve a request."}}} + + def loc(self, node): + props = node["properties"] + return props["file"], props["start_line"], props["end_line"] + + def prop(self, node, key, default=None): + return node.get("properties", {}).get(key, default) + + def source_excerpt(self, node): + return "serve()" + + def _read_file(self, _path): + return "def serve():\n pass\n" + + nodes = [{"id": "n", "file": "src/app.py", "line": 4, + "label": "serve", "snippet": "serve"}] + bundle._enrich_graph_nodes(nodes, _Graph()) + self.assertEqual("Serve a request.", nodes[0]["documentation"]) + def test_count_source_lines_sums_physical_lines_dedups_and_falls_back(self): # a.py: 3 physical lines read off disk; b.py: unreadable, falls back to # its file-node span end (2); the duplicate a.py node is counted once. diff --git a/lachesis/pipeline.py b/lachesis/pipeline.py index b2c6716c..6a1dc372 100644 --- a/lachesis/pipeline.py +++ b/lachesis/pipeline.py @@ -4,7 +4,10 @@ import hashlib import os from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple + +if TYPE_CHECKING: + from .config import PathFilter from .core.contract import ContractError as FrontendError, FrontendSnapshot from .core.composition import _EdgeKeys @@ -101,6 +104,7 @@ def source_inventory( source_dir: str, include_tests: bool = True, include_paths: Sequence[str] = (), + path_filter: Optional["PathFilter"] = None, ) -> List[str]: """Discover every supported source file, including tests and specifications. @@ -108,6 +112,14 @@ def source_inventory( because its path looks like a test. Callers that explicitly need a production-only inventory may still pass ``include_tests=False``. + ``path_filter`` is the general form of that opt-out. When supplied (the CLI resolves + one from ``lachesis.yml``, whose built-in default excludes tests, examples, docs, + fixtures, benchmarks and vendored trees), a walked file is dropped when the filter + reports its repo-relative path as excluded. It never vetoes an *explicitly named* + ``include_paths`` file — the guided-scope guarantee that a deliberately scoped file + is always analysed outranks the exclusion default — but it does apply while walking + an explicitly named ``include_paths`` *directory*, matching the main walk. + ``include_paths`` names extra files or directories to fold into the inventory even when they lie *outside* ``source_dir``. This is the guided-scope guarantee: when a build is deliberately narrowed to a sub-tree to fit a time budget, the advisory's @@ -179,6 +191,13 @@ def _walk(base_dir: str, containment_root: str) -> List[str]: continue if is_test is not None and is_test(path): continue + if path_filter is not None: + # Match on the repo-relative, forward-slashed display form so the + # decision is independent of where the tree lives on disk and lines + # up with the ``display_path`` the frontend later records. + rel = os.path.relpath(path, containment_root) + if path_filter.excluded(rel): + continue collected.append(path) return collected @@ -260,6 +279,7 @@ def run_project( include_paths: Sequence[str] = (), *, enrich: bool = False, + path_filter: Optional["PathFilter"] = None, ) -> Tuple[CodeGraph, List[FrontendSnapshot]]: """Run selected frontends and compose the canonical core graph. @@ -276,7 +296,8 @@ def run_project( source_dir = os.path.abspath(source_dir) registry = registry or default_registry() groups = registry.partition( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths)) + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter)) snapshots = [] for frontend_id in sorted(groups): frontend = registry.get(frontend_id) @@ -342,6 +363,7 @@ def run_project_streaming( timeout_seconds: int = 300, include_tests: bool = True, include_paths: Sequence[str] = (), + path_filter: Optional["PathFilter"] = None, ): """Run frontends one at a time and return shard readers plus metadata. @@ -355,7 +377,8 @@ def run_project_streaming( output_root = os.path.abspath(output_root) registry = registry or default_registry() groups = registry.partition( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths)) + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter)) snapshots = [] readers = [] from .resources import c_chunk_files, frontend_jobs as configured_frontend_jobs @@ -438,6 +461,7 @@ def run_project_streaming_parallel( *, max_files_per_package: Optional[int] = None, workspace_root: Optional[str] = None, + path_filter: Optional["PathFilter"] = None, ): """Stream package/shard compiler jobs without composing their snapshots. @@ -454,11 +478,13 @@ def run_project_streaming_parallel( output_root = os.path.abspath(output_root) registry = registry or default_registry(workspace_root) packages = detect_packages( - source_dir, source_inventory(source_dir, include_tests=include_tests), + source_dir, + source_inventory(source_dir, include_tests=include_tests, path_filter=path_filter), ) packages = split_large_packages(source_dir, packages, max_files_per_package) jobs = package_jobs(source_dir, output_root, registry, - include_tests=include_tests, packages=packages) + include_tests=include_tests, packages=packages, + path_filter=path_filter) if not jobs: supported = sorted({ extension for item in registry.frontends for extension in item.extensions @@ -553,6 +579,7 @@ def source_content_hash( source_dir: str, include_tests: bool = True, include_paths: Sequence[str] = (), + path_filter: Optional["PathFilter"] = None, ) -> str: """One digest over every source file a build of ``source_dir`` would see. @@ -564,7 +591,8 @@ def source_content_hash( unchanged content keeps the cache, and a restored older file loses it. """ digests = _group_digests( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths), + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter), os.path.abspath(source_dir)) digest = hashlib.sha256() for path in sorted(digests): @@ -658,6 +686,7 @@ def run_project_incremental( include_paths: Sequence[str] = (), *, enrich: bool = True, + path_filter: Optional["PathFilter"] = None, ) -> Tuple[CodeGraph, List[FrontendSnapshot]]: """Like ``run_project`` but reuse a frontend's prior on-disk bundle when none of its source files changed, recompiling only the frontends that did. @@ -674,7 +703,8 @@ def run_project_incremental( registry = registry or default_registry() manifest_path = manifest_path or default_manifest_path(output_root) groups = registry.partition( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths)) + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter)) prior = _load_manifest(manifest_path) snapshots: List[FrontendSnapshot] = [] @@ -735,6 +765,7 @@ def package_jobs( registry: FrontendRegistry, include_tests: bool = True, packages: Optional[Dict[str, List[str]]] = None, + path_filter: Optional["PathFilter"] = None, ) -> List[Tuple[str, str, str, str, List[str]]]: """The (frontend_id, package, compile_root, output_dir, roots) units of a build. @@ -753,7 +784,8 @@ def package_jobs( output_root = os.path.abspath(output_root) if packages is None: packages = detect_packages( - source_dir, source_inventory(source_dir, include_tests=include_tests), + source_dir, + source_inventory(source_dir, include_tests=include_tests, path_filter=path_filter), ) jobs = [] for (frontend_id, package), roots in registry.partition_by_package(packages).items(): @@ -862,6 +894,7 @@ def run_project_parallel( max_workers: Optional[int] = None, max_files_per_package: Optional[int] = None, workspace_root: Optional[str] = None, + path_filter: Optional["PathFilter"] = None, ) -> Tuple[CodeGraph, List[FrontendSnapshot], int]: """Compile each (frontend, package) unit in its own process, then compose. @@ -888,11 +921,13 @@ def run_project_parallel( output_root = os.path.abspath(output_root) registry = registry or default_registry(workspace_root) packages = detect_packages( - source_dir, source_inventory(source_dir, include_tests=include_tests), + source_dir, + source_inventory(source_dir, include_tests=include_tests, path_filter=path_filter), ) packages = split_large_packages(source_dir, packages, max_files_per_package) jobs = package_jobs(source_dir, output_root, registry, - include_tests=include_tests, packages=packages) + include_tests=include_tests, packages=packages, + path_filter=path_filter) if not jobs: supported = sorted({ extension for item in registry.frontends for extension in item.extensions diff --git a/lachesis/test_config.py b/lachesis/test_config.py new file mode 100644 index 00000000..6afb3058 --- /dev/null +++ b/lachesis/test_config.py @@ -0,0 +1,163 @@ +"""Project configuration (``lachesis.yml``): classifier, globs, precedence, env. + +The classifier is the default build- and export-time filter, so its two failure +modes both matter: dropping a product module whose *name* merely contains a keyword +(``testing.py``), and keeping scaffolding it should drop. The glob translator, the +exclude/include precedence, and the runtime-env passthrough are pinned here too. The +YAML-loading tests are skipped when PyYAML is absent — that is a genuine environment +gap, not a defect, and the lazy-import contract is exactly that the core runs without +it. +""" +import os +import tempfile +import unittest +from pathlib import Path + +import pytest + +from lachesis import config + + +class ClassifierTests(unittest.TestCase): + def test_keeps_product_modules_that_contain_a_keyword(self): + for path in ("src/flask/testing.py", "src/flask/templating.py", + "pkg/documentation.py", "a/b/specs_helper.py"): + self.assertFalse(config.is_nonproduct(path), path) + + def test_drops_scaffolding_by_segment_and_basename(self): + for path in ("tests/test_cli.py", "examples/tutorial/app.py", "docs/conf.py", + "benchmarks/bench_x.py", "fixtures/data.py", "vendor/lib.py", + "third_party/x.py", "node_modules/y.js", "conftest.py", + "src/foo_test.py", "a/thing.spec.ts", "a/thing.test.js"): + self.assertTrue(config.is_nonproduct(path), path) + + def test_drops_vendored_generated_and_build_config(self): + # Dependencies, generated output, and build configs are not product source; + # a graph over an application has no business modelling them. Language-agnostic, + # so one rule covers Python/TS/JS/C at once. + for path in ("dist/bundle.js", "scripts/check-dist-rules.py", + "lib/parser.min.js", "assets/app.min.css", + "lib/stringify.d.ts", "types/index.d.ts", + "rollup.config.js", "webpack.config.ts", "vite.config.mjs", + "jest.config.cjs"): + self.assertTrue(config.is_nonproduct(path), path) + + def test_generated_patterns_do_not_over_match_product(self): + # A normal ``.ts`` module, a bare ``config.js`` (no ``.config.js`` + # shape), and a product module under ``distributed/`` must all survive. + for path in ("src/reader.ts", "lib/config.js", "src/distributed/queue.py", + "src/scripting/engine.py"): + self.assertFalse(config.is_nonproduct(path), path) + + +class GlobTests(unittest.TestCase): + def test_bare_segment_matches_anywhere(self): + pat = config._glob_to_regex("tests") + self.assertTrue(pat.search("a/tests/b.py")) + self.assertTrue(pat.search("tests/b.py")) + self.assertFalse(pat.search("a/testsuite/b.py")) + + def test_doublestar_and_single_star(self): + self.assertTrue(config._glob_to_regex("src/**/gen_*.py").search("src/a/b/gen_x.py")) + self.assertFalse(config._glob_to_regex("src/*.py").search("src/a/b.py")) + + +class PathFilterTests(unittest.TestCase): + def test_default_excludes_nonproduct(self): + pf = config.PathFilter() + self.assertTrue(pf.excluded("tests/test_a.py")) + self.assertFalse(pf.excluded("src/app.py")) + + def test_explicit_empty_exclude_keeps_everything(self): + # `exclude: []` clears the default; nothing is excluded. + pf = config.parse({"build": {"exclude": []}}).build.paths + self.assertFalse(pf.excluded("tests/test_a.py")) + + def test_explicit_exclude_replaces_default(self): + pf = config.parse({"build": {"exclude": ["examples"]}}).build.paths + self.assertTrue(pf.excluded("examples/x.py")) + # Tests are no longer dropped: the explicit list is the whole policy now. + self.assertFalse(pf.excluded("tests/test_a.py")) + + def test_include_allowlist_wins(self): + pf = config.parse({"build": {"include": ["tests/keep_me.py"]}}).build.paths + self.assertFalse(pf.excluded("tests/keep_me.py")) + self.assertTrue(pf.excluded("tests/other.py")) + + +class ParseTests(unittest.TestCase): + def test_unknown_section_warns_not_fatal(self): + cfg = config.parse({"nonsense": 1, "build": {"max_files": 10}}) + self.assertEqual(cfg.build.max_files, 10) + self.assertTrue(any("nonsense" in w for w in cfg.warnings)) + + def test_bad_int_is_dropped_with_a_warning(self): + cfg = config.parse({"build": {"max_nodes": "lots"}}) + self.assertIsNone(cfg.build.max_nodes) + self.assertTrue(any("max_nodes" in w for w in cfg.warnings)) + + def test_export_paths_default_to_build_paths(self): + cfg = config.parse({"build": {"exclude": ["examples"]}}) + self.assertTrue(cfg.export.paths.excluded("examples/x.py")) + + def test_unknown_runtime_var_warns_but_applies(self): + cfg = config.parse({"runtime": {"LACHESIS_NOT_REAL": "1"}}) + self.assertIn("LACHESIS_NOT_REAL", cfg.runtime) + self.assertTrue(any("LACHESIS_NOT_REAL" in w for w in cfg.warnings)) + + +class RuntimeEnvTests(unittest.TestCase): + def test_apply_sets_env_but_env_wins(self): + cfg = config.parse({ + "runtime": {"LACHESIS_MEMORY_BUDGET_MB": 2048}, + "atropos": {"root": "/opt/atropos", "timings": True}, + }) + saved = {k: os.environ.get(k) for k in + ("LACHESIS_MEMORY_BUDGET_MB", "ATROPOS_ROOT", "LACHESIS_ATROPOS_TIMINGS")} + try: + for k in saved: + os.environ.pop(k, None) + config.apply_runtime_env(cfg) + self.assertEqual(os.environ["LACHESIS_MEMORY_BUDGET_MB"], "2048") + self.assertEqual(os.environ["ATROPOS_ROOT"], "/opt/atropos") + self.assertEqual(os.environ["LACHESIS_ATROPOS_TIMINGS"], "1") + # An inherited env var wins over the file (setdefault). + os.environ["LACHESIS_MEMORY_BUDGET_MB"] = "9999" + config.apply_runtime_env(cfg) + self.assertEqual(os.environ["LACHESIS_MEMORY_BUDGET_MB"], "9999") + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +class LoadTests(unittest.TestCase): + def test_no_file_is_all_defaults_still_excluding_nonproduct(self): + with tempfile.TemporaryDirectory() as d: + cfg = config.load(start=d) + self.assertIsNone(cfg.source) + self.assertTrue(cfg.build.paths.excluded("tests/test_a.py")) + + def test_finds_file_walking_up(self): + pytest.importorskip("yaml") + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "lachesis.yml").write_text("build:\n max_files: 7\n", "utf-8") + sub = root / "a" / "b" + sub.mkdir(parents=True) + cfg = config.load(start=str(sub)) + self.assertEqual(cfg.build.max_files, 7) + # find_config resolves symlinks (e.g. macOS /var -> /private/var), so + # compare resolved paths rather than the raw tempdir string. + self.assertEqual(Path(cfg.source).resolve(), + (root / "lachesis.yml").resolve()) + + def test_explicit_missing_config_is_an_error(self): + with self.assertRaises(config.ConfigError): + config.load(explicit="/no/such/lachesis.yml") + + +if __name__ == "__main__": + unittest.main() diff --git a/lachesis/test_pipeline_inventory.py b/lachesis/test_pipeline_inventory.py index 5adc590e..bd30c3ad 100644 --- a/lachesis/test_pipeline_inventory.py +++ b/lachesis/test_pipeline_inventory.py @@ -76,6 +76,49 @@ def test_explicit_file_survives_the_test_path_heuristic(self): self.assertIn(str(test_file.resolve()), {os.path.realpath(p) for p in kept}) + def _tree(self, root: Path) -> None: + (root / "src" / "pkg").mkdir(parents=True) + (root / "tests").mkdir() + (root / "examples").mkdir() + (root / "docs").mkdir() + (root / "src" / "pkg" / "app.py").write_text("def a():\n return 1\n", "utf-8") + # A product module whose name merely contains a keyword must survive. + (root / "src" / "pkg" / "testing.py").write_text("def t():\n return 1\n", "utf-8") + (root / "tests" / "test_app.py").write_text("def test():\n pass\n", "utf-8") + (root / "examples" / "demo.py").write_text("def d():\n return 1\n", "utf-8") + (root / "docs" / "conf.py").write_text("x = 1\n", "utf-8") + (root / "conftest.py").write_text("import pytest\n", "utf-8") + + def test_path_filter_drops_nonproduct_by_default(self): + from lachesis.config import Config + with tempfile.TemporaryDirectory() as project: + root = Path(project) + self._tree(root) + kept = {os.path.relpath(p, root) + for p in source_inventory(str(root), path_filter=Config().build.paths)} + self.assertEqual(kept, {os.path.join("src", "pkg", "app.py"), + os.path.join("src", "pkg", "testing.py")}) + + def test_no_path_filter_keeps_everything(self): + with tempfile.TemporaryDirectory() as project: + root = Path(project) + self._tree(root) + kept = {os.path.relpath(p, root) for p in source_inventory(str(root))} + self.assertIn("conftest.py", kept) + self.assertIn(os.path.join("tests", "test_app.py"), kept) + + def test_content_hash_tracks_the_filtered_set(self): + # The cache-validity key must describe exactly the file set the build sees, or a + # filtered build could hit a stale cache written by an unfiltered one. + from lachesis.config import Config + from lachesis.pipeline import source_content_hash + with tempfile.TemporaryDirectory() as project: + root = Path(project) + self._tree(root) + unfiltered = source_content_hash(str(root)) + filtered = source_content_hash(str(root), path_filter=Config().build.paths) + self.assertNotEqual(unfiltered, filtered) + if __name__ == "__main__": unittest.main() diff --git a/native/lifetime_kernel/Cargo.toml b/native/lifetime_kernel/Cargo.toml index 53cac828..0f217dec 100644 --- a/native/lifetime_kernel/Cargo.toml +++ b/native/lifetime_kernel/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lachesis-lifetime-kernel" -version = "0.5.1" +version = "0.5.2" edition = "2021" [lib] diff --git a/pyproject.toml b/pyproject.toml index c705aff8..21d1ddaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lachesis-cpg" -version = "0.5.1" +version = "0.5.2" description = "A compiler-precise code property graph with an embedded columnar store and a navigation layer for security reasoning over source code." readme = "README.md" requires-python = ">=3.10" @@ -40,10 +40,14 @@ classifiers = [ dependencies = ["kuzu>=0.11,<0.12", "pyarrow>=17,<26", "protobuf>=6.30,<7"] [project.optional-dependencies] -dev = ["pytest>=8"] +dev = ["pytest>=8", "pyyaml>=6"] # Runtime only. Model weights are fetched separately by `lachesis concept-model # download`; neither FastEmbed nor the model is part of the core wheel. concept-search = ["fastembed>=0.8,<0.9"] +# Project configuration (`lachesis.yml`). PyYAML is imported lazily and only when a +# config file is actually present, so the stdlib-only core is unaffected for anyone +# who never writes one; a present-but-unparseable file is a hard, explained error. +config = ["pyyaml>=6"] [project.scripts] # One entrypoint. The reader is a single `lachesis` with subcommands; every pass diff --git a/server.json b/server.json index fffd7c71..3853b85e 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.UnboundCompute/lachesis", "title": "Lachesis", "description": "Compiler-precise code property graph for C, Python, and TypeScript, navigable over MCP.", - "version": "0.5.1", + "version": "0.5.2", "repository": { "url": "https://github.com/UnboundCompute/lachesis", "source": "github" @@ -13,7 +13,7 @@ "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "lachesis-cpg", - "version": "0.5.1", + "version": "0.5.2", "runtimeHint": "uvx", "runtimeArguments": [ { "type": "named", "name": "--from", "value": "lachesis-cpg" },