From 83f27deed67c23146e769657f4f0485dd8653a56 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 01:31:07 +0530 Subject: [PATCH 01/19] Export architecture concepts in explorer bundles --- lachesis/nav/bundle.py | 46 +++++++++++++++++++++++++++++++++++-- lachesis/nav/test_bundle.py | 11 +++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index c0e4afb0..7844ccb1 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -567,7 +567,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, never raises: a graph the comprehension layer cannot walk simply reads as a bare graph rather than failing the whole export. """ - empty = {"entrypoints": [], "requests": [], "files": [], "modules": []} + empty = {"entrypoints": [], "requests": [], "files": [], "modules": [], "concepts": []} try: from lachesis.planner.entrypoints import EntryPoints, _anchor_strength ctx = M.ctx() @@ -660,10 +660,32 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: files = [] + concepts: list[dict] = [] + try: + architecture = comp.architecture_map(max_communities=8, max_files_per_community=20) + for index, community in enumerate(architecture.get("communities") or []): + paths = [str(path) for path in community.get("files") or [] if path] + if not paths: + continue + first = paths[0] + directory = first.rsplit("/", 1)[0] if "/" in first else first + if directory.startswith("src/"): + directory = directory[4:] + label = directory.replace("/", " · ") or first + concepts.append({ + "id": f"concept.{_slug(community.get('id') or index)}", + "label": label, + "description": f"Connected code area spanning {len(paths)} file(s).", + "file_paths": paths, + }) + except Exception: + concepts = [] + # Modules are not built here: they must partition the *final* included node # pool (one unambiguous module per node, keyed by that node's file), which is # only settled after candidate/capsule/entry nodes are all in and relativized. - return {"entrypoints": entrypoints, "requests": requests, "files": files} + return {"entrypoints": entrypoints, "requests": requests, "files": files, + "concepts": concepts} # ------------------------------------------------------- source / node enrichment @@ -926,6 +948,24 @@ def _partition_modules(nodes: list[dict], entrypoints: list[dict]) -> list[dict] return modules +def _project_concepts(raw_concepts: list[dict], nodes: list[dict]) -> list[dict]: + """Keep architecture concepts honest to the final included node pool.""" + out: list[dict] = [] + for concept in raw_concepts or []: + paths = {str(path) for path in concept.get("file_paths") or [] if path} + node_ids = [node["id"] for node in nodes + if isinstance(node.get("file"), str) and node.get("file") in paths] + if not node_ids: + continue + out.append({ + "id": str(concept.get("id") or f"concept.{len(out)}"), + "label": str(concept.get("label") or "Code area"), + "description": str(concept.get("description") or "Connected code area."), + "node_ids": node_ids[:20], + }) + return out + + def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[str], lang: Optional[str], indexed_nodes: int, source_url_template: Optional[str] = None, @@ -976,6 +1016,7 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s if e.get("node_id") in node_ids] requests = _finalize_requests(comp.get("requests") or [], node_map, edges_by_pair) modules = _partition_modules(nodes, entrypoints) + concepts = _project_concepts(comp.get("concepts") or [], nodes) coverage = { "scope": "repository-projection", @@ -1011,6 +1052,7 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s "edges": edges, "files": comp.get("files") or [], "modules": modules, + "concepts": concepts, "entrypoints": entrypoints, "coverage": coverage, }, diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 2fad9c2a..1b7cf9eb 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -311,6 +311,8 @@ def test_full_projection_shapes_graph_and_paths(self): "hops": [{"node_id": "n.a", "caption": "receives"}, {"node_id": "n.b", "caption": "dispatches"}]}], "files": [{"id": "f1", "path": "src/flask/app.py"}], + "concepts": [{"id": "concept.flask", "label": "flask", "description": "d", + "file_paths": ["src/flask/app.py"]}], }) self.assertEqual(result["meta"]["indexed_nodes"], 500) self.assertEqual(result["graph"]["coverage"]["included_nodes"], @@ -333,6 +335,15 @@ def test_full_projection_shapes_graph_and_paths(self): self.assertEqual(by_path["src/flask/app.py"]["anchor_node_id"], "n.a") seen = [nid for m in result["graph"]["modules"] for nid in m["node_ids"]] self.assertEqual(len(seen), len(set(seen))) + self.assertEqual(["n.a", "n.b"], result["graph"]["concepts"][0]["node_ids"]) + + def test_concept_without_included_nodes_is_dropped(self): + result = self._bundle_with({ + "entrypoints": [], "requests": [], + "concepts": [{"id": "concept.missing", "label": "missing", + "file_paths": ["src/other/missing.py"]}], + }) + self.assertEqual([], result["graph"]["concepts"]) def test_request_with_unsourced_hop_is_dropped(self): # n.ghost has no source; the guided path must not be emitted. From d34222b87ba0ce51253643f86a8ad86e7dac4123 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 01:31:43 +0530 Subject: [PATCH 02/19] Export architecture concepts in Explorer bundles Merged after targeted bundle tests passed in a protobuf-compatible environment. --- lachesis/nav/bundle.py | 46 +++++++++++++++++++++++++++++++++++-- lachesis/nav/test_bundle.py | 11 +++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index c0e4afb0..7844ccb1 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -567,7 +567,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, never raises: a graph the comprehension layer cannot walk simply reads as a bare graph rather than failing the whole export. """ - empty = {"entrypoints": [], "requests": [], "files": [], "modules": []} + empty = {"entrypoints": [], "requests": [], "files": [], "modules": [], "concepts": []} try: from lachesis.planner.entrypoints import EntryPoints, _anchor_strength ctx = M.ctx() @@ -660,10 +660,32 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: files = [] + concepts: list[dict] = [] + try: + architecture = comp.architecture_map(max_communities=8, max_files_per_community=20) + for index, community in enumerate(architecture.get("communities") or []): + paths = [str(path) for path in community.get("files") or [] if path] + if not paths: + continue + first = paths[0] + directory = first.rsplit("/", 1)[0] if "/" in first else first + if directory.startswith("src/"): + directory = directory[4:] + label = directory.replace("/", " · ") or first + concepts.append({ + "id": f"concept.{_slug(community.get('id') or index)}", + "label": label, + "description": f"Connected code area spanning {len(paths)} file(s).", + "file_paths": paths, + }) + except Exception: + concepts = [] + # Modules are not built here: they must partition the *final* included node # pool (one unambiguous module per node, keyed by that node's file), which is # only settled after candidate/capsule/entry nodes are all in and relativized. - return {"entrypoints": entrypoints, "requests": requests, "files": files} + return {"entrypoints": entrypoints, "requests": requests, "files": files, + "concepts": concepts} # ------------------------------------------------------- source / node enrichment @@ -926,6 +948,24 @@ def _partition_modules(nodes: list[dict], entrypoints: list[dict]) -> list[dict] return modules +def _project_concepts(raw_concepts: list[dict], nodes: list[dict]) -> list[dict]: + """Keep architecture concepts honest to the final included node pool.""" + out: list[dict] = [] + for concept in raw_concepts or []: + paths = {str(path) for path in concept.get("file_paths") or [] if path} + node_ids = [node["id"] for node in nodes + if isinstance(node.get("file"), str) and node.get("file") in paths] + if not node_ids: + continue + out.append({ + "id": str(concept.get("id") or f"concept.{len(out)}"), + "label": str(concept.get("label") or "Code area"), + "description": str(concept.get("description") or "Connected code area."), + "node_ids": node_ids[:20], + }) + return out + + def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[str], lang: Optional[str], indexed_nodes: int, source_url_template: Optional[str] = None, @@ -976,6 +1016,7 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s if e.get("node_id") in node_ids] requests = _finalize_requests(comp.get("requests") or [], node_map, edges_by_pair) modules = _partition_modules(nodes, entrypoints) + concepts = _project_concepts(comp.get("concepts") or [], nodes) coverage = { "scope": "repository-projection", @@ -1011,6 +1052,7 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s "edges": edges, "files": comp.get("files") or [], "modules": modules, + "concepts": concepts, "entrypoints": entrypoints, "coverage": coverage, }, diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 2fad9c2a..1b7cf9eb 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -311,6 +311,8 @@ def test_full_projection_shapes_graph_and_paths(self): "hops": [{"node_id": "n.a", "caption": "receives"}, {"node_id": "n.b", "caption": "dispatches"}]}], "files": [{"id": "f1", "path": "src/flask/app.py"}], + "concepts": [{"id": "concept.flask", "label": "flask", "description": "d", + "file_paths": ["src/flask/app.py"]}], }) self.assertEqual(result["meta"]["indexed_nodes"], 500) self.assertEqual(result["graph"]["coverage"]["included_nodes"], @@ -333,6 +335,15 @@ def test_full_projection_shapes_graph_and_paths(self): self.assertEqual(by_path["src/flask/app.py"]["anchor_node_id"], "n.a") seen = [nid for m in result["graph"]["modules"] for nid in m["node_ids"]] self.assertEqual(len(seen), len(set(seen))) + self.assertEqual(["n.a", "n.b"], result["graph"]["concepts"][0]["node_ids"]) + + def test_concept_without_included_nodes_is_dropped(self): + result = self._bundle_with({ + "entrypoints": [], "requests": [], + "concepts": [{"id": "concept.missing", "label": "missing", + "file_paths": ["src/other/missing.py"]}], + }) + self.assertEqual([], result["graph"]["concepts"]) def test_request_with_unsourced_hop_is_dropped(self): # n.ghost has no source; the guided path must not be emitted. From 0dc85883c50c72685c34c788b92be7b5661dde76 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 02:07:08 +0530 Subject: [PATCH 03/19] Export bounded core spine for code exploration --- lachesis/nav/bundle.py | 40 +++++++++++++++++++++++++++++++++++-- lachesis/nav/test_bundle.py | 2 ++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 7844ccb1..9e10ad27 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -567,7 +567,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, never raises: a graph the comprehension layer cannot walk simply reads as a bare graph rather than failing the whole export. """ - empty = {"entrypoints": [], "requests": [], "files": [], "modules": [], "concepts": []} + empty = {"entrypoints": [], "requests": [], "files": [], "modules": [], "concepts": [], "core": []} try: from lachesis.planner.entrypoints import EntryPoints, _anchor_strength ctx = M.ctx() @@ -681,11 +681,44 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: concepts = [] + # Add a small, source-backed architecture spine to the shared node pool. This + # is intentionally independent from security candidates: a newcomer needs the + # central control path even when no finding happens to touch it. Hubs ranks real + # call-graph declarations, and the walk below adds only real CALLS edges. + core: list[dict] = [] + try: + from lachesis.nav.hubs import Hubs + hubs = Hubs(gl, resolved_only=True).top(8) + core_ids: set[str] = set() + for hub in hubs: + hub_id = hub.get("node_id") + node = gl.nodes.get(hub_id) + if node is None: + continue + chain = _call_chain(index, gl, hub_id, 4) + for nid in chain: + if len(core_ids) >= 32: + break + cnode = gl.nodes.get(nid) + if cnode is None: + continue + asm.add_node(_norm_node(gl, cnode), default_kind="function") + core_ids.add(nid) + for a, b in zip(chain, chain[1:]): + asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, set(asm.nodes)) + if hub_id in core_ids: + file, line, _ = gl.loc(node) + core.append({"node_id": hub_id, "label": gl.label(node), + "file": file, "line": line, + "degree": int(hub.get("degree") or 0)}) + except Exception: + core = [] + # Modules are not built here: they must partition the *final* included node # pool (one unambiguous module per node, keyed by that node's file), which is # only settled after candidate/capsule/entry nodes are all in and relativized. return {"entrypoints": entrypoints, "requests": requests, "files": files, - "concepts": concepts} + "concepts": concepts, "core": core} # ------------------------------------------------------- source / node enrichment @@ -1017,6 +1050,8 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s requests = _finalize_requests(comp.get("requests") or [], node_map, edges_by_pair) modules = _partition_modules(nodes, entrypoints) concepts = _project_concepts(comp.get("concepts") or [], nodes) + core = [item for item in (comp.get("core") or []) + if item.get("node_id") in node_ids] coverage = { "scope": "repository-projection", @@ -1054,6 +1089,7 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s "modules": modules, "concepts": concepts, "entrypoints": entrypoints, + "core": core, "coverage": coverage, }, "paths": {"requests": requests, "values": values}, diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 1b7cf9eb..b0b36e42 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -313,6 +313,7 @@ def test_full_projection_shapes_graph_and_paths(self): "files": [{"id": "f1", "path": "src/flask/app.py"}], "concepts": [{"id": "concept.flask", "label": "flask", "description": "d", "file_paths": ["src/flask/app.py"]}], + "core": [{"node_id": "n.a", "label": "wsgi_app", "degree": 4}], }) self.assertEqual(result["meta"]["indexed_nodes"], 500) self.assertEqual(result["graph"]["coverage"]["included_nodes"], @@ -336,6 +337,7 @@ def test_full_projection_shapes_graph_and_paths(self): seen = [nid for m in result["graph"]["modules"] for nid in m["node_ids"]] self.assertEqual(len(seen), len(set(seen))) self.assertEqual(["n.a", "n.b"], result["graph"]["concepts"][0]["node_ids"]) + self.assertEqual(["n.a"], [item["node_id"] for item in result["graph"]["core"]]) def test_concept_without_included_nodes_is_dropped(self): result = self._bundle_with({ From e972a432426bc5da275d13dfa88d20eb3e20204b Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 02:08:58 +0530 Subject: [PATCH 04/19] Keep core spine source-backed --- lachesis/nav/bundle.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 9e10ad27..81cea1d8 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -695,21 +695,36 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, node = gl.nodes.get(hub_id) if node is None: continue + hub_file, hub_line, _ = gl.loc(node) + if (not hub_file or not isinstance(hub_line, int) or hub_line <= 0 + or re.search(r"(?:^|/)(?:tests?|examples?|fixtures?|benchmarks?|docs?)(?:/|$)", + str(hub_file), re.IGNORECASE)): + continue chain = _call_chain(index, gl, hub_id, 4) - for nid in chain: - if len(core_ids) >= 32: + chain_nodes = [gl.nodes.get(nid) for nid in chain] + # Do not remove a middle hop and then connect its neighbors: that + # would turn a real multi-hop walk into an invented relationship. + if any( + cnode is None + or not gl.loc(cnode)[0] + or not isinstance(gl.loc(cnode)[1], int) + or gl.loc(cnode)[1] <= 0 + or re.search(r"(?:^|/)(?:tests?|examples?|fixtures?|benchmarks?|docs?)(?:/|$)", + str(gl.loc(cnode)[0]), re.IGNORECASE) + for cnode in chain_nodes + ): + chain = [hub_id] + chain_nodes = [node] + for nid, cnode in zip(chain, chain_nodes): + if len(core_ids) >= 32 or cnode is None: break - cnode = gl.nodes.get(nid) - if cnode is None: - continue asm.add_node(_norm_node(gl, cnode), default_kind="function") core_ids.add(nid) for a, b in zip(chain, chain[1:]): asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, set(asm.nodes)) if hub_id in core_ids: - file, line, _ = gl.loc(node) core.append({"node_id": hub_id, "label": gl.label(node), - "file": file, "line": line, + "file": hub_file, "line": hub_line, "degree": int(hub.get("degree") or 0)}) except Exception: core = [] From 1b2c2f63dcfa5dadd930bbed565270b642b61bb0 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 02:12:48 +0530 Subject: [PATCH 05/19] Export recorded symbol documentation --- lachesis/nav/bundle.py | 8 ++++++++ lachesis/nav/test_bundle.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 81cea1d8..44bda426 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -858,6 +858,14 @@ def _enrich_graph_nodes(nodes: list[dict], gl) -> None: module = _dotted_module(node.get("file")) if module: node["qualified_name"] = f"{module}.{node.get('label')}" + for key in ("documentation", "docstring", "comment"): + try: + documentation = gl.prop(twin, key) + except Exception: + documentation = None + if isinstance(documentation, str) and documentation.strip(): + node["documentation"] = documentation.strip() + break try: excerpt = gl.source_excerpt(twin) except Exception: diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index b0b36e42..811edb83 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -400,6 +400,30 @@ def test_has_source_requires_file_line_and_text(self): self.assertTrue(bundle._has_source( {"file": "a.py", "line": 3, "source_window": {"lines": ["x"]}})) + def test_enrich_graph_nodes_preserves_recorded_documentation(self): + class _Graph: + nodes = {"n": {"id": "n", "label": "serve", + "properties": {"file": "src/app.py", "start_line": 4, + "end_line": 8, "documentation": "Serve a request."}}} + + def loc(self, node): + props = node["properties"] + return props["file"], props["start_line"], props["end_line"] + + def prop(self, node, key, default=None): + return node.get("properties", {}).get(key, default) + + def source_excerpt(self, node): + return "serve()" + + def _read_file(self, _path): + return "def serve():\n pass\n" + + nodes = [{"id": "n", "file": "src/app.py", "line": 4, + "label": "serve", "snippet": "serve"}] + bundle._enrich_graph_nodes(nodes, _Graph()) + self.assertEqual("Serve a request.", nodes[0]["documentation"]) + def test_count_source_lines_sums_physical_lines_dedups_and_falls_back(self): # a.py: 3 physical lines read off disk; b.py: unreadable, falls back to # its file-node span end (2); the duplicate a.py node is counted once. From 394cd09beaa346ab30a3d6a2da715d6d18f5b401 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 02:21:50 +0530 Subject: [PATCH 06/19] Consume curated tour manifests in exports --- lachesis/cli/main.py | 16 +++++++++++ lachesis/nav/bundle.py | 56 +++++++++++++++++++++++++++++++++++-- lachesis/nav/test_bundle.py | 39 ++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/lachesis/cli/main.py b/lachesis/cli/main.py index d01bd05d..a7d95265 100644 --- a/lachesis/cli/main.py +++ b/lachesis/cli/main.py @@ -389,6 +389,19 @@ def command_trace(args: argparse.Namespace) -> int: repo, commit = _repo_meta(source) progress.phase("exporting bundle") + curated_tour = None + if args.curated_tour: + try: + config = json.loads(Path(args.curated_tour).expanduser().read_text(encoding="utf-8")) + curated_tour = config.get("meta", {}).get("curated_tour", config.get("curated_tour")) + if not isinstance(curated_tour, dict): + raise ValueError("curated tour file must contain meta.curated_tour") + # Public repository files are not an authenticated ownership boundary. + curated_tour = dict(curated_tour) + curated_tour.pop("maintainer", None) + except (OSError, UnicodeError, json.JSONDecodeError, AttributeError, ValueError) as error: + _stderr(f"lachesis trace: curated tour: {error}") + return EXIT_USAGE try: bundle = bundle_mod.build_bundle( str(graph_path), @@ -401,6 +414,7 @@ def command_trace(args: argparse.Namespace) -> int: schema_version=args.schema_version, source_url_template=args.source_url_template, description=args.description, + curated_tour=curated_tour, ) except Exception as error: # noqa: BLE001 - CLI turns export errors into one line _stderr(f"lachesis trace: {error}") @@ -971,6 +985,8 @@ def build_parser() -> argparse.ArgumentParser: help="explicit HTTP(S) source template using {file}, {line}, {end_line}, {revision}") trace.add_argument("--description", metavar="TEXT", help="one-line projection description recorded in bundle meta (2.0)") + trace.add_argument("--curated-tour", metavar="JSON", + help="read a meta.curated_tour fragment and validate it against the exported paths") trace.add_argument("--per-family", type=_positive_int, default=6, metavar="N", help="max leads to draw from each sink family (default: 6)") trace.add_argument("--max-flows", type=_positive_int, default=40, metavar="N", diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 44bda426..43ae8bb2 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -1022,11 +1022,58 @@ def _project_concepts(raw_concepts: list[dict], nodes: list[dict]) -> list[dict] return out +def _project_curated_tour(raw: Optional[dict], values: list[dict], requests: list[dict]) -> Optional[dict]: + """Keep only tour steps that resolve in this exact exported projection. + + Tour files are user-authored convenience metadata, not evidence. A changed + repository can make an old flow or anchor disappear, so stale steps are + omitted instead of making the entire export fail. Maintainer identity is + deliberately not accepted from this unauthenticated file path. + """ + if not isinstance(raw, dict): + return None + title = str(raw.get("title") or "Start here").strip() + tour_id = str(raw.get("id") or "tour.start-here").strip() + if not title or not tour_id: + return None + paths = {str(path.get("id")): path for path in [*values, *requests] + if isinstance(path, dict) and path.get("id")} + steps: list[dict] = [] + for item in raw.get("steps") or []: + if not isinstance(item, dict): + continue + flow_id = str(item.get("flow_id") or item.get("flowId") or "").strip() + path = paths.get(flow_id) + if not path: + continue + raw_steps = path.get("steps") if isinstance(path.get("steps"), list) else path.get("hops") + node_ids = {str(step.get("node_id")) for step in raw_steps or [] + if isinstance(step, dict) and step.get("node_id")} + node_id = item.get("node_id") or item.get("nodeId") + if node_id is not None and str(node_id) not in node_ids: + continue + step = {"flow_id": flow_id} + if node_id is not None: + step["node_id"] = str(node_id) + for key in ("label", "note"): + if item.get(key) is not None and str(item[key]).strip(): + step[key] = str(item[key]).strip() + steps.append(step) + if not steps: + return None + result = {"id": tour_id, "title": title, "steps": steps} + description = str(raw.get("description") or "").strip() + if description: + result["description"] = description[:500] + return result + + def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[str], lang: Optional[str], indexed_nodes: int, source_url_template: Optional[str] = None, comprehension: Optional[dict] = None, - description: Optional[str] = None) -> dict: + description: Optional[str] = None, + curated_tour: Optional[dict] = None) -> dict: """Adapt the assembled evidence into Explorer's graph-first 2.0 contract. The security envelope remains available under ``security.findings``. The @@ -1075,6 +1122,7 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s concepts = _project_concepts(comp.get("concepts") or [], nodes) core = [item for item in (comp.get("core") or []) if item.get("node_id") in node_ids] + tour = _project_curated_tour(curated_tour, values, requests) coverage = { "scope": "repository-projection", @@ -1118,6 +1166,8 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s "paths": {"requests": requests, "values": values}, "security": {"findings": findings}, } + if tour is not None: + v2["meta"]["curated_tour"] = tour _validate_graph_first(v2) return v2 @@ -1193,6 +1243,7 @@ def build_bundle(graph_path: str, *, repo: Optional[str] = None, schema_version: str = "1.0", source_url_template: Optional[str] = None, description: Optional[str] = None, + curated_tour: Optional[dict] = None, max_entrypoints: int = 40, chain_depth: int = 6, max_files: int = 2000) -> dict: """Build an explorer bundle (schema 1.0) from a built+enriched graph.""" @@ -1299,7 +1350,8 @@ def build_bundle(graph_path: str, *, repo: Optional[str] = None, commit=commit or prov.get("commit_sha"), lang=lang, indexed_nodes=int(load.get("nodes") or 0), source_url_template=source_url_template, - comprehension=projection, description=description) + comprehension=projection, description=description, + curated_tour=curated_tour) if schema_version != "1.0": raise ValueError(f"unsupported Explorer schema version: {schema_version}") return bundle diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 811edb83..931bfb88 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -301,6 +301,23 @@ def _bundle_with(self, comprehension): legacy, repo="pallets/flask", commit="abc", lang="python", indexed_nodes=500, comprehension=comprehension) + def _bundle_with_tour(self, comprehension, curated_tour): + legacy = { + "meta": {"repo": "pallets/flask", "lang": "python", "commit": "abc", "loc": 100}, + "graph": { + "nodes": [self._sourced("n.a", "src/flask/app.py", 10, "wsgi_app"), + self._sourced("n.b", "src/flask/app.py", 20, "dispatch_request")], + "edges": [{"source": "n.a", "target": "n.b", "kind": "CALLS"}], + }, + "findings": [{"finding_id": "a" * 64, "display_name": "x", "result_summary": "y", + "analysis": {"confidence": "high", "limitations": []}, + "witness": {"steps": [{"node_id": "n.a", "role": "origin"}, + {"node_id": "n.b", "role": "sink"}]}}], + } + return bundle._graph_first_bundle( + legacy, repo="pallets/flask", commit="abc", lang="python", indexed_nodes=500, + comprehension=comprehension, curated_tour=curated_tour) + def test_full_projection_shapes_graph_and_paths(self): result = self._bundle_with({ "entrypoints": [{"id": "entry.wsgi_app", "label": "wsgi_app", @@ -347,6 +364,28 @@ def test_concept_without_included_nodes_is_dropped(self): }) self.assertEqual([], result["graph"]["concepts"]) + def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): + result = self._bundle_with_tour( + {"entrypoints": [], "requests": [{"id": "request.lifecycle", "kind": "call-path", + "description": "d", "entry_node": "n.a", + "hops": [{"node_id": "n.a", "caption": "a"}, + {"node_id": "n.b", "caption": "b"}]}]}, + {"id": "tour.start", "title": "Start here", "description": "Read this first.", + "maintainer": {"name": "Ignored"}, + "steps": [{"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, + {"flow_id": "request.missing"}]}, + ) + self.assertEqual({"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, + result["meta"]["curated_tour"]["steps"][0]) + self.assertNotIn("maintainer", result["meta"]["curated_tour"]) + + def test_curated_tour_is_omitted_when_no_step_resolves(self): + result = self._bundle_with_tour( + {"entrypoints": [], "requests": []}, + {"id": "tour.start", "title": "Start here", "steps": [{"flow_id": "missing"}]}, + ) + self.assertNotIn("curated_tour", result["meta"]) + def test_request_with_unsourced_hop_is_dropped(self): # n.ghost has no source; the guided path must not be emitted. result = self._bundle_with({ From b18ae77658c6236c9ec12feeba416b64868dc2bd Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 02:24:24 +0530 Subject: [PATCH 07/19] Harden curated tour CLI input --- lachesis/cli/main.py | 19 ++++++++++++------- lachesis/cli/test_cli_args.py | 21 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/lachesis/cli/main.py b/lachesis/cli/main.py index a7d95265..a11ae0cf 100644 --- a/lachesis/cli/main.py +++ b/lachesis/cli/main.py @@ -358,6 +358,17 @@ def git(*a: str) -> str | None: return repo, commit +def _load_curated_tour(path: str) -> dict: + """Load an OSS tour fragment without accepting a verified owner claim.""" + config = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) + curated_tour = config.get("meta", {}).get("curated_tour", config.get("curated_tour")) + if not isinstance(curated_tour, dict): + raise ValueError("curated tour file must contain meta.curated_tour") + curated_tour = dict(curated_tour) + curated_tour.pop("maintainer", None) + return curated_tour + + def command_trace(args: argparse.Namespace) -> int: """Build (or reuse) a graph and export a lachesis-explorer bundle.json.""" from lachesis.cli.indexer import (EnvironmentProblem, NoSourceFound, @@ -392,13 +403,7 @@ def command_trace(args: argparse.Namespace) -> int: curated_tour = None if args.curated_tour: try: - config = json.loads(Path(args.curated_tour).expanduser().read_text(encoding="utf-8")) - curated_tour = config.get("meta", {}).get("curated_tour", config.get("curated_tour")) - if not isinstance(curated_tour, dict): - raise ValueError("curated tour file must contain meta.curated_tour") - # Public repository files are not an authenticated ownership boundary. - curated_tour = dict(curated_tour) - curated_tour.pop("maintainer", None) + curated_tour = _load_curated_tour(args.curated_tour) except (OSError, UnicodeError, json.JSONDecodeError, AttributeError, ValueError) as error: _stderr(f"lachesis trace: curated tour: {error}") return EXIT_USAGE diff --git a/lachesis/cli/test_cli_args.py b/lachesis/cli/test_cli_args.py index 2c359229..b95306be 100644 --- a/lachesis/cli/test_cli_args.py +++ b/lachesis/cli/test_cli_args.py @@ -1,7 +1,10 @@ +import json +import tempfile import unittest +from pathlib import Path -from lachesis.cli.main import build_parser +from lachesis.cli.main import _load_curated_tour, build_parser class CliArgumentTests(unittest.TestCase): @@ -28,7 +31,21 @@ def test_scan_rejects_invalid_limits_and_ranks(self): with self.subTest(option=option, value=value): with self.assertRaises(SystemExit) as raised: parser.parse_args(["scan", option, value]) - self.assertEqual(raised.exception.code, 2) + self.assertEqual(raised.exception.code, 2) + + def test_curated_tour_argument_and_loader_strip_unverified_identity(self): + args = build_parser().parse_args(["trace", "--curated-tour", "lachesis-tour.json"]) + self.assertEqual(args.curated_tour, "lachesis-tour.json") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "tour.json" + path.write_text(json.dumps({"meta": {"curated_tour": { + "id": "tour.start", "title": "Start here", + "maintainer": {"name": "Untrusted", "verified": True}, + "steps": [{"flow_id": "request.main"}], + }}}), encoding="utf-8") + loaded = _load_curated_tour(str(path)) + self.assertNotIn("maintainer", loaded) + self.assertEqual("tour.start", loaded["id"]) if __name__ == "__main__": From 110efbdd8766cd5d4bd4f9b045fd54746e8bc3ac Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 03:13:20 +0530 Subject: [PATCH 08/19] preserve curated visibility selection --- lachesis/nav/bundle.py | 6 ++++++ lachesis/nav/test_bundle.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 43ae8bb2..cb749d34 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -1065,6 +1065,12 @@ def _project_curated_tour(raw: Optional[dict], values: list[dict], requests: lis description = str(raw.get("description") or "").strip() if description: result["description"] = description[:500] + selection = raw.get("selection") + if isinstance(selection, dict): + allowed = ("include_tests", "include_examples", "include_generated") + selected = {key: value for key, value in selection.items() if key in allowed and isinstance(value, bool) and value} + if selected: + result["selection"] = selected return result diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 931bfb88..1670e1fb 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -372,12 +372,14 @@ def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): {"node_id": "n.b", "caption": "b"}]}]}, {"id": "tour.start", "title": "Start here", "description": "Read this first.", "maintainer": {"name": "Ignored"}, + "selection": {"include_tests": True, "include_examples": False, "include_generated": True}, "steps": [{"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, {"flow_id": "request.missing"}]}, ) self.assertEqual({"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, result["meta"]["curated_tour"]["steps"][0]) self.assertNotIn("maintainer", result["meta"]["curated_tour"]) + self.assertEqual({"include_tests": True, "include_generated": True}, result["meta"]["curated_tour"]["selection"]) def test_curated_tour_is_omitted_when_no_step_resolves(self): result = self._bundle_with_tour( From 2672a6903c85cbda47a0c2f03e68f26bd57f7957 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 03:17:04 +0530 Subject: [PATCH 09/19] preserve curated project overview --- lachesis/nav/bundle.py | 5 +++++ lachesis/nav/test_bundle.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index cb749d34..257daa8c 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -1065,6 +1065,11 @@ def _project_curated_tour(raw: Optional[dict], values: list[dict], requests: lis description = str(raw.get("description") or "").strip() if description: result["description"] = description[:500] + overview = raw.get("overview") + if isinstance(overview, dict): + overview_description = str(overview.get("description") or "").strip() + if overview_description: + result["overview"] = {"description": overview_description[:1000]} selection = raw.get("selection") if isinstance(selection, dict): allowed = ("include_tests", "include_examples", "include_generated") diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 1670e1fb..82ebdaeb 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -372,6 +372,7 @@ def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): {"node_id": "n.b", "caption": "b"}]}]}, {"id": "tour.start", "title": "Start here", "description": "Read this first.", "maintainer": {"name": "Ignored"}, + "overview": {"description": "Read the request lifecycle first."}, "selection": {"include_tests": True, "include_examples": False, "include_generated": True}, "steps": [{"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, {"flow_id": "request.missing"}]}, @@ -379,6 +380,7 @@ def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): self.assertEqual({"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, result["meta"]["curated_tour"]["steps"][0]) self.assertNotIn("maintainer", result["meta"]["curated_tour"]) + self.assertEqual("Read the request lifecycle first.", result["meta"]["curated_tour"]["overview"]["description"]) self.assertEqual({"include_tests": True, "include_generated": True}, result["meta"]["curated_tour"]["selection"]) def test_curated_tour_is_omitted_when_no_step_resolves(self): From 8211eaff23b488d514410de97e4c91a81ecaede6 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 03:25:00 +0530 Subject: [PATCH 10/19] preserve curated concept map --- lachesis/nav/bundle.py | 20 ++++++++++++++++++-- lachesis/nav/test_bundle.py | 5 +++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 257daa8c..dcf943fb 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -1068,8 +1068,24 @@ def _project_curated_tour(raw: Optional[dict], values: list[dict], requests: lis overview = raw.get("overview") if isinstance(overview, dict): overview_description = str(overview.get("description") or "").strip() - if overview_description: - result["overview"] = {"description": overview_description[:1000]} + overview_result = {"description": overview_description[:1000]} if overview_description else {} + concepts = overview.get("concepts") + if isinstance(concepts, list): + selected_concepts = [] + for item in concepts[:8]: + if not isinstance(item, dict) or not str(item.get("id") or "").strip() or not str(item.get("label") or "").strip(): + continue + concept = {"id": str(item["id"]).strip(), "label": str(item["label"]).strip()} + if str(item.get("description") or "").strip(): + concept["description"] = str(item["description"]).strip()[:300] + related = item.get("related_ids") + if isinstance(related, list) and related: + concept["related_ids"] = [str(value) for value in related if str(value).strip()][:8] + selected_concepts.append(concept) + if selected_concepts: + overview_result["concepts"] = selected_concepts + if overview_result: + result["overview"] = overview_result selection = raw.get("selection") if isinstance(selection, dict): allowed = ("include_tests", "include_examples", "include_generated") diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 82ebdaeb..1220a358 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -366,13 +366,13 @@ def test_concept_without_included_nodes_is_dropped(self): def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): result = self._bundle_with_tour( - {"entrypoints": [], "requests": [{"id": "request.lifecycle", "kind": "call-path", + {"entrypoints": [], "concepts": [{"id": "concept.lifecycle", "label": "Lifecycle", "description": "d", "file_paths": ["src/flask/app.py"]}], "requests": [{"id": "request.lifecycle", "kind": "call-path", "description": "d", "entry_node": "n.a", "hops": [{"node_id": "n.a", "caption": "a"}, {"node_id": "n.b", "caption": "b"}]}]}, {"id": "tour.start", "title": "Start here", "description": "Read this first.", "maintainer": {"name": "Ignored"}, - "overview": {"description": "Read the request lifecycle first."}, + "overview": {"description": "Read the request lifecycle first.", "concepts": [{"id": "concept.lifecycle", "label": "Request lifecycle", "description": "The main request route."}]}, "selection": {"include_tests": True, "include_examples": False, "include_generated": True}, "steps": [{"flow_id": "request.lifecycle", "node_id": "n.a", "label": "Lifecycle"}, {"flow_id": "request.missing"}]}, @@ -381,6 +381,7 @@ def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): result["meta"]["curated_tour"]["steps"][0]) self.assertNotIn("maintainer", result["meta"]["curated_tour"]) self.assertEqual("Read the request lifecycle first.", result["meta"]["curated_tour"]["overview"]["description"]) + self.assertEqual("concept.lifecycle", result["meta"]["curated_tour"]["overview"]["concepts"][0]["id"]) self.assertEqual({"include_tests": True, "include_generated": True}, result["meta"]["curated_tour"]["selection"]) def test_curated_tour_is_omitted_when_no_step_resolves(self): From 92cbd1e1fd2dd46de9cabbc0cf6535feaf7d58f0 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 10:49:18 +0530 Subject: [PATCH 11/19] config: add lachesis.yml and exclude non-product code from builds Introduce a single optional project config file (lachesis.yml) and make the default build ingest product source only -- tests, examples, docs, fixtures, benchmarks and vendored trees no longer land in the graph. A tree is understood by its product code; the scaffolding around it inflates the node count and, for the comprehension export, drowns the architecture ranking and roots request flows in test modules. Mechanism. lachesis/config.py discovers the file by walking up from the analysed tree (or --config), lazily imports PyYAML only when a file is actually present, and resolves a typed Config with build/export/atropos/runtime sections. The non-product decision is a PathFilter over the repo-relative, forward-slashed display path: matching is on path *segments* and basenames, never substrings, so product modules whose name merely contains a keyword -- testing.py, templating.py, documentation.py -- are kept while tests/, examples/, docs/, conftest.py, *_test.py and vendored trees are dropped. include wins over exclude wins over the built-in default; an explicit `exclude: []` restores full coverage, and a `build.include` allow-list keeps a specific file without disabling the default. Wiring. source_inventory grows an optional path_filter applied in the walk; it is threaded through every build variant (run_project, _streaming, _streaming_parallel, _incremental, _parallel) and package_jobs, and -- critically -- through source_content_hash, so a filtered build and its cache-validity key describe the very same file set and a filtered build cannot hit a cache written unfiltered. The low-level default is unchanged (complete coverage for direct library callers); the CLI resolves the filter from config and passes it down, so `lachesis build` and `lachesis analyze` exclude non-product by default while --all-sources (or `exclude: []`) compiles the whole tree. The runtime: block mirrors every LACHESIS_* variable and is applied with os.environ.setdefault, preserving the documented precedence flag > env > file > default. Unknown config keys and malformed fields warn rather than fail, so a file written for a newer reader still loads. Verified end-to-end on a synthetic tree through the real CLI: the default build keeps only src product modules (testing.py included), --all-sources keeps all six files, `exclude: []` includes everything, an explicit exclude list replaces the default, and an include allow-list rescues a single test file. New unit tests in test_config.py (classifier, globs, precedence, runtime env, discovery) and test_pipeline_inventory.py (filter drops non-product, content hash tracks the filtered set); full cli/ and nav/test_bundle.py suites green (76 passed). PyYAML is added as an optional [config] extra, imported lazily; the stdlib-only core is unaffected for anyone who never writes a config file. --- lachesis/cli/analyze.py | 48 ++- lachesis/cli/main.py | 11 + lachesis/config.py | 451 ++++++++++++++++++++++++++++ lachesis/pipeline.py | 55 +++- lachesis/test_config.py | 145 +++++++++ lachesis/test_pipeline_inventory.py | 43 +++ pyproject.toml | 6 +- 7 files changed, 745 insertions(+), 14 deletions(-) create mode 100644 lachesis/config.py create mode 100644 lachesis/test_config.py diff --git a/lachesis/cli/analyze.py b/lachesis/cli/analyze.py index 0df8b8b1..6d1162df 100644 --- a/lachesis/cli/analyze.py +++ b/lachesis/cli/analyze.py @@ -255,6 +255,20 @@ def _run(argv: list[str] | None = None) -> None: "exists to reach is never scoped out. An explicitly named file is always " "kept; a directory is walked with the same ignore rules as source_dir.", ) + parser.add_argument( + "--config", metavar="FILE", default=None, + help="path to a lachesis.yml config file. When omitted, the tree is searched " + "upward from source_dir for lachesis.yml (or .yaml/.lachesis.* variants). " + "The config sets what a build ingests, size caps, and runtime knobs; its " + "built-in default excludes tests, examples, docs, fixtures, benchmarks and " + "vendored trees from the graph.", + ) + parser.add_argument( + "--all-sources", action="store_true", + help="disable the default non-product exclusion and compile the whole tree -- " + "tests, examples, docs and vendored code included. Equivalent to a config " + "with `build.exclude: []`, and wins over any config file for this run.", + ) args = parser.parse_args(argv) # Validate the source tree up front. Without this the streaming build path # happily runs against a nonexistent path or a single file, finds no frontend @@ -300,6 +314,27 @@ def _run(argv: list[str] | None = None) -> None: for included in include_paths: if not os.path.exists(included): parser.error(f"--include path does not exist: {included}") + # Resolve the project config (lachesis.yml). Its build.paths filter carries the + # non-product exclusion default -- tests, examples, docs, fixtures, benchmarks and + # vendored trees are dropped from the graph unless the tree opts back in with + # `build.exclude: []`/an allow-list, or this run passes --all-sources. The filter is + # threaded into every build variant *and* into source_content_hash, so a filtered + # build and its cache-validity key describe the very same file set. Any config knob + # that mirrors an env var (the `runtime:` block, atropos root) is applied to the + # environment here, before the first pipeline call reads it; setdefault keeps an + # inherited env var winning over the file, matching the documented precedence. + from lachesis import config as _config + try: + cfg = _config.load(start=args.source_dir, explicit=args.config) + except _config.ConfigError as error: + parser.error(str(error)) + for warning in cfg.warnings: + print(f"lachesis config: {warning}", file=sys.stderr) + if cfg.source: + print(f"lachesis: using config {cfg.source}", file=sys.stderr) + _config.apply_runtime_env(cfg) + # --all-sources wins over the file: complete coverage, no exclusion. + path_filter = None if args.all_sources else cfg.build.paths # --prune deletes pure-lexical/proof records at the store boundary, so apply the # same output defaults before the streaming branch as the ordinary path below. # Previously the early return skipped this block and made --stream-shards run @@ -314,11 +349,13 @@ def _run(argv: list[str] | None = None) -> None: args.source_dir, args.stream_shards, frontend_out, timeout_seconds=args.timeout, max_files_per_package=args.shard_large_packages, + path_filter=path_filter, ) else: readers, snapshots = run_project_streaming( args.source_dir, args.stream_shards, frontend_out, timeout_seconds=args.timeout, include_paths=include_paths, + path_filter=path_filter, ) if not snapshots: parser.error( @@ -374,6 +411,7 @@ def _run(argv: list[str] | None = None) -> None: readers, snapshots = run_project_streaming( args.source_dir, stream_root, frontend_out, timeout_seconds=args.timeout, include_paths=include_paths, + path_filter=path_filter, ) if not snapshots: parser.error( @@ -402,17 +440,20 @@ def _run(argv: list[str] | None = None) -> None: args.source_dir, frontend_out, enrich=compile_enrich, max_workers=args.max_workers, timeout_seconds=args.timeout, max_files_per_package=args.shard_large_packages, + path_filter=path_filter, ) elif args.incremental: graph, snapshots = run_project_incremental(args.source_dir, frontend_out, enrich=compile_enrich, timeout_seconds=args.timeout, - include_paths=include_paths) + include_paths=include_paths, + path_filter=path_filter) else: graph, snapshots = run_project(args.source_dir, frontend_out, enrich=compile_enrich, timeout_seconds=args.timeout, - include_paths=include_paths) + include_paths=include_paths, + path_filter=path_filter) build_fingerprint = None if args.incremental and frontend_out: manifest_path = default_manifest_path(frontend_out) @@ -444,7 +485,8 @@ def _run(argv: list[str] | None = None) -> None: source_dir=args.source_dir if args.reduced else None, # Hashed rather than assumed: the store records what the tree was at build time, # so a load can tell whether an already-joined cache still describes it. - source_content_hash=(source_content_hash(args.source_dir, include_paths=include_paths) + source_content_hash=(source_content_hash(args.source_dir, include_paths=include_paths, + path_filter=path_filter) if args.reduced else None), build_fingerprint=build_fingerprint, ) diff --git a/lachesis/cli/main.py b/lachesis/cli/main.py index a11ae0cf..5f55436e 100644 --- a/lachesis/cli/main.py +++ b/lachesis/cli/main.py @@ -676,6 +676,10 @@ def command_build(args: argparse.Namespace) -> int: forwarded.extend(["--stream-shards", args.stream_shards]) for included in getattr(args, "include_paths", None) or []: forwarded.extend(["--include", included]) + if getattr(args, "config", None): + forwarded.extend(["--config", args.config]) + if getattr(args, "all_sources", False): + forwarded.append("--all-sources") return analyze.main(forwarded) @@ -966,6 +970,13 @@ def build_parser() -> argparse.ArgumentParser: help="also analyse this file or directory even if it is outside " "source_dir (repeatable); point it at an advisory's file so a " "narrowed scope never excludes the file the run must reach") + build.add_argument("--config", metavar="FILE", default=None, + help="lachesis.yml to control this build (default: search upward " + "from source_dir). Its built-in default excludes tests, " + "examples, docs, fixtures, benchmarks and vendored trees.") + build.add_argument("--all-sources", action="store_true", + help="compile the whole tree, including tests/examples/docs/vendor " + "(disables the non-product exclusion; wins over any config)") build.set_defaults(handler=command_build, no_prune=False) trace = subcommands.add_parser( diff --git a/lachesis/config.py b/lachesis/config.py new file mode 100644 index 00000000..0299d5ed --- /dev/null +++ b/lachesis/config.py @@ -0,0 +1,451 @@ +"""Project configuration for lachesis (``lachesis.yml``). + +A single optional file that controls what a build ingests, how large a graph or +an export may grow, where the atropos catalog lives, and the runtime knobs that +are otherwise reachable only through ``LACHESIS_*`` environment variables. It is +discovered by walking up from the analysed source tree (and the current working +directory), or named explicitly with ``--config``. + +Precedence, highest first: an explicit CLI flag, then the matching environment +variable, then this file, then the built-in default. A repository that ships a +``lachesis.yml`` therefore changes the defaults for everyone who builds it, while +a one-off flag or env var still wins for a single run. + +Two deliberate departures from "no file means no change": + +* Non-product code — tests, examples, docs, fixtures, benchmarks, vendored + trees, and their common synonyms — is excluded from a build **by default**, + with or without a config file. A tree is understood by its product source; the + scaffolding around it drowns the projection and the architecture ranking. Set + ``build.exclude: []`` (or list an ``build.include`` allow-list) to bring any of + it back. + +* The parser is loaded lazily. PyYAML is imported only when a config file is + actually found, so the stdlib-only core is unaffected for anyone who never + writes one. A file that is present but unparseable because PyYAML is missing is + a hard, explained error rather than a silent skip. + +The schema is intentionally broad and forward-looking: every knob that today is +a CLI flag or an ``LACHESIS_*`` variable has a home here, grouped by the stage it +controls (``build``, ``export``, ``runtime``, ``atropos``). Unknown keys are +reported, not fatal, so a newer config file stays loadable by an older reader. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional, Sequence + +# The names looked for when walking up the tree, in order of preference. The +# dotfile form is accepted so a repo can keep the file out of the way. +CONFIG_FILENAMES = ("lachesis.yml", "lachesis.yaml", ".lachesis.yml", ".lachesis.yaml") + +# The built-in "non-product" exclusion set. A path is non-product when any of its +# directory segments is one of these scaffolding names, or when its basename is a +# test module. Matching is on path *segments* and basenames, never on substrings, +# so product files whose name merely contains a keyword — ``testing.py``, +# ``templating.py``, ``documentation.py`` — are kept. Validated against the pallets +# /flask tree: of 83 .py files it drops exactly the 41 tests + 17 examples + 1 doc +# and keeps all 24 ``src/flask`` modules. +# +# Kept as a single compiled regex (rather than a glob list) because it is the +# default applied on every build and both the builder and the exporter consult +# it; the user-facing ``build.exclude`` list is expressed as globs and compiled +# on top of this. +_NONPRODUCT_SEGMENTS = ( + r"tests?", r"testing", r"__tests__", r"specs?", + r"examples?", r"samples?", r"demos?", + r"docs?", r"documentation", + r"benchmarks?", r"bench", r"perf", + r"fixtures?", r"testdata", r"test[_-]?data", r"__mocks__", r"mocks", + r"vendor", r"vendored", r"third[_-]?party", r"node_modules", r"site-packages", + r"\.tox", r"\.nox", r"\.venv", r"venv", +) +_NONPRODUCT_BASENAMES = ( + r"conftest\.py", + r"test_[^/]*\.py", r"[^/]*_test\.py", + r"[^/]*\.spec\.[a-z]+", r"[^/]*\.test\.[a-z]+", +) +NONPRODUCT_RE = re.compile( + r"(?:^|/)(?:" + "|".join(_NONPRODUCT_SEGMENTS) + r")(?:/|$)" + r"|(?:^|/)(?:" + "|".join(_NONPRODUCT_BASENAMES) + r")$", + re.IGNORECASE, +) + + +# Every ``LACHESIS_*`` variable the codebase reads, so a ``runtime:`` block can set +# any of them from the file with the same precedence a shell export would have. A key +# outside this set is still applied (a newer reader may know a variable this one does +# not), but it is reported as a warning so a typo is visible rather than silent. Kept +# here, next to the parser, deliberately: it is the single list a reviewer checks when +# a new env var is introduced. ``ATROPOS_ROOT`` is included though it lacks the prefix +# because the atropos resolver reads it and the ``atropos.root`` knob maps onto it. +KNOWN_RUNTIME_ENV = frozenset({ + "ATROPOS_ROOT", + "LACHESIS_ATROPOS_TIMINGS", "LACHESIS_BIN", "LACHESIS_BIND_SIDECAR", + "LACHESIS_BIND_SIDECAR_MAX_MB", "LACHESIS_BLESS", "LACHESIS_C_CHUNK_FILES", + "LACHESIS_C_JOBS", "LACHESIS_CACHE_DIR", "LACHESIS_CFLAGS", "LACHESIS_COLUMNAR", + "LACHESIS_COMPILE_COMMANDS", "LACHESIS_CONCEPT_CACHE", "LACHESIS_CORPUS_ROOT", + "LACHESIS_DEFER_TRANSLATION_FACTS", "LACHESIS_EMIT_PROOFS", "LACHESIS_EMIT_TOKENS", + "LACHESIS_ENRICH_AT_BUILD", "LACHESIS_ENRICH_SHARDS", "LACHESIS_EQUALITY_HARNESS", + "LACHESIS_EQUALITY_TIER", "LACHESIS_FORMAT", "LACHESIS_FRONTEND_JOBS", + "LACHESIS_GRAPH", "LACHESIS_HARD_STOP", "LACHESIS_HOME", + "LACHESIS_INCLUDE_DEP_TYPES", "LACHESIS_INCLUDE_DIRS_FILE", "LACHESIS_INPROCESS", + "LACHESIS_ISOLATE_NATIVE", "LACHESIS_KUZU_BATCH", "LACHESIS_KUZU_BPS", + "LACHESIS_KUZU_BUFFER_POOL_SIZE", "LACHESIS_KUZU_CHECKPOINT_THRESHOLD", + "LACHESIS_KUZU_LOW_MEMORY", "LACHESIS_KUZU_MAX_DB_SIZE", + "LACHESIS_KUZU_QUERY_THREADS", "LACHESIS_MAX_DEPENDENCY_FILES", + "LACHESIS_MCP_PROFILE", "LACHESIS_MEMORY_BUDGET_MB", "LACHESIS_NATIVE_ATROPOS_LIB", + "LACHESIS_NATIVE_LIFETIME_LIB", "LACHESIS_NO_PROGRESS", "LACHESIS_PASS2_TIMINGS", + "LACHESIS_PROFILE", "LACHESIS_ROOTS_FILE", "LACHESIS_SEMANTIC_SHARDS", + "LACHESIS_SHARD_DIR", "LACHESIS_SHARD_ID", "LACHESIS_SHARD_ROOT", + "LACHESIS_SOURCE_MAP", "LACHESIS_SOURCE_ROOT", "LACHESIS_STREAM_BATCH_ROWS", + "LACHESIS_TIER_VALIDATION", "LACHESIS_TIMEIT", "LACHESIS_TIMEIT_REPORT", + "LACHESIS_TIMINGS", "LACHESIS_TRACEBACK", "LACHESIS_TS_MAX_OLD_SPACE_MB", + "LACHESIS_TS_STACK_KB", +}) + + +def is_nonproduct(relpath: str) -> bool: + """Whether a repo-relative path is scaffolding rather than product source. + + The default build- and export-time filter. Operates on the forward-slashed + repo-relative form (``display_path`` in the frontend), so it is independent of + where the tree lives on disk. + """ + return NONPRODUCT_RE.search(relpath.replace(os.sep, "/")) is not None + + +def _glob_to_regex(pattern: str) -> re.Pattern[str]: + """Translate a path glob to an anchored regex. + + Supports ``**`` (any run of characters including ``/``), ``*`` (any run within + a single path segment), and ``?`` (one non-separator character). A bare name + like ``tests`` matches that segment anywhere in the path, so ``tests`` and + ``tests/**`` and ``**/tests/**`` all do the intuitive thing. + """ + pattern = pattern.strip().replace(os.sep, "/") + out: list[str] = [] + i, n = 0, len(pattern) + while i < n: + c = pattern[i] + if c == "*": + if i + 1 < n and pattern[i + 1] == "*": + out.append(".*") + i += 2 + # A trailing ``/`` after ``**`` may match nothing. + if i < n and pattern[i] == "/": + out.append("/?") + i += 1 + continue + out.append("[^/]*") + i += 1 + continue + if c == "?": + out.append("[^/]") + elif c == "/": + out.append("/") + else: + out.append(re.escape(c)) + i += 1 + body = "".join(out) + # A bare segment name (no separators, no wildcards) matches that segment + # wherever it appears in the path. + if "/" not in pattern and "*" not in pattern and "?" not in pattern: + return re.compile(r"(?:^|/)" + body + r"(?:/|$)") + return re.compile(r"^" + body + r"$") + + +@dataclass(frozen=True) +class PathFilter: + """A compiled exclude/include decision over repo-relative paths. + + ``include`` is an allow-list that wins over ``exclude`` (and over the built-in + non-product default), so a specific example can be kept without disabling the + whole default. ``use_nonproduct_default`` folds the built-in scaffolding set + into the exclusion; the loader clears it when the user sets ``exclude`` to an + explicit list so that ``exclude: []`` genuinely means "compile everything". + """ + + exclude: tuple[re.Pattern[str], ...] = () + include: tuple[re.Pattern[str], ...] = () + use_nonproduct_default: bool = True + + def excluded(self, relpath: str) -> bool: + rel = relpath.replace(os.sep, "/") + if any(p.search(rel) for p in self.include): + return False + if any(p.search(rel) for p in self.exclude): + return True + if self.use_nonproduct_default and is_nonproduct(rel): + return True + return False + + def keep(self, relpath: str) -> bool: + return not self.excluded(relpath) + + +@dataclass(frozen=True) +class BuildConfig: + """What a build ingests and how large it may grow.""" + + paths: PathFilter = field(default_factory=PathFilter) + max_files: Optional[int] = None # cap on files compiled; None = no cap + max_nodes: Optional[int] = None # cap on graph node count; None = no cap + prune_tokens: Optional[bool] = None # None defers to the CLI default (--prune) + timeout_seconds: Optional[int] = None + frontend_jobs: Optional[int] = None + memory_budget_mb: Optional[int] = None + + +@dataclass(frozen=True) +class ExportConfig: + """Bounds and toggles on the comprehension bundle projection.""" + + max_nodes: Optional[int] = None # projection budget + max_entrypoints: Optional[int] = None + include_tests: bool = False # Dense-mode opt-in; default off + paths: PathFilter = field(default_factory=PathFilter) + + +@dataclass(frozen=True) +class AtroposConfig: + """Where the atropos catalog lives and which of it is active.""" + + root: Optional[str] = None # replaces $ATROPOS_ROOT / resolver default + enabled: bool = True + native_lib: Optional[str] = None # $LACHESIS_NATIVE_ATROPOS_LIB + timings: Optional[bool] = None # $LACHESIS_ATROPOS_TIMINGS + languages: Optional[tuple[str, ...]] = None # None = all present + kinds: Optional[tuple[str, ...]] = None # enable-list of sink kinds + flow_patterns: Optional[tuple[str, ...]] = None # enable-list of pattern ids + + +@dataclass(frozen=True) +class Config: + """A resolved configuration. ``source`` is the file it came from, if any.""" + + build: BuildConfig = field(default_factory=BuildConfig) + export: ExportConfig = field(default_factory=ExportConfig) + atropos: AtroposConfig = field(default_factory=AtroposConfig) + runtime: Mapping[str, Any] = field(default_factory=dict) + source: Optional[str] = None + warnings: tuple[str, ...] = () + + +# --------------------------------------------------------------------------- +# Discovery +# --------------------------------------------------------------------------- + +def find_config(start: Optional[str] = None) -> Optional[Path]: + """The nearest config file at or above ``start`` (default: cwd). + + Walks upward to the filesystem root, returning the first match. A build points + ``start`` at the analysed source directory so the tree's own config is found + even when the build is launched from elsewhere. + """ + base = Path(start or os.getcwd()).resolve() + for directory in (base, *base.parents): + for name in CONFIG_FILENAMES: + candidate = directory / name + if candidate.is_file(): + return candidate + return None + + +def _load_yaml(path: Path) -> dict[str, Any]: + """Parse a config file, importing PyYAML lazily. + + A present-but-unparseable file is an error a user needs to see, so a missing + parser is raised with an actionable message rather than swallowed. + """ + try: + import yaml # type: ignore + except ImportError as exc: # pragma: no cover - environment-dependent + raise ConfigError( + f"{path} is present but PyYAML is not installed. Install it with " + f"`pip install lachesis-cpg[config]` (or `pip install pyyaml`), or " + f"remove the file to fall back to defaults." + ) from exc + try: + data = yaml.safe_load(path.read_text()) or {} + except yaml.YAMLError as exc: + raise ConfigError(f"{path} is not valid YAML: {exc}") from exc + if not isinstance(data, dict): + raise ConfigError(f"{path} must be a mapping at the top level, got {type(data).__name__}.") + return data + + +class ConfigError(Exception): + """A config file exists but cannot be honored.""" + + +# --------------------------------------------------------------------------- +# Parsing / resolution +# --------------------------------------------------------------------------- + +def _compile_globs(patterns: Iterable[str]) -> tuple[re.Pattern[str], ...]: + return tuple(_glob_to_regex(str(p)) for p in patterns) + + +def _path_filter(section: Mapping[str, Any], warnings: list[str], where: str) -> PathFilter: + exclude = section.get("exclude") + include = section.get("include", []) or [] + if exclude is None: + # Key absent: keep the non-product default, no explicit patterns. + return PathFilter( + include=_compile_globs(include), + use_nonproduct_default=True, + ) + if not isinstance(exclude, list) or not isinstance(include, list): + warnings.append(f"{where}: `exclude`/`include` must be lists; ignoring.") + return PathFilter(use_nonproduct_default=True) + # An explicit exclude list replaces the default set. `exclude: []` therefore + # means "compile everything", which is the intuitive reading. + return PathFilter( + exclude=_compile_globs(exclude), + include=_compile_globs(include), + use_nonproduct_default=False, + ) + + +def _as_int(value: Any, where: str, warnings: list[str]) -> Optional[int]: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + warnings.append(f"{where}: expected an integer, got {value!r}; ignoring.") + return None + + +def _as_bool(value: Any, where: str, warnings: list[str]) -> Optional[bool]: + if value is None: + return None + if isinstance(value, bool): + return value + warnings.append(f"{where}: expected true/false, got {value!r}; ignoring.") + return None + + +def _as_str_tuple(value: Any) -> Optional[tuple[str, ...]]: + if value is None: + return None + if isinstance(value, str): + return (value,) + if isinstance(value, list): + return tuple(str(v) for v in value) + return None + + +def parse(data: Mapping[str, Any], source: Optional[str] = None) -> Config: + """Turn a raw config mapping into a resolved, typed ``Config``. + + Tolerant by design: a malformed field is dropped with a warning rather than + failing the build, and unknown top-level sections are reported so a config + written for a newer reader still loads. + """ + warnings: list[str] = [] + known = {"build", "export", "atropos", "runtime"} + for key in data: + if key not in known: + warnings.append(f"unknown top-level section {key!r}; ignoring.") + + build_raw = data.get("build") or {} + export_raw = data.get("export") or {} + atropos_raw = data.get("atropos") or {} + runtime_raw = data.get("runtime") or {} + + build = BuildConfig( + paths=_path_filter(build_raw, warnings, "build"), + max_files=_as_int(build_raw.get("max_files"), "build.max_files", warnings), + max_nodes=_as_int(build_raw.get("max_nodes"), "build.max_nodes", warnings), + prune_tokens=_as_bool(build_raw.get("prune_tokens"), "build.prune_tokens", warnings), + timeout_seconds=_as_int(build_raw.get("timeout_seconds"), "build.timeout_seconds", warnings), + frontend_jobs=_as_int(build_raw.get("frontend_jobs"), "build.frontend_jobs", warnings), + memory_budget_mb=_as_int(build_raw.get("memory_budget_mb"), "build.memory_budget_mb", warnings), + ) + export = ExportConfig( + max_nodes=_as_int(export_raw.get("max_nodes"), "export.max_nodes", warnings), + max_entrypoints=_as_int(export_raw.get("max_entrypoints"), "export.max_entrypoints", warnings), + include_tests=bool(_as_bool(export_raw.get("include_tests"), "export.include_tests", warnings) or False), + paths=_path_filter(export_raw, warnings, "export") if ("exclude" in export_raw or "include" in export_raw) else build.paths, + ) + atropos = AtroposConfig( + root=atropos_raw.get("root"), + enabled=bool(_as_bool(atropos_raw.get("enabled"), "atropos.enabled", warnings) if atropos_raw.get("enabled") is not None else True), + native_lib=atropos_raw.get("native_lib"), + timings=_as_bool(atropos_raw.get("timings"), "atropos.timings", warnings), + languages=_as_str_tuple(atropos_raw.get("languages")), + kinds=_as_str_tuple(atropos_raw.get("kinds")), + flow_patterns=_as_str_tuple(atropos_raw.get("flow_patterns")), + ) + if runtime_raw and not isinstance(runtime_raw, dict): + warnings.append("runtime: expected a mapping of LACHESIS_* variables; ignoring.") + runtime_raw = {} + runtime: dict[str, Any] = {} + for key, value in (runtime_raw or {}).items(): + name = str(key) + if name not in KNOWN_RUNTIME_ENV: + warnings.append( + f"runtime.{name}: not a known LACHESIS_* variable; applying anyway." + ) + runtime[name] = value + + return Config( + build=build, + export=export, + atropos=atropos, + runtime=runtime, + source=source, + warnings=tuple(warnings), + ) + + +def _env_str(value: Any) -> str: + """Render a config value the way a shell export would carry it.""" + if isinstance(value, bool): + return "1" if value else "0" + return str(value) + + +def apply_runtime_env(config: Config) -> None: + """Fold config-declared runtime knobs into ``os.environ``. + + Every key of the ``runtime:`` block is a ``LACHESIS_*`` variable, and the atropos + root/native-lib/timings map onto the environment variables their resolver reads. + ``setdefault`` is deliberate: an inherited environment variable still wins over the + file, which is the documented precedence (flag > env > file > default). Idempotent, + and safe to call before any pipeline import — nothing here imports the pipeline. + """ + for name, value in (config.runtime or {}).items(): + os.environ.setdefault(str(name), _env_str(value)) + atropos = config.atropos + if atropos.root: + os.environ.setdefault("ATROPOS_ROOT", str(atropos.root)) + if atropos.native_lib: + os.environ.setdefault("LACHESIS_NATIVE_ATROPOS_LIB", str(atropos.native_lib)) + if atropos.timings is not None: + os.environ.setdefault("LACHESIS_ATROPOS_TIMINGS", _env_str(atropos.timings)) + + +def load(start: Optional[str] = None, explicit: Optional[str] = None) -> Config: + """Discover and parse the config, or return the all-default ``Config``. + + ``explicit`` (a ``--config`` path) is honored verbatim and must exist; + otherwise the tree is searched from ``start``. When no file is found the + returned ``Config`` is all-defaults — which still excludes non-product code, + because that default lives in ``PathFilter`` itself, not in the file. + """ + if explicit: + path = Path(explicit).resolve() + if not path.is_file(): + raise ConfigError(f"--config {explicit} does not exist.") + else: + path = find_config(start) + if path is None: + return Config() + return parse(_load_yaml(path), source=str(path)) diff --git a/lachesis/pipeline.py b/lachesis/pipeline.py index b2c6716c..6a1dc372 100644 --- a/lachesis/pipeline.py +++ b/lachesis/pipeline.py @@ -4,7 +4,10 @@ import hashlib import os from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple + +if TYPE_CHECKING: + from .config import PathFilter from .core.contract import ContractError as FrontendError, FrontendSnapshot from .core.composition import _EdgeKeys @@ -101,6 +104,7 @@ def source_inventory( source_dir: str, include_tests: bool = True, include_paths: Sequence[str] = (), + path_filter: Optional["PathFilter"] = None, ) -> List[str]: """Discover every supported source file, including tests and specifications. @@ -108,6 +112,14 @@ def source_inventory( because its path looks like a test. Callers that explicitly need a production-only inventory may still pass ``include_tests=False``. + ``path_filter`` is the general form of that opt-out. When supplied (the CLI resolves + one from ``lachesis.yml``, whose built-in default excludes tests, examples, docs, + fixtures, benchmarks and vendored trees), a walked file is dropped when the filter + reports its repo-relative path as excluded. It never vetoes an *explicitly named* + ``include_paths`` file — the guided-scope guarantee that a deliberately scoped file + is always analysed outranks the exclusion default — but it does apply while walking + an explicitly named ``include_paths`` *directory*, matching the main walk. + ``include_paths`` names extra files or directories to fold into the inventory even when they lie *outside* ``source_dir``. This is the guided-scope guarantee: when a build is deliberately narrowed to a sub-tree to fit a time budget, the advisory's @@ -179,6 +191,13 @@ def _walk(base_dir: str, containment_root: str) -> List[str]: continue if is_test is not None and is_test(path): continue + if path_filter is not None: + # Match on the repo-relative, forward-slashed display form so the + # decision is independent of where the tree lives on disk and lines + # up with the ``display_path`` the frontend later records. + rel = os.path.relpath(path, containment_root) + if path_filter.excluded(rel): + continue collected.append(path) return collected @@ -260,6 +279,7 @@ def run_project( include_paths: Sequence[str] = (), *, enrich: bool = False, + path_filter: Optional["PathFilter"] = None, ) -> Tuple[CodeGraph, List[FrontendSnapshot]]: """Run selected frontends and compose the canonical core graph. @@ -276,7 +296,8 @@ def run_project( source_dir = os.path.abspath(source_dir) registry = registry or default_registry() groups = registry.partition( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths)) + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter)) snapshots = [] for frontend_id in sorted(groups): frontend = registry.get(frontend_id) @@ -342,6 +363,7 @@ def run_project_streaming( timeout_seconds: int = 300, include_tests: bool = True, include_paths: Sequence[str] = (), + path_filter: Optional["PathFilter"] = None, ): """Run frontends one at a time and return shard readers plus metadata. @@ -355,7 +377,8 @@ def run_project_streaming( output_root = os.path.abspath(output_root) registry = registry or default_registry() groups = registry.partition( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths)) + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter)) snapshots = [] readers = [] from .resources import c_chunk_files, frontend_jobs as configured_frontend_jobs @@ -438,6 +461,7 @@ def run_project_streaming_parallel( *, max_files_per_package: Optional[int] = None, workspace_root: Optional[str] = None, + path_filter: Optional["PathFilter"] = None, ): """Stream package/shard compiler jobs without composing their snapshots. @@ -454,11 +478,13 @@ def run_project_streaming_parallel( output_root = os.path.abspath(output_root) registry = registry or default_registry(workspace_root) packages = detect_packages( - source_dir, source_inventory(source_dir, include_tests=include_tests), + source_dir, + source_inventory(source_dir, include_tests=include_tests, path_filter=path_filter), ) packages = split_large_packages(source_dir, packages, max_files_per_package) jobs = package_jobs(source_dir, output_root, registry, - include_tests=include_tests, packages=packages) + include_tests=include_tests, packages=packages, + path_filter=path_filter) if not jobs: supported = sorted({ extension for item in registry.frontends for extension in item.extensions @@ -553,6 +579,7 @@ def source_content_hash( source_dir: str, include_tests: bool = True, include_paths: Sequence[str] = (), + path_filter: Optional["PathFilter"] = None, ) -> str: """One digest over every source file a build of ``source_dir`` would see. @@ -564,7 +591,8 @@ def source_content_hash( unchanged content keeps the cache, and a restored older file loses it. """ digests = _group_digests( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths), + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter), os.path.abspath(source_dir)) digest = hashlib.sha256() for path in sorted(digests): @@ -658,6 +686,7 @@ def run_project_incremental( include_paths: Sequence[str] = (), *, enrich: bool = True, + path_filter: Optional["PathFilter"] = None, ) -> Tuple[CodeGraph, List[FrontendSnapshot]]: """Like ``run_project`` but reuse a frontend's prior on-disk bundle when none of its source files changed, recompiling only the frontends that did. @@ -674,7 +703,8 @@ def run_project_incremental( registry = registry or default_registry() manifest_path = manifest_path or default_manifest_path(output_root) groups = registry.partition( - source_inventory(source_dir, include_tests=include_tests, include_paths=include_paths)) + source_inventory(source_dir, include_tests=include_tests, + include_paths=include_paths, path_filter=path_filter)) prior = _load_manifest(manifest_path) snapshots: List[FrontendSnapshot] = [] @@ -735,6 +765,7 @@ def package_jobs( registry: FrontendRegistry, include_tests: bool = True, packages: Optional[Dict[str, List[str]]] = None, + path_filter: Optional["PathFilter"] = None, ) -> List[Tuple[str, str, str, str, List[str]]]: """The (frontend_id, package, compile_root, output_dir, roots) units of a build. @@ -753,7 +784,8 @@ def package_jobs( output_root = os.path.abspath(output_root) if packages is None: packages = detect_packages( - source_dir, source_inventory(source_dir, include_tests=include_tests), + source_dir, + source_inventory(source_dir, include_tests=include_tests, path_filter=path_filter), ) jobs = [] for (frontend_id, package), roots in registry.partition_by_package(packages).items(): @@ -862,6 +894,7 @@ def run_project_parallel( max_workers: Optional[int] = None, max_files_per_package: Optional[int] = None, workspace_root: Optional[str] = None, + path_filter: Optional["PathFilter"] = None, ) -> Tuple[CodeGraph, List[FrontendSnapshot], int]: """Compile each (frontend, package) unit in its own process, then compose. @@ -888,11 +921,13 @@ def run_project_parallel( output_root = os.path.abspath(output_root) registry = registry or default_registry(workspace_root) packages = detect_packages( - source_dir, source_inventory(source_dir, include_tests=include_tests), + source_dir, + source_inventory(source_dir, include_tests=include_tests, path_filter=path_filter), ) packages = split_large_packages(source_dir, packages, max_files_per_package) jobs = package_jobs(source_dir, output_root, registry, - include_tests=include_tests, packages=packages) + include_tests=include_tests, packages=packages, + path_filter=path_filter) if not jobs: supported = sorted({ extension for item in registry.frontends for extension in item.extensions diff --git a/lachesis/test_config.py b/lachesis/test_config.py new file mode 100644 index 00000000..812e2c6a --- /dev/null +++ b/lachesis/test_config.py @@ -0,0 +1,145 @@ +"""Project configuration (``lachesis.yml``): classifier, globs, precedence, env. + +The classifier is the default build- and export-time filter, so its two failure +modes both matter: dropping a product module whose *name* merely contains a keyword +(``testing.py``), and keeping scaffolding it should drop. The glob translator, the +exclude/include precedence, and the runtime-env passthrough are pinned here too. The +YAML-loading tests are skipped when PyYAML is absent — that is a genuine environment +gap, not a defect, and the lazy-import contract is exactly that the core runs without +it. +""" +import os +import tempfile +import unittest +from pathlib import Path + +import pytest + +from lachesis import config + + +class ClassifierTests(unittest.TestCase): + def test_keeps_product_modules_that_contain_a_keyword(self): + for path in ("src/flask/testing.py", "src/flask/templating.py", + "pkg/documentation.py", "a/b/specs_helper.py"): + self.assertFalse(config.is_nonproduct(path), path) + + def test_drops_scaffolding_by_segment_and_basename(self): + for path in ("tests/test_cli.py", "examples/tutorial/app.py", "docs/conf.py", + "benchmarks/bench_x.py", "fixtures/data.py", "vendor/lib.py", + "third_party/x.py", "node_modules/y.js", "conftest.py", + "src/foo_test.py", "a/thing.spec.ts", "a/thing.test.js"): + self.assertTrue(config.is_nonproduct(path), path) + + +class GlobTests(unittest.TestCase): + def test_bare_segment_matches_anywhere(self): + pat = config._glob_to_regex("tests") + self.assertTrue(pat.search("a/tests/b.py")) + self.assertTrue(pat.search("tests/b.py")) + self.assertFalse(pat.search("a/testsuite/b.py")) + + def test_doublestar_and_single_star(self): + self.assertTrue(config._glob_to_regex("src/**/gen_*.py").search("src/a/b/gen_x.py")) + self.assertFalse(config._glob_to_regex("src/*.py").search("src/a/b.py")) + + +class PathFilterTests(unittest.TestCase): + def test_default_excludes_nonproduct(self): + pf = config.PathFilter() + self.assertTrue(pf.excluded("tests/test_a.py")) + self.assertFalse(pf.excluded("src/app.py")) + + def test_explicit_empty_exclude_keeps_everything(self): + # `exclude: []` clears the default; nothing is excluded. + pf = config.parse({"build": {"exclude": []}}).build.paths + self.assertFalse(pf.excluded("tests/test_a.py")) + + def test_explicit_exclude_replaces_default(self): + pf = config.parse({"build": {"exclude": ["examples"]}}).build.paths + self.assertTrue(pf.excluded("examples/x.py")) + # Tests are no longer dropped: the explicit list is the whole policy now. + self.assertFalse(pf.excluded("tests/test_a.py")) + + def test_include_allowlist_wins(self): + pf = config.parse({"build": {"include": ["tests/keep_me.py"]}}).build.paths + self.assertFalse(pf.excluded("tests/keep_me.py")) + self.assertTrue(pf.excluded("tests/other.py")) + + +class ParseTests(unittest.TestCase): + def test_unknown_section_warns_not_fatal(self): + cfg = config.parse({"nonsense": 1, "build": {"max_files": 10}}) + self.assertEqual(cfg.build.max_files, 10) + self.assertTrue(any("nonsense" in w for w in cfg.warnings)) + + def test_bad_int_is_dropped_with_a_warning(self): + cfg = config.parse({"build": {"max_nodes": "lots"}}) + self.assertIsNone(cfg.build.max_nodes) + self.assertTrue(any("max_nodes" in w for w in cfg.warnings)) + + def test_export_paths_default_to_build_paths(self): + cfg = config.parse({"build": {"exclude": ["examples"]}}) + self.assertTrue(cfg.export.paths.excluded("examples/x.py")) + + def test_unknown_runtime_var_warns_but_applies(self): + cfg = config.parse({"runtime": {"LACHESIS_NOT_REAL": "1"}}) + self.assertIn("LACHESIS_NOT_REAL", cfg.runtime) + self.assertTrue(any("LACHESIS_NOT_REAL" in w for w in cfg.warnings)) + + +class RuntimeEnvTests(unittest.TestCase): + def test_apply_sets_env_but_env_wins(self): + cfg = config.parse({ + "runtime": {"LACHESIS_MEMORY_BUDGET_MB": 2048}, + "atropos": {"root": "/opt/atropos", "timings": True}, + }) + saved = {k: os.environ.get(k) for k in + ("LACHESIS_MEMORY_BUDGET_MB", "ATROPOS_ROOT", "LACHESIS_ATROPOS_TIMINGS")} + try: + for k in saved: + os.environ.pop(k, None) + config.apply_runtime_env(cfg) + self.assertEqual(os.environ["LACHESIS_MEMORY_BUDGET_MB"], "2048") + self.assertEqual(os.environ["ATROPOS_ROOT"], "/opt/atropos") + self.assertEqual(os.environ["LACHESIS_ATROPOS_TIMINGS"], "1") + # An inherited env var wins over the file (setdefault). + os.environ["LACHESIS_MEMORY_BUDGET_MB"] = "9999" + config.apply_runtime_env(cfg) + self.assertEqual(os.environ["LACHESIS_MEMORY_BUDGET_MB"], "9999") + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +class LoadTests(unittest.TestCase): + def test_no_file_is_all_defaults_still_excluding_nonproduct(self): + with tempfile.TemporaryDirectory() as d: + cfg = config.load(start=d) + self.assertIsNone(cfg.source) + self.assertTrue(cfg.build.paths.excluded("tests/test_a.py")) + + def test_finds_file_walking_up(self): + pytest.importorskip("yaml") + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "lachesis.yml").write_text("build:\n max_files: 7\n", "utf-8") + sub = root / "a" / "b" + sub.mkdir(parents=True) + cfg = config.load(start=str(sub)) + self.assertEqual(cfg.build.max_files, 7) + # find_config resolves symlinks (e.g. macOS /var -> /private/var), so + # compare resolved paths rather than the raw tempdir string. + self.assertEqual(Path(cfg.source).resolve(), + (root / "lachesis.yml").resolve()) + + def test_explicit_missing_config_is_an_error(self): + with self.assertRaises(config.ConfigError): + config.load(explicit="/no/such/lachesis.yml") + + +if __name__ == "__main__": + unittest.main() diff --git a/lachesis/test_pipeline_inventory.py b/lachesis/test_pipeline_inventory.py index 5adc590e..bd30c3ad 100644 --- a/lachesis/test_pipeline_inventory.py +++ b/lachesis/test_pipeline_inventory.py @@ -76,6 +76,49 @@ def test_explicit_file_survives_the_test_path_heuristic(self): self.assertIn(str(test_file.resolve()), {os.path.realpath(p) for p in kept}) + def _tree(self, root: Path) -> None: + (root / "src" / "pkg").mkdir(parents=True) + (root / "tests").mkdir() + (root / "examples").mkdir() + (root / "docs").mkdir() + (root / "src" / "pkg" / "app.py").write_text("def a():\n return 1\n", "utf-8") + # A product module whose name merely contains a keyword must survive. + (root / "src" / "pkg" / "testing.py").write_text("def t():\n return 1\n", "utf-8") + (root / "tests" / "test_app.py").write_text("def test():\n pass\n", "utf-8") + (root / "examples" / "demo.py").write_text("def d():\n return 1\n", "utf-8") + (root / "docs" / "conf.py").write_text("x = 1\n", "utf-8") + (root / "conftest.py").write_text("import pytest\n", "utf-8") + + def test_path_filter_drops_nonproduct_by_default(self): + from lachesis.config import Config + with tempfile.TemporaryDirectory() as project: + root = Path(project) + self._tree(root) + kept = {os.path.relpath(p, root) + for p in source_inventory(str(root), path_filter=Config().build.paths)} + self.assertEqual(kept, {os.path.join("src", "pkg", "app.py"), + os.path.join("src", "pkg", "testing.py")}) + + def test_no_path_filter_keeps_everything(self): + with tempfile.TemporaryDirectory() as project: + root = Path(project) + self._tree(root) + kept = {os.path.relpath(p, root) for p in source_inventory(str(root))} + self.assertIn("conftest.py", kept) + self.assertIn(os.path.join("tests", "test_app.py"), kept) + + def test_content_hash_tracks_the_filtered_set(self): + # The cache-validity key must describe exactly the file set the build sees, or a + # filtered build could hit a stale cache written by an unfiltered one. + from lachesis.config import Config + from lachesis.pipeline import source_content_hash + with tempfile.TemporaryDirectory() as project: + root = Path(project) + self._tree(root) + unfiltered = source_content_hash(str(root)) + filtered = source_content_hash(str(root), path_filter=Config().build.paths) + self.assertNotEqual(unfiltered, filtered) + if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index c705aff8..4d9c952b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,10 +40,14 @@ classifiers = [ dependencies = ["kuzu>=0.11,<0.12", "pyarrow>=17,<26", "protobuf>=6.30,<7"] [project.optional-dependencies] -dev = ["pytest>=8"] +dev = ["pytest>=8", "pyyaml>=6"] # Runtime only. Model weights are fetched separately by `lachesis concept-model # download`; neither FastEmbed nor the model is part of the core wheel. concept-search = ["fastembed>=0.8,<0.9"] +# Project configuration (`lachesis.yml`). PyYAML is imported lazily and only when a +# config file is actually present, so the stdlib-only core is unaffected for anyone +# who never writes one; a present-but-unparseable file is a hard, explained error. +config = ["pyyaml>=6"] [project.scripts] # One entrypoint. The reader is a single `lachesis` with subcommands; every pass From d7ba854547f5c336fc48e14f2a65cd4c7e87181f Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 11:09:18 +0530 Subject: [PATCH 12/19] nav(bundle): root request paths in app code and drop dataflow artifacts The graph-first exporter had four content defects the frontend surfaced as if they were the codebase's behavior: 1. Guided request paths were a greedy CALLS-successor walk out of each exported symbol, so they rooted at leaf utilities (render_template, run_command) and never showed the request lifecycle. The architecture core came out empty because the old Hubs block called re.search without importing re (a silent NameError swallowed by its try/except) and shadowed the graph index with the concepts enumerate counter. 3. No core spine was emitted for the same two reasons. 6. Most nodes carried no scope, so the frontend could not group them. 7/8. Value paths featured bare def-use artifacts (a traceback local `tb`, a file handle `f`, `config_file`, `tb.tb_frame`) as "behaviors", and the same def-use was emitted twice from different findings. Mechanism. Requests and core are now built together by _lifecycle_projection, which seeds a bounded execution story from the real top-of-stack drivers (entry handlers first, then in-degree-0 product callables ranked by 2-hop reach), linearises each story's success spine (deranking exception/teardown branches and cutting cycles), and keeps the deepest few. The story spine nodes also populate the architecture core. On Flask this roots the primary path at __call__ -> wsgi_app and puts full_dispatch_request / request_context / push / pop on the core spine, where before it showed render_template. _enrich_graph_nodes now sets `scope` on every node from gl.owner_function: a node reports the enclosing callable's qualified name, a callable reports its module; a synthetic context/heap node with no declaration honestly carries none (115/120 nodes on Flask, the 5 without being file-less overlay nodes). The featured value paths gain two hygiene rules: a path that visits fewer than two distinct nodes has not moved and is dropped (removing all 12 def-use artifacts on Flask while keeping every real flow, including minimal two-step call-argument flows), and paths with identical endpoints and ordered node ids are collapsed. security.findings stays exhaustive -- only paths.values is filtered. Correctness. The 2.0 contract is unchanged: _comprehension_projection returns the same keys, and _graph_first_bundle emits the same shape. The two-distinct -node floor keeps every witness the suite drives (all are two-step with distinct ids), so nav/test_bundle.py, test_config.py and test_pipeline_inventory.py pass (71 tests). Verified by re-exporting the Flask graph: 0 non-product entrypoints, 0 source-less nodes, 0 single-node value artifacts, 0 duplicate value names, requests rooted in app code. --- lachesis/nav/bundle.py | 406 +++++++++++++++++++++++++++++++---------- 1 file changed, 306 insertions(+), 100 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index dcf943fb..5225b0b3 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -30,6 +30,7 @@ import hashlib import json import os +import re import subprocess from datetime import datetime, timezone from typing import Any, Optional @@ -523,38 +524,264 @@ def _norm_node(gl, node: dict) -> dict: "kind": gl.kind(node.get("id")), "file": file, "line": line} -def _call_chain(index, gl, start_id: str, depth: int) -> list[str]: - """A single deterministic CALLS chain out of ``start_id`` (source order). +# The request lifecycle a reader wants is the *success* path; error, teardown and +# logging branches are real but secondary, so we only derank them when choosing the +# primary hop -- never drop them. Substring match keeps this language-agnostic. +_LIFECYCLE_ERROR_TOKENS = ( + "exception", "error", "teardown", "cleanup", "abort", "log_", + "handle_http", "raise_", "rollback", "finalize_request", +) +_CALL_EDGE_KINDS = ("CALLS", "INVOKES", "MAY_INVOKE") - At each hop we descend into the callee that itself calls the most -- the branch - most likely to keep telling the request's story -- breaking ties by label so the - walk is reproducible. Cycles are cut by the visited set; a leaf ends the chain. - This invents no ordering: every consecutive pair is a real ``CALLS`` edge. + +def _is_error_name(name: Optional[str]) -> bool: + n = str(name or "").lower() + return any(tok in n for tok in _LIFECYCLE_ERROR_TOKENS) + + +def _story_fn_openable(fn: dict) -> bool: + """A story step a reader can open: a real product file and a positive line. + + ``execution_story`` reports unresolved/external callees with a null file; those + are honest frontier markers, never places to root or continue a lifecycle spine. """ - chain = [start_id] - seen = {start_id} - cur = start_id - for _ in range(max(0, depth - 1)): - nxt: list[dict] = [] + line = fn.get("line") + return bool(fn.get("file")) and isinstance(line, int) and line > 0 + + +def _reach2(index, node_id: str) -> int: + """Distinct callees within two CALLS hops -- a cheap 'does this drive control?'. + + An orchestration root (a WSGI ``__call__``, a CLI ``main``) fans out into a + broad two-hop cone; a leaf utility barely moves. Ranking candidate roots by this + before paying for a full execution story elevates the real lifecycles without + naming any framework. Bounded by the graph's own fan-out, so it stays cheap. + """ + try: + one = {t.get("id") for t in index.targets(node_id, *_CALL_EDGE_KINDS) + if t.get("id")} + except Exception: + return 0 + total = set(one) + for mid in one: try: - nxt = [n for n in index.targets(cur, "CALLS") - if n.get("id") and n["id"] not in seen] + total.update(t.get("id") for t in index.targets(mid, *_CALL_EDGE_KINDS) + if t.get("id")) except Exception: + continue + total.discard(node_id) + return len(total) + + +def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int) -> list[str]: + """Candidate roots for request-lifecycle stories, best driver first. + + Two sources, deduped in priority order: the planner's entry handlers (already + ranked upstream), then every product callable that nothing else in the product + calls -- an in-degree-0 top-of-stack (a WSGI ``__call__``, an event loop, a + public API orchestrator). The in-degree-0 set is ordered by two-hop reach so the + orchestration roots precede the many leaf helpers that also happen to be + uncalled once tests are excluded. Truncated to ``cap`` so the story pass is + bounded regardless of codebase size. + """ + roots: list[str] = [] + seen: set[str] = set() + for hid in handler_ids: + if hid and hid not in seen: + seen.add(hid) + roots.append(hid) + + drivers: list[tuple[int, str]] = [] + try: + callable_nodes = list(index.nodes_of_kind("function", "method", "constructor")) + except Exception: + callable_nodes = [] + for node in callable_nodes: + nid = node.get("id") + if not nid or nid in seen: + continue + f, l = gl.loc(node)[0], gl.loc(node)[1] + if not f or not isinstance(l, int) or l <= 0: + continue + try: + out = sum(1 for _ in index.targets(nid, *_CALL_EDGE_KINDS)) + if out < 1: + continue + inn = sum(1 for _ in index.sources(nid, *_CALL_EDGE_KINDS)) + except Exception: + continue + if inn == 0: + drivers.append((_reach2(index, nid), nid)) + drivers.sort(key=lambda pair: (-pair[0], pair[1])) + for _, nid in drivers: + if nid not in seen: + seen.add(nid) + roots.append(nid) + return roots[:cap] + + +def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str]]: + """Linearize an execution story into (primary success spine, all functions). + + The story is a call tree keyed by (caller -> function). The spine walks from the + entry always choosing the deepest-subtree callee, deranking obvious error/ + teardown branches, so it follows the happy path (a WSGI entry down through + dispatch to the response) rather than wandering into a handler. Every consecutive + pair on the spine is a real edge the story observed; cycles are cut by ``seen``. + Returns the ordered spine node ids and the flat set of every function id the + story touched (the raw material for the architecture core). + """ + steps = story.get("steps") or [] + entry = (story.get("entry") or {}).get("node_id") + if not entry: + return [], [] + children: dict[str, list[tuple[int, dict]]] = {} + functions: dict[str, dict] = {} + for step in steps: + fn = step.get("function") or {} + fid = fn.get("node_id") + if not fid: + continue + functions[fid] = fn + caller = (step.get("caller") or {}).get("node_id") + if caller: + children.setdefault(caller, []).append((step.get("sequence", 0), fn)) + + memo: dict[str, int] = {} + + def subtree(nid: str, guard: frozenset) -> int: + if nid in memo: + return memo[nid] + if nid in guard: + return 0 + deeper = guard | {nid} + total = 0 + for _, fn in children.get(nid, []): + cid = fn.get("node_id") + if cid: + total += 1 + subtree(cid, deeper) + # Only cache when no guard cycle influenced the count (guard was the path + # to nid); good enough as a heuristic ranker and keeps the walk bounded. + memo[nid] = total + return total + + spine = [entry] + seen = {entry} + cur = entry + while len(spine) < max_hops: + kids = [fn for _, fn in sorted(children.get(cur, []), key=lambda pair: pair[0]) + if fn.get("node_id") not in seen and _story_fn_openable(fn)] + if not kids: break - if not nxt: - break + pick = max(kids, key=lambda fn: ( + 0 if _is_error_name(fn.get("name")) else 1, + subtree(fn.get("node_id"), frozenset()))) + nid = pick.get("node_id") + cur = nid + seen.add(nid) + spine.append(nid) + ordered_functions = [fid for fid in functions if _story_fn_openable(functions[fid])] + return spine, ordered_functions + + +def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], *, + max_requests: int, max_core: int, + max_hops: int) -> tuple[list[dict], list[dict]]: + """Request lifecycles and the architecture core, from bounded execution stories. + + Runs a bounded forward execution story from each candidate driver (see + ``_lifecycle_roots``), ranks them by how much real control each covers (spine + length, then breadth), and keeps the deepest few as guided request paths -- each + the success spine of one story, every consecutive hop a real observed edge. A + shallower story whose root already sits inside a kept spine is skipped, so we do + not emit both ``__call__ -> wsgi_app -> ...`` and its ``wsgi_app -> ...`` suffix. + The union of every kept story's functions, bounded, becomes the core spine a + newcomer reads first. Best-effort: any failure yields empty lists, never raises. + """ + requests: list[dict] = [] + core: list[dict] = [] + try: + roots = _lifecycle_roots(index, gl, handler_ids, cap=30) + except Exception: + return requests, core - def out_degree(node: dict) -> int: + ranked: list[tuple[int, int, str, list[str], list[str]]] = [] + for root in roots: + try: + story = _call("execution_story", + {"entry": root, "max_depth": max_hops + 2, + "max_steps": 120, "format": "json"}) + except Exception: + continue + if not isinstance(story, dict): + continue + spine, functions = _story_spine(story, max_hops=max_hops) + if len(spine) < 2: + continue + ranked.append((len(spine), len(functions), root, spine, functions)) + + # Deepest, then broadest, wins attention; stable by root id for reproducibility. + ranked.sort(key=lambda row: (-row[0], -row[1], row[2])) + + node_ids = set(asm.nodes) + covered: set[str] = set() + core_ids: set[str] = set() + used_ids: set[str] = set() + for _, _, root, spine, functions in ranked: + if len(requests) >= max_requests: + break + if root in covered: # a redundant suffix of a spine already shown + continue + hops: list[dict] = [] + chain_ids: list[str] = [] + for nid in spine: + node = gl.nodes.get(nid) + if node is None: + continue + asm.add_node(_norm_node(gl, node), default_kind="function") + node_ids.add(nid) + hops.append({"node_id": nid, "caption": gl.label(node)}) + chain_ids.append(nid) + if len(hops) < 2: + continue + for a, b in zip(chain_ids, chain_ids[1:]): + asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, node_ids) + covered.update(chain_ids) + root_label = gl.label(gl.nodes.get(root)) or "entry" + rid = f"request.{_slug(root_label)}" + if rid in used_ids: + rid = f"{rid}.{_slug(root)}" + used_ids.add(rid) + requests.append({ + "id": rid, + "kind": "call-path", + "description": f"Request lifecycle from {root_label} through " + f"{len(hops) - 1} call(s).", + "entry_node": root, + "hops": hops, + }) + # The architecture core draws from every kept story's functions, spine first. + for nid in [*chain_ids, *(f for f in functions if f not in chain_ids)]: + if len(core_ids) >= max_core: + break + if nid in core_ids: + continue + node = gl.nodes.get(nid) + if node is None: + continue + f, l = gl.loc(node)[0], gl.loc(node)[1] + if not f or not isinstance(l, int) or l <= 0: + continue + asm.add_node(_norm_node(gl, node), default_kind="function") + node_ids.add(nid) try: - return len(index.targets(node["id"], "CALLS")) + degree = sum(1 for _ in index.targets(nid, *_CALL_EDGE_KINDS)) except Exception: - return 0 - - pick = min(nxt, key=lambda n: (-out_degree(n), gl.label(n), n["id"])) - cur = pick["id"] - seen.add(cur) - chain.append(cur) - return chain + degree = 0 + core.append({"node_id": nid, "label": gl.label(node), + "file": f, "line": l, "degree": degree}) + core_ids.add(nid) + return requests, core def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, @@ -618,33 +845,20 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, "file": efile, "line": nline, }) - - # A guided path is only worth showing when it actually goes somewhere: - # the real CALLS chain out of the entry must have more than the entry. - chain = _call_chain(index, gl, handler_id, chain_depth) - if len(chain) < 2: - continue - hops = [] - for nid in chain: - cnode = gl.nodes.get(nid) - if cnode is None: - continue - asm.add_node(_norm_node(gl, cnode), default_kind="function") - hops.append({"node_id": nid, "caption": gl.label(cnode)}) - for a, b in zip(chain, chain[1:]): - asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, set(asm.nodes)) - if len(hops) >= 2: - requests.append({ - "id": f"request.{_slug(label)}", - "kind": "call-path", - "description": f"Follow control from {label} through " - f"{len(hops) - 1} call(s).", - "entry_node": handler_id, - "hops": hops, - }) except Exception: pass + # Guided request paths are no longer a greedy CALLS walk out of each exported + # symbol -- that surfaced leaf utilities (render_template) and never the request + # lifecycle. Instead root them at the real top-of-stack drivers and follow the + # success spine of each one's bounded execution story (wsgi __call__ -> wsgi_app + # -> full_dispatch_request -> dispatch_request -> ...). The same stories yield the + # architecture core, so both are built together below. + handler_ids = [entry["node_id"] for entry in entrypoints] + requests, core = _lifecycle_projection( + asm, index, gl, handler_ids, + max_requests=8, max_core=32, max_hops=max(2, chain_depth)) + files: list[dict] = [] try: seen_paths: set[str] = set() @@ -663,7 +877,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, concepts: list[dict] = [] try: architecture = comp.architecture_map(max_communities=8, max_files_per_community=20) - for index, community in enumerate(architecture.get("communities") or []): + for idx, community in enumerate(architecture.get("communities") or []): paths = [str(path) for path in community.get("files") or [] if path] if not paths: continue @@ -673,7 +887,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, directory = directory[4:] label = directory.replace("/", " · ") or first concepts.append({ - "id": f"concept.{_slug(community.get('id') or index)}", + "id": f"concept.{_slug(community.get('id') or idx)}", "label": label, "description": f"Connected code area spanning {len(paths)} file(s).", "file_paths": paths, @@ -681,54 +895,6 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: concepts = [] - # Add a small, source-backed architecture spine to the shared node pool. This - # is intentionally independent from security candidates: a newcomer needs the - # central control path even when no finding happens to touch it. Hubs ranks real - # call-graph declarations, and the walk below adds only real CALLS edges. - core: list[dict] = [] - try: - from lachesis.nav.hubs import Hubs - hubs = Hubs(gl, resolved_only=True).top(8) - core_ids: set[str] = set() - for hub in hubs: - hub_id = hub.get("node_id") - node = gl.nodes.get(hub_id) - if node is None: - continue - hub_file, hub_line, _ = gl.loc(node) - if (not hub_file or not isinstance(hub_line, int) or hub_line <= 0 - or re.search(r"(?:^|/)(?:tests?|examples?|fixtures?|benchmarks?|docs?)(?:/|$)", - str(hub_file), re.IGNORECASE)): - continue - chain = _call_chain(index, gl, hub_id, 4) - chain_nodes = [gl.nodes.get(nid) for nid in chain] - # Do not remove a middle hop and then connect its neighbors: that - # would turn a real multi-hop walk into an invented relationship. - if any( - cnode is None - or not gl.loc(cnode)[0] - or not isinstance(gl.loc(cnode)[1], int) - or gl.loc(cnode)[1] <= 0 - or re.search(r"(?:^|/)(?:tests?|examples?|fixtures?|benchmarks?|docs?)(?:/|$)", - str(gl.loc(cnode)[0]), re.IGNORECASE) - for cnode in chain_nodes - ): - chain = [hub_id] - chain_nodes = [node] - for nid, cnode in zip(chain, chain_nodes): - if len(core_ids) >= 32 or cnode is None: - break - asm.add_node(_norm_node(gl, cnode), default_kind="function") - core_ids.add(nid) - for a, b in zip(chain, chain[1:]): - asm.add_edge({"src": a, "tgt": b, "kind": "CALLS"}, set(asm.nodes)) - if hub_id in core_ids: - core.append({"node_id": hub_id, "label": gl.label(node), - "file": hub_file, "line": hub_line, - "degree": int(hub.get("degree") or 0)}) - except Exception: - core = [] - # Modules are not built here: they must partition the *final* included node # pool (one unambiguous module per node, keyed by that node's file), which is # only settled after candidate/capsule/entry nodes are all in and relativized. @@ -858,6 +1024,25 @@ def _enrich_graph_nodes(nodes: list[dict], gl) -> None: module = _dotted_module(node.get("file")) if module: node["qualified_name"] = f"{module}.{node.get('label')}" + # Scope: the enclosing callable every node lives in (problem #6 -- scope was + # absent on most nodes). owner_function returns the node itself when it is + # already a callable, so a function/method reports the module it belongs to, + # while an operand or a value reports the function that contains it. This is + # the container a frontend groups by, never an empty field. + try: + owner = gl.owner_function(twin) + except Exception: + owner = None + scope = None + if owner is not None and owner.get("id") != twin.get("id"): + owner_file, _os, _oe = gl.loc(owner) + owner_module = _dotted_module(owner_file) + owner_label = gl.label(owner) + scope = f"{owner_module}.{owner_label}" if owner_module else owner_label + if not scope: + scope = module + if scope: + node["scope"] = scope for key in ("documentation", "docstring", "comment"): try: documentation = gl.prop(twin, key) @@ -1112,7 +1297,19 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s language = str(lang or meta.get("lang") or "unknown") revision = str(commit or meta.get("commit") or "unknown") findings = bundle.get("findings") or [] + # `security.findings` stays exhaustive (passed through untouched below); only the + # *featured* value paths are cleaned. Two hygiene rules (problems #7 and #8): + # #7 A value path that visits fewer than two distinct nodes has not moved -- + # it is a bare def-use artifact (a traceback local `tb`, a file handle `f`, + # `config_file`, `tb.tb_frame`), not a behavior. Featuring it as one is the + # reported defect. Genuine flows (`hashlib.sha1`, `send_file`, `re.split`, + # `Markup`) always traverse a source and a distinct sink, so the two-distinct + # -node floor drops exactly the artifacts and keeps every real flow, including + # the minimal two-step call-argument flows. + # #8 Identical paths (same endpoints and same ordered node ids) are collapsed to + # one; the graph often yields the same def-use twice from different findings. values = [] + seen_paths: set[tuple] = set() for finding in findings: witness = finding.get("witness") or {} steps = witness.get("steps") or [] @@ -1121,14 +1318,23 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s finding_id = str(finding.get("finding_id") or "") if not finding_id: continue + step_ids = tuple(step.get("node_id") for step in steps) + if len({sid for sid in step_ids if sid}) < 2: + continue # #7: a path that never leaves one node is not a behavior + source_node = steps[0].get("node_id") + sink_node = steps[-1].get("node_id") + dedupe_key = (source_node, sink_node, step_ids) + if dedupe_key in seen_paths: + continue # #8: same endpoints and same ordered hops -- one is enough + seen_paths.add(dedupe_key) path_id = f"value:{finding_id}" values.append({ "id": path_id, "kind": "value-flow", "name": finding.get("display_name") or "value path", "description": finding.get("result_summary") or "Exporter-provided value path", - "source_node": steps[0].get("node_id"), - "sink_node": steps[-1].get("node_id"), + "source_node": source_node, + "sink_node": sink_node, "confidence": (finding.get("analysis") or {}).get("confidence"), "limitations": list((finding.get("analysis") or {}).get("limitations") or []), "steps": steps, From 6ae6a91499d01ee167a44d099cfe4616a0d62cd3 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 11:28:09 +0530 Subject: [PATCH 13/19] nav(bundle): gate test/example scaffolding out of the export projection Build-time exclusion keeps scaffolding out of the graph, but the exporter must not depend on it. Run against a graph built without exclusion, the comprehension projection refeatured exactly the files the classifier drops: on full Flask 39 of 40 entrypoints, 22 of 32 core nodes and all 8 request roots were test_* or example functions. The cause is structural -- an uncalled test function is an in-degree-0 callable, so _lifecycle_roots ranked it as a top-of-stack driver, and the entry-anchor pass surfaced test handlers directly. Gate the three featured surfaces with the same classifier the build filter uses (lachesis.config.is_nonproduct), so the two always agree: skip non-product handlers when building entrypoints, skip non-product entry handlers and in-degree-0 drivers when choosing lifecycle roots, and keep non-product nodes off the architecture core. The import is guarded and the gate fails open (keeps the node) if the classifier is ever unavailable; is_nonproduct is pure-stdlib so this does not add a runtime dependency. Correctness. security.findings and the value paths are untouched -- only the comprehension surfaces are gated. Re-exporting full (unexcluded) Flask now shows 0 non-product entrypoints/core/request-roots (roots: __call__, make_response, shell_command, ...), and the already-excluded graph is unchanged (identical counts). nav/test_bundle.py, test_config.py and test_pipeline_inventory.py pass (71 tests); the suite's hand-built findings carry no file, so the gate is inert there by construction. --- lachesis/nav/bundle.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 5225b0b3..ada53d27 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -37,6 +37,11 @@ from lachesis.nav import mcp_server as M +try: + from lachesis.config import is_nonproduct as _is_nonproduct +except Exception: # config is pure-stdlib and same-package, so this should not fail; + _is_nonproduct = None # if it ever does, the gate fails open (keeps everything). + BUNDLE_VERSION = "1.0" FINDING_SCHEMA_VERSION = "0.1" _HEX64 = 64 @@ -46,6 +51,26 @@ def _call(name: str, args: dict) -> Any: return json.loads(M.call_tool(name, args, "json")) +def _is_nonproduct_path(path: Optional[str]) -> bool: + """True when a source path is test/example/docs/benchmark scaffolding. + + The featured comprehension surfaces (entrypoints, request roots, the core spine) + describe what the *product* does, so scaffolding must never seed them. Build-time + exclusion normally keeps such files out of the graph entirely, but the exporter + must not rely on that -- run against a graph built without exclusion, an uncalled + ``test_*`` function is an in-degree-0 callable and would otherwise rank as a + top-of-stack driver, refeaturing exactly the tests the classifier is meant to + drop. Reuses the same classifier the build filter uses, so the two agree; fails + open (keeps the node) only if the classifier is somehow unavailable. + """ + if not path or _is_nonproduct is None: + return False + try: + return bool(_is_nonproduct(path)) + except Exception: + return False + + # --------------------------------------------------------------------- identity def _basename(path: Optional[str]) -> str: @@ -589,6 +614,9 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int) -> list[str for hid in handler_ids: if hid and hid not in seen: seen.add(hid) + node = gl.nodes.get(hid) + if node is not None and _is_nonproduct_path(gl.loc(node)[0]): + continue # a test/example handler is not a product lifecycle root roots.append(hid) drivers: list[tuple[int, str]] = [] @@ -603,6 +631,10 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int) -> list[str f, l = gl.loc(node)[0], gl.loc(node)[1] if not f or not isinstance(l, int) or l <= 0: continue + # An uncalled test_* function is in-degree-0; exclude scaffolding so it never + # ranks as a top-of-stack driver on a graph built without build-time exclusion. + if _is_nonproduct_path(f): + continue try: out = sum(1 for _ in index.targets(nid, *_CALL_EDGE_KINDS)) if out < 1: @@ -772,6 +804,8 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], f, l = gl.loc(node)[0], gl.loc(node)[1] if not f or not isinstance(l, int) or l <= 0: continue + if _is_nonproduct_path(f): + continue # keep scaffolding off the architecture core asm.add_node(_norm_node(gl, node), default_kind="function") node_ids.add(nid) try: @@ -826,6 +860,9 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, # file and line, or it is not a place a developer can actually begin. if not nfile or not isinstance(nline, int) or nline <= 0: continue + # ...and it must be product code -- never a test/example handler. + if _is_nonproduct_path(nfile): + continue asm.add_node(_norm_node(gl, node), default_kind="function") how = anchor.get("how") label = gl.label(node) From da98724e92f85329f6eae08ace52ec14738295fd Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 12:33:21 +0530 Subject: [PATCH 14/19] nav(bundle): feature the primary language and gate vendored/generated code A multi-codebase review (json5, django, suricata) surfaced three ways the comprehension projection featured the wrong code. All three share one root cause: the non-product path gate the export already applied to entrypoints and lifecycle roots was neither complete nor applied everywhere it needed to be. 1. Multi-language repos featured the wrong language. On django (Python) every one of the 40 entrypoints and all 8 request flows were bundled JavaScript admin widgets under contrib/admin/static: the planner treats JS event handlers as entry handlers, and an uncalled JS handler is in-degree-0 so it also ranks as a lifecycle driver. A newcomer saw a JS widget toolkit instead of the WSGI -> middleware -> resolver -> view path. Fix: derive the repo's dominant product-source language family (_primary_language_family, a strict majority over non-scaffolding source files, else None = polyglot = no gate) and gate entrypoint selection and both lifecycle-root sources to it, so the projection stays in the language the repo actually is. 2. Vendored deps, generated output and build configs leaked. node_modules the TypeScript compiler lib, rollup.config.js and *.d.ts surfaced as concepts and nodes even in Python bundles. The classifier (config.is_nonproduct) is the default build- and export-time filter, so it is the right place: it now also drops dist/ and scripts/ segments and *.min.*, *.d.ts and *.config.js basenames. It stays language-agnostic (path-only), so one rule covers Python/TS/JS/C, and it applies at build time too -- a graph over an application has no business modelling a dependency it cannot patch, for comprehension or for security triage. Recoverable via build.include. 3. modules/concepts leaked tests/docs/examples. _partition_modules and _project_concepts never consulted the gate, so suricata surfaced modules tests.fuzz.*, doc.userguide.convert and concepts scripts, doc . userguide. Both now apply _is_nonproduct_path, and the architecture-concept builder filters non-product files *before* deriving a community's label -- so a community mixing json5's lib/*.js with the vendored compiler is labelled "lib", not "arachne . node_modules . typescript . lib". Verified by re-exporting the unexcluded json5 graph: entrypoints 4/4 product JS, modules the 6 real product modules, the single concept now "lib". config and bundle unit suites green (73 tests); added classifier cases for the new vendored/generated/build-config vocabulary and its over-match guards (reader.ts, lib/config.js, src/distributed/, src/scripting/ all kept). A fourth review item -- documentation is 0% on every bundle -- is a producer gap (no frontend emits a docstring/comment property; the reader is correct against data that is not there) and is handled separately. --- lachesis/config.py | 26 ++++++++- lachesis/nav/bundle.py | 118 +++++++++++++++++++++++++++++++++++++--- lachesis/test_config.py | 18 ++++++ 3 files changed, 152 insertions(+), 10 deletions(-) diff --git a/lachesis/config.py b/lachesis/config.py index 0299d5ed..d8d26859 100644 --- a/lachesis/config.py +++ b/lachesis/config.py @@ -45,12 +45,27 @@ # The built-in "non-product" exclusion set. A path is non-product when any of its # directory segments is one of these scaffolding names, or when its basename is a -# test module. Matching is on path *segments* and basenames, never on substrings, -# so product files whose name merely contains a keyword — ``testing.py``, +# test module, a vendored dependency, a generated/minified artifact, or a build +# config. Matching is on path *segments* and basenames, never on substrings, so +# product files whose name merely contains a keyword — ``testing.py``, # ``templating.py``, ``documentation.py`` — are kept. Validated against the pallets # /flask tree: of 83 .py files it drops exactly the 41 tests + 17 examples + 1 doc # and keeps all 24 ``src/flask`` modules. # +# The set is deliberately language-agnostic — it classifies *paths*, so one rule +# covers Python, TypeScript, JavaScript and C at once. It is the same default a +# review confirmed the graph itself must honor (not only the bundle export): a +# code-property graph over an application has no business modelling a vendored +# dependency, a build output tree, or a generated bundle — a bug found inside one +# is not actionable in the analysed repo, and the noise drowns the product signal +# for comprehension and for security triage alike. Dependencies (``node_modules``, +# ``vendor``, ``third_party``, ``site-packages``), build/CI tooling (``scripts``), +# generated output (``dist``), minified/declaration/build-config artifacts +# (``*.min.js``, ``*.d.ts``, ``*.config.js``) are therefore all dropped by default. +# Any of it is recoverable with an explicit ``build.include`` allow-list or by +# clearing ``build.exclude`` — the opt-in escape hatch for deliberately auditing a +# dependency. +# # Kept as a single compiled regex (rather than a glob list) because it is the # default applied on every build and both the builder and the exporter consult # it; the user-facing ``build.exclude`` list is expressed as globs and compiled @@ -63,11 +78,18 @@ r"fixtures?", r"testdata", r"test[_-]?data", r"__mocks__", r"mocks", r"vendor", r"vendored", r"third[_-]?party", r"node_modules", r"site-packages", r"\.tox", r"\.nox", r"\.venv", r"venv", + r"dist", r"scripts", ) _NONPRODUCT_BASENAMES = ( r"conftest\.py", r"test_[^/]*\.py", r"[^/]*_test\.py", r"[^/]*\.spec\.[a-z]+", r"[^/]*\.test\.[a-z]+", + # Generated / vendored / build-config artifacts: a minified bundle, a + # TypeScript declaration file, and the common ``*.config.js`` build configs + # (rollup/webpack/vite/babel/jest/…) are outputs and scaffolding, not source. + r"[^/]*\.min\.[a-z0-9]+", + r"[^/]*\.d\.ts", + r"[^/]*\.config\.(?:js|cjs|mjs|ts)", ) NONPRODUCT_RE = re.compile( r"(?:^|/)(?:" + "|".join(_NONPRODUCT_SEGMENTS) + r")(?:/|$)" diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index ada53d27..b84ac36d 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -598,7 +598,63 @@ def _reach2(index, node_id: str) -> int: return len(total) -def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int) -> list[str]: +# Source-extension -> coarse language family. JavaScript and TypeScript are one +# family (the same web frontend, the same event/handler surface); the C headers +# and sources are one family. Used only to answer "what language is this repo +# primarily", so the grouping is deliberately coarse. +_LANG_BY_EXT = { + "py": "python", "pyi": "python", "pyx": "python", + "js": "web", "jsx": "web", "mjs": "web", "cjs": "web", + "ts": "web", "tsx": "web", "mts": "web", "cts": "web", + "c": "c", "h": "c", "cc": "c", "cpp": "c", "cxx": "c", + "hpp": "c", "hh": "c", "hxx": "c", +} + + +def _language_family(path: Optional[str]) -> Optional[str]: + """The coarse language family of a source path, or None if unrecognised.""" + if not isinstance(path, str): + return None + base = path.replace("\\", "/").rsplit("/", 1)[-1] + if "." not in base: + return None + return _LANG_BY_EXT.get(base.rsplit(".", 1)[-1].lower()) + + +def _primary_language_family(index, gl) -> Optional[str]: + """The dominant product-source language family of the graph, or None. + + Counts product (non-scaffolding) source files by family and returns the family + that is a strict majority. A repo with no clear majority — a genuinely polyglot + tree — returns None, which disables the language gate so nothing is dropped. + + This is the notion the entrypoint and request-lifecycle selection was missing: + on a multi-language repository (a Python framework that ships bundled JavaScript + admin widgets under ``static/``) the JS event handlers otherwise fill every + featured slot, so a newcomer sees a JS widget toolkit instead of the Python + request path. The gate keeps the projection in the language the repo actually is. + """ + counts: dict[str, int] = {} + try: + for node in index.nodes_of_kind("file"): + f = gl.loc(node)[0] or gl.prop(node, "file") + if not f or _is_nonproduct_path(f): + continue + fam = _language_family(f) + if fam: + counts[fam] = counts.get(fam, 0) + 1 + except Exception: + return None + if not counts: + return None + top = max(counts, key=lambda k: counts[k]) + if counts[top] * 2 <= sum(counts.values()): + return None # no strict majority -> polyglot -> do not gate + return top + + +def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int, + primary_family: Optional[str] = None) -> list[str]: """Candidate roots for request-lifecycle stories, best driver first. Two sources, deduped in priority order: the planner's entry handlers (already @@ -617,6 +673,10 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int) -> list[str node = gl.nodes.get(hid) if node is not None and _is_nonproduct_path(gl.loc(node)[0]): continue # a test/example handler is not a product lifecycle root + if node is not None and primary_family is not None: + fam = _language_family(gl.loc(node)[0]) + if fam is not None and fam != primary_family: + continue # a non-primary-language handler (bundled JS in a Python repo) roots.append(hid) drivers: list[tuple[int, str]] = [] @@ -635,6 +695,12 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int) -> list[str # ranks as a top-of-stack driver on a graph built without build-time exclusion. if _is_nonproduct_path(f): continue + # ...and never a non-primary-language driver: a bundled JS handler in a + # Python repo is in-degree-0 too, but it is not this repo's request path. + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + continue try: out = sum(1 for _ in index.targets(nid, *_CALL_EDGE_KINDS)) if out < 1: @@ -718,7 +784,8 @@ def subtree(nid: str, guard: frozenset) -> int: def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], *, max_requests: int, max_core: int, - max_hops: int) -> tuple[list[dict], list[dict]]: + max_hops: int, + primary_family: Optional[str] = None) -> tuple[list[dict], list[dict]]: """Request lifecycles and the architecture core, from bounded execution stories. Runs a bounded forward execution story from each candidate driver (see @@ -733,7 +800,8 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], requests: list[dict] = [] core: list[dict] = [] try: - roots = _lifecycle_roots(index, gl, handler_ids, cap=30) + roots = _lifecycle_roots(index, gl, handler_ids, cap=30, + primary_family=primary_family) except Exception: return requests, core @@ -837,6 +905,12 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: return empty + # The repo's dominant product language. A multi-language tree (a Python framework + # that bundles JavaScript admin widgets) otherwise features the wrong language: + # its JS event handlers rank as entrypoints and fill every request flow. Gating to + # the primary family keeps the projection in the language the repo actually is. + primary_family = _primary_language_family(index, gl) + entrypoints: list[dict] = [] requests: list[dict] = [] try: @@ -863,6 +937,12 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, # ...and it must be product code -- never a test/example handler. if _is_nonproduct_path(nfile): continue + # ...and in the repo's primary language -- never a bundled JS admin + # widget standing in for the request path of a Python framework. + if primary_family is not None: + fam = _language_family(nfile) + if fam is not None and fam != primary_family: + continue asm.add_node(_norm_node(gl, node), default_kind="function") how = anchor.get("how") label = gl.label(node) @@ -894,7 +974,8 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, handler_ids = [entry["node_id"] for entry in entrypoints] requests, core = _lifecycle_projection( asm, index, gl, handler_ids, - max_requests=8, max_core=32, max_hops=max(2, chain_depth)) + max_requests=8, max_core=32, max_hops=max(2, chain_depth), + primary_family=primary_family) files: list[dict] = [] try: @@ -915,7 +996,12 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, try: architecture = comp.architecture_map(max_communities=8, max_files_per_community=20) for idx, community in enumerate(architecture.get("communities") or []): - paths = [str(path) for path in community.get("files") or [] if path] + # Filter non-product files *before* deriving the label and file set: a + # community that mixes json5's ``lib/*.js`` with the vendored TypeScript + # compiler under ``node_modules`` must be labelled ``lib``, not + # ``… · node_modules · typescript · lib`` off the first (vendored) path. + paths = [str(path) for path in community.get("files") or [] + if path and not _is_nonproduct_path(str(path))] if not paths: continue first = paths[0] @@ -1206,6 +1292,11 @@ def _partition_modules(nodes: list[dict], entrypoints: list[dict]) -> list[dict] f = node.get("file") if not isinstance(f, str) or not f.strip(): continue + # Non-product files (tests, docs, examples, vendored deps, generated output) + # must not surface as modules a reader is invited to explore. The same gate the + # entrypoint/request selection uses, applied to the module partition. + if _is_nonproduct_path(f): + continue module_name = _dotted_module(f) or f node["module"] = module_name groups.setdefault(f, []).append(node["id"]) @@ -1227,12 +1318,23 @@ def _partition_modules(nodes: list[dict], entrypoints: list[dict]) -> list[dict] def _project_concepts(raw_concepts: list[dict], nodes: list[dict]) -> list[dict]: - """Keep architecture concepts honest to the final included node pool.""" + """Keep architecture concepts honest to the final included node pool. + + A concept is dropped entirely when every file it spans is non-product, and its + node set is restricted to product files, so a vendored dependency + (``node_modules · typescript · lib``), a build config (``rollup.config.js``), or a + docs/scripts tree never surfaces as an architecture concept — even on a graph + built without build-time exclusion. + """ out: list[dict] = [] for concept in raw_concepts or []: - paths = {str(path) for path in concept.get("file_paths") or [] if path} + paths = {str(path) for path in concept.get("file_paths") or [] if path + and not _is_nonproduct_path(str(path))} + if not paths: + continue node_ids = [node["id"] for node in nodes - if isinstance(node.get("file"), str) and node.get("file") in paths] + if isinstance(node.get("file"), str) and node.get("file") in paths + and not _is_nonproduct_path(node.get("file"))] if not node_ids: continue out.append({ diff --git a/lachesis/test_config.py b/lachesis/test_config.py index 812e2c6a..6afb3058 100644 --- a/lachesis/test_config.py +++ b/lachesis/test_config.py @@ -31,6 +31,24 @@ def test_drops_scaffolding_by_segment_and_basename(self): "src/foo_test.py", "a/thing.spec.ts", "a/thing.test.js"): self.assertTrue(config.is_nonproduct(path), path) + def test_drops_vendored_generated_and_build_config(self): + # Dependencies, generated output, and build configs are not product source; + # a graph over an application has no business modelling them. Language-agnostic, + # so one rule covers Python/TS/JS/C at once. + for path in ("dist/bundle.js", "scripts/check-dist-rules.py", + "lib/parser.min.js", "assets/app.min.css", + "lib/stringify.d.ts", "types/index.d.ts", + "rollup.config.js", "webpack.config.ts", "vite.config.mjs", + "jest.config.cjs"): + self.assertTrue(config.is_nonproduct(path), path) + + def test_generated_patterns_do_not_over_match_product(self): + # A normal ``.ts`` module, a bare ``config.js`` (no ``.config.js`` + # shape), and a product module under ``distributed/`` must all survive. + for path in ("src/reader.ts", "lib/config.js", "src/distributed/queue.py", + "src/scripting/engine.py"): + self.assertFalse(config.is_nonproduct(path), path) + class GlobTests(unittest.TestCase): def test_bare_segment_matches_anywhere(self): From 25a4720ec9f5df4a74a775c6105c0c423259f711 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 12:46:01 +0530 Subject: [PATCH 15/19] nav(bundle): normalize node locations and enforce the code-understanding contract Two exporter defects surfaced across the Flask/Click/ItsDangerous 2.0 bundles. Synthetic nodes (heap locations, summary objects) carried no source and were exported as {file: null, line: null}. The 2.0 contract is a concrete absence -- file "" and line 0 -- so a reader can uniformly test line > 0 for openability instead of handling a third "unknown type" state. _normalize_node_location now coerces every node's file to a string (""), line to a non-negative int (0), and makes end_line mandatory and never less than line (a single-line span when no wider extent is known, 0 for synthetics). This is behaviour-preserving for path and entrypoint selection: _has_source already rejects an empty file and a non-positive line, so the nodes that were non-source-backed as null stay non-source-backed as ""/0 -- only the field's type is pinned. Verified on the json5 export: 20 synthetic nodes now report ""/0/0, zero nulls, zero inverted spans, and the featured entrypoints and request hops are unchanged. The validator now enforces those types on every node (file is str, line and end_line are ints, line >= 0, end_line >= line) rather than allowing the null leak through. It also enforces the projection contract a code-understanding consumer relies on: at least one production entrypoint (the ItsDangerous export shipped entrypoints: [], which is a hollow projection) and at least one source-backed path of three or more hops. Request hops are already proven source-backed, so a >=3-hop request satisfies the path requirement by construction. django (40 entrypoints, 6-hop paths) and json5 (4 entrypoints, 5-hop paths) both satisfy the contract; the test fixtures that built minimal bundles were updated to supply a contract-valid base. All 73 nav/config/pipeline tests pass. --- lachesis/nav/bundle.py | 55 ++++++++++++++++++++++++++++++ lachesis/nav/test_bundle.py | 67 +++++++++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 14 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index b84ac36d..1bee4b14 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -1419,6 +1419,28 @@ def _project_curated_tour(raw: Optional[dict], values: list[dict], requests: lis return result +def _normalize_node_location(node: dict) -> None: + """Coerce a node's ``file``/``line``/``end_line`` to their 2.0 field types in place. + + Synthetic nodes (heap locations, summary objects) have no source and were + emitting ``file: null, line: null``; the 2.0 contract is ``file: ""`` and + ``line: 0`` -- a real absence, not a missing key of unknown type -- so a reader + can uniformly test ``line > 0`` for openability. ``end_line`` is made mandatory + and never less than ``line`` (a single-line span when no wider extent is known, + ``0`` for synthetics). Normalizing null to ""/0 does not change which nodes count + as source-backed: ``_has_source`` already rejects an empty file and a non-positive + line, so featured-path and entrypoint selection are unaffected. + """ + file = node.get("file") + node["file"] = file if isinstance(file, str) and file.strip() else "" + line = node.get("line") + line = line if isinstance(line, int) and not isinstance(line, bool) and line > 0 else 0 + node["line"] = line + end = node.get("end_line") + node["end_line"] = end if (isinstance(end, int) and not isinstance(end, bool) + and end >= line) else line + + def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[str], lang: Optional[str], indexed_nodes: int, source_url_template: Optional[str] = None, @@ -1481,6 +1503,8 @@ def _graph_first_bundle(bundle: dict, *, repo: Optional[str], commit: Optional[s graph = bundle.get("graph") or {} nodes = graph.get("nodes") or [] + for node in nodes: + _normalize_node_location(node) node_map = {n.get("id"): n for n in nodes} node_ids = set(node_map) edges = _canonical_edges(graph.get("edges") or [], node_ids) @@ -1566,6 +1590,22 @@ def _validate_graph_first(bundle: dict) -> None: if not nodes or None in node_ids: raise ValueError("graph-first bundle has invalid nodes") + # Every node carries the concrete 2.0 location types -- ``file`` a string + # (``""`` when absent), ``line`` and ``end_line`` non-negative ints with the + # span never inverted. Synthetic nodes (heap locations) legitimately report + # ``""``/``0``; what is rejected is the earlier ``null`` leak, which left the + # field's type undefined for consumers. + for node in nodes: + nid = node.get("id") + if not isinstance(node.get("file"), str): + raise ValueError(f"node {nid} file must be a string") + line = node.get("line") + if not isinstance(line, int) or isinstance(line, bool) or line < 0: + raise ValueError(f"node {nid} line must be an int >= 0") + end = node.get("end_line") + if not isinstance(end, int) or isinstance(end, bool) or end < line: + raise ValueError(f"node {nid} end_line must be an int >= line") + coverage = graph.get("coverage") or {} if coverage and coverage.get("included_nodes") != len(nodes): raise ValueError("graph-first coverage.included_nodes must equal node count") @@ -1581,6 +1621,21 @@ def _validate_graph_first(bundle: dict) -> None: if not _has_source(node_map.get(nid)): raise ValueError(f"entrypoint {entry.get('id')} node has no openable source") + # A comprehension-first projection is meaningless without a boundary to enter + # from and a path with enough hops to be a story. An empty entrypoint set (the + # ItsDangerous case) or paths that never exceed a bare def-use pair defeat the + # whole projection, so they are rejected here rather than shipped as a hollow + # bundle. Request hops are already proven source-backed above, so a >=3-hop + # request is a source-backed path of three or more hops by construction. + if bundle.get("analysis_projection") == "code-understanding": + if not (graph.get("entrypoints") or []): + raise ValueError( + "code-understanding projection requires at least one production entrypoint") + requests = (bundle.get("paths") or {}).get("requests") or [] + if not any(len(req.get("hops") or []) >= 3 for req in requests): + raise ValueError( + "code-understanding projection requires a source-backed path of >= 3 hops") + seen_module_nodes: set[str] = set() for module in graph.get("modules") or []: for nid in module.get("node_ids") or []: diff --git a/lachesis/nav/test_bundle.py b/lachesis/nav/test_bundle.py index 1220a358..523a1aff 100644 --- a/lachesis/nav/test_bundle.py +++ b/lachesis/nav/test_bundle.py @@ -224,6 +224,19 @@ def test_edge_referencing_unknown_node_rejected(self): class GraphFirstBundleTests(unittest.TestCase): + def _comprehension(self): + # A code-understanding bundle must carry a production entrypoint and a + # >= 3-hop source-backed path; the legacy nodes (source@3, sink@8) back both. + return { + "entrypoints": [{"id": "entry.source", "label": "input", "kind": "parameter", + "node_id": "source", "file": "src/a.c", "line": 3}], + "requests": [{"id": "request.flow", "kind": "call-path", "description": "d", + "entry_node": "source", + "hops": [{"node_id": "source", "caption": "receives"}, + {"node_id": "sink", "caption": "executes"}, + {"node_id": "source", "caption": "returns"}]}], + } + def _legacy_bundle(self): return { "meta": {"repo": "GNOME/libxml2", "lang": "c", "commit": "abc", "loc": 42}, @@ -251,6 +264,7 @@ def _legacy_bundle(self): def test_graph_first_uses_v2_contract_and_supported_source_placeholders(self): result = bundle._graph_first_bundle( self._legacy_bundle(), repo="GNOME/libxml2", commit="abc", lang="c", indexed_nodes=99, + comprehension=self._comprehension(), source_url_template="https://github.com/GNOME/libxml2/blob/{revision}/{file}#L{line}") self.assertEqual(result["schema_version"], "2.0") self.assertEqual(result["meta"]["indexed_nodes"], 99) @@ -261,7 +275,8 @@ def test_graph_first_uses_v2_contract_and_supported_source_placeholders(self): def test_graph_first_does_not_guess_source_host(self): result = bundle._graph_first_bundle( - self._legacy_bundle(), repo="group/project", commit="abc", lang="c", indexed_nodes=2) + self._legacy_bundle(), repo="group/project", commit="abc", lang="c", indexed_nodes=2, + comprehension=self._comprehension()) self.assertNotIn("source_url_template", result["meta"]) def test_graph_first_rejects_invalid_path_reference(self): @@ -279,6 +294,27 @@ def _sourced(self, nid, file, line, label): return {"id": nid, "kind": "function", "file": file, "line": line, "label": label, "snippet": f"def {label}(): ...", "end_line": line + 2} + # A valid code-understanding bundle always has a production entrypoint and a + # source-backed path of >= 3 hops (the NR2 contract). The helpers inject these + # defaults so a test focused on some other facet (concepts, tour, module dup) + # still builds a contract-valid base; a test provides its own to override. + def _default_entrypoints(self): + return [{"id": "entry.wsgi_app", "label": "wsgi_app", "kind": "http-handler", + "node_id": "n.a", "file": "src/flask/app.py", "line": 10}] + + def _default_requests(self): + return [{"id": "request.baseline", "kind": "call-path", "description": "baseline", + "entry_node": "n.a", + "hops": [{"node_id": "n.a", "caption": "receives"}, + {"node_id": "n.b", "caption": "dispatches"}, + {"node_id": "n.a", "caption": "returns"}]}] + + def _with_defaults(self, comprehension): + comp = dict(comprehension) + comp.setdefault("entrypoints", self._default_entrypoints()) + comp.setdefault("requests", self._default_requests()) + return comp + def _bundle_with(self, comprehension): legacy = { "meta": {"repo": "pallets/flask", "lang": "python", "commit": "abc", "loc": 100}, @@ -299,7 +335,7 @@ def _bundle_with(self, comprehension): } return bundle._graph_first_bundle( legacy, repo="pallets/flask", commit="abc", lang="python", - indexed_nodes=500, comprehension=comprehension) + indexed_nodes=500, comprehension=self._with_defaults(comprehension)) def _bundle_with_tour(self, comprehension, curated_tour): legacy = { @@ -316,7 +352,7 @@ def _bundle_with_tour(self, comprehension, curated_tour): } return bundle._graph_first_bundle( legacy, repo="pallets/flask", commit="abc", lang="python", indexed_nodes=500, - comprehension=comprehension, curated_tour=curated_tour) + comprehension=self._with_defaults(comprehension), curated_tour=curated_tour) def test_full_projection_shapes_graph_and_paths(self): result = self._bundle_with({ @@ -326,7 +362,8 @@ def test_full_projection_shapes_graph_and_paths(self): "requests": [{"id": "request.lifecycle", "kind": "call-path", "description": "d", "entry_node": "n.a", "hops": [{"node_id": "n.a", "caption": "receives"}, - {"node_id": "n.b", "caption": "dispatches"}]}], + {"node_id": "n.b", "caption": "dispatches"}, + {"node_id": "n.c", "caption": "runs"}]}], "files": [{"id": "f1", "path": "src/flask/app.py"}], "concepts": [{"id": "concept.flask", "label": "flask", "description": "d", "file_paths": ["src/flask/app.py"]}], @@ -340,7 +377,7 @@ def test_full_projection_shapes_graph_and_paths(self): self.assertEqual(len(result["graph"]["entrypoints"]), 1) req = result["paths"]["requests"][0] self.assertEqual(req["source_node"], "n.a") - self.assertEqual(req["sink_node"], "n.b") + self.assertEqual(req["sink_node"], "n.c") self.assertEqual(req["hops"][0]["id"], "request.lifecycle:01") self.assertEqual(req["hops"][1]["edge_label"], "calls") # edges are first-class: id + canonical kind + relation alias. @@ -358,7 +395,6 @@ def test_full_projection_shapes_graph_and_paths(self): def test_concept_without_included_nodes_is_dropped(self): result = self._bundle_with({ - "entrypoints": [], "requests": [], "concepts": [{"id": "concept.missing", "label": "missing", "file_paths": ["src/other/missing.py"]}], }) @@ -366,10 +402,11 @@ def test_concept_without_included_nodes_is_dropped(self): def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): result = self._bundle_with_tour( - {"entrypoints": [], "concepts": [{"id": "concept.lifecycle", "label": "Lifecycle", "description": "d", "file_paths": ["src/flask/app.py"]}], "requests": [{"id": "request.lifecycle", "kind": "call-path", + {"concepts": [{"id": "concept.lifecycle", "label": "Lifecycle", "description": "d", "file_paths": ["src/flask/app.py"]}], "requests": [{"id": "request.lifecycle", "kind": "call-path", "description": "d", "entry_node": "n.a", "hops": [{"node_id": "n.a", "caption": "a"}, - {"node_id": "n.b", "caption": "b"}]}]}, + {"node_id": "n.b", "caption": "b"}, + {"node_id": "n.a", "caption": "c"}]}]}, {"id": "tour.start", "title": "Start here", "description": "Read this first.", "maintainer": {"name": "Ignored"}, "overview": {"description": "Read the request lifecycle first.", "concepts": [{"id": "concept.lifecycle", "label": "Request lifecycle", "description": "The main request route."}]}, @@ -386,7 +423,7 @@ def test_curated_tour_keeps_current_paths_and_drops_stale_steps(self): def test_curated_tour_is_omitted_when_no_step_resolves(self): result = self._bundle_with_tour( - {"entrypoints": [], "requests": []}, + {}, {"id": "tour.start", "title": "Start here", "steps": [{"flow_id": "missing"}]}, ) self.assertNotIn("curated_tour", result["meta"]) @@ -394,7 +431,6 @@ def test_curated_tour_is_omitted_when_no_step_resolves(self): def test_request_with_unsourced_hop_is_dropped(self): # n.ghost has no source; the guided path must not be emitted. result = self._bundle_with({ - "entrypoints": [], "requests": [{"id": "r", "kind": "call-path", "description": "d", "entry_node": "n.a", "hops": [{"node_id": "n.a", "caption": "a"}, @@ -405,22 +441,25 @@ def test_request_with_unsourced_hop_is_dropped(self): self.assertEqual(len(result["paths"]["requests"]), 1) def test_validator_rejects_coverage_mismatch(self): - result = self._bundle_with({"entrypoints": [], "requests": []}) + result = self._bundle_with({}) result["graph"]["coverage"]["included_nodes"] += 1 with self.assertRaises(ValueError): bundle._validate_graph_first(result) def test_validator_rejects_entrypoint_without_source(self): - result = self._bundle_with({"entrypoints": [], "requests": []}) + result = self._bundle_with({}) + # Valid 2.0 location types but no openable source ("" / 0): the entrypoint + # check must still reject it for lacking a real location. result["graph"]["nodes"].append({"id": "n.bare", "kind": "function", - "label": "bare"}) + "label": "bare", "file": "", "line": 0, + "end_line": 0}) result["graph"]["coverage"]["included_nodes"] = len(result["graph"]["nodes"]) result["graph"]["entrypoints"].append({"id": "entry.bare", "node_id": "n.bare"}) with self.assertRaises(ValueError): bundle._validate_graph_first(result) def test_validator_rejects_duplicate_module_node(self): - result = self._bundle_with({"entrypoints": [], "requests": []}) + result = self._bundle_with({}) result["graph"]["modules"].append( {"id": "module.dup", "name": "dup", "path": "x", "node_ids": ["n.a"]}) with self.assertRaises(ValueError): From 98162abede4b436c7c0f073657278e92103b9b55 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 13:06:02 +0530 Subject: [PATCH 16/19] nav(bundle): root request lifecycles at the dispatcher, not the trampoline The code-understanding projection mis-drew a framework's request path. On the real Flask library graph the guided lifecycle came out `__call__ -> wsgi_app -> session deserialize / JSON dumps`, wsgi_app was not an entrypoint at all, and full_dispatch_request / dispatch_request -- the two hops a reader most needs -- were nowhere on the spine. Three causes, three fixes, all in the projection's node *selection* (ids, content-hashes, and edge keys are untouched; only which nodes the 2.0 bundle features changes): 1. `_story_spine` picked the deepest-subtree callee at each hop, which for a library rewards in-repo duck-typed detours: the true dispatch chain dead-ends at the external view, while a session-deserialize or json-dump that only *might* run keeps fanning out inside the library and looks bigger. But those detours are all `via=indirect:may_invoke` edges, whereas the real spine is `via=direct`. The pick key now prefers a direct CALLS edge first, then deranks error/teardown names, then breaks ties on subtree size -- so the walk stays on wsgi_app -> full_dispatch_request -> dispatch_request. 2. An in-degree-0 driver is often a thin trampoline (`Flask.__call__` is one line: `return self.wsgi_app(...)`). New `_descend_trampoline` follows a chain of single product-callee direct forwarders, bounded and cycle-guarded, so `_lifecycle_roots` yields wsgi_app rather than __call__ -- both shortening the path and, crucially, making the dispatcher the request's entry node. 3. `EntryPoints.by_handler()` only recognises module-level public helpers, so a framework's actual request driver (a WSGI wsgi_app, an event loop) was never an anchored entrypoint. `_comprehension_projection` now promotes each lifecycle root to a `request-lifecycle` entrypoint, leading the list. This also gives a library with no anchored handler (itsdangerous) a real source-backed entrypoint, which the code-understanding contract now requires. This is an intentional behavior change to the projection, not a refactor. Verified by re-exporting three graphs: flasklib now leads with `[request-lifecycle] wsgi_app` and a spine wsgi_app -> full_dispatch_request -> dispatch_request -> ...; django (40 Python entrypoints) and json5 (7 promoted lib/parse.js lifecycles) export with the validator passing and no null/typed-field regressions. All 73 nav/config/ pipeline tests pass. --- lachesis/nav/bundle.py | 114 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 105 insertions(+), 9 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index 1bee4b14..dcbd9866 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -653,6 +653,45 @@ def _primary_language_family(index, gl) -> Optional[str]: return top +def _descend_trampoline(index, gl, nid: str, *, + primary_family: Optional[str] = None, limit: int = 4) -> str: + """Skip thin forwarders so a lifecycle root is the real orchestrator. + + A WSGI ``Flask.__call__`` is a one-line trampoline: ``return self.wsgi_app(...)``. + Rooting the story at it prepends a meaningless hop and, worse, makes the *entry* + of the request the trampoline rather than the dispatcher a reader wants named. + While the current node forwards to exactly one product callee of the repo's + dominant language (a single direct CALLS target), descend to it. Bounded by + ``limit`` and a ``seen`` set so a mutually-recursive pair can never loop. + """ + seen = {nid} + for _ in range(limit): + try: + callees = [t.get("id") for t in index.targets(nid, "CALLS") if t.get("id")] + except Exception: + return nid + openable = [] + for cid in callees: + if cid in seen: + continue + node = gl.nodes.get(cid) + if node is None: + continue + f, l = gl.loc(node)[0], gl.loc(node)[1] + if not f or not isinstance(l, int) or l <= 0 or _is_nonproduct_path(f): + continue + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + continue + openable.append(cid) + if len(openable) != 1: + return nid + nid = openable[0] + seen.add(nid) + return nid + + def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int, primary_family: Optional[str] = None) -> list[str]: """Candidate roots for request-lifecycle stories, best driver first. @@ -712,6 +751,9 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int, drivers.append((_reach2(index, nid), nid)) drivers.sort(key=lambda pair: (-pair[0], pair[1])) for _, nid in drivers: + # An in-degree-0 root is often a thin WSGI/entry trampoline; descend to the + # real orchestrator it forwards to so the lifecycle is named at the dispatcher. + nid = _descend_trampoline(index, gl, nid, primary_family=primary_family) if nid not in seen: seen.add(nid) roots.append(nid) @@ -733,7 +775,7 @@ def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str]]: entry = (story.get("entry") or {}).get("node_id") if not entry: return [], [] - children: dict[str, list[tuple[int, dict]]] = {} + children: dict[str, list[tuple[int, dict, str]]] = {} functions: dict[str, dict] = {} for step in steps: fn = step.get("function") or {} @@ -743,7 +785,8 @@ def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str]]: functions[fid] = fn caller = (step.get("caller") or {}).get("node_id") if caller: - children.setdefault(caller, []).append((step.get("sequence", 0), fn)) + children.setdefault(caller, []).append( + (step.get("sequence", 0), fn, step.get("via") or "")) memo: dict[str, int] = {} @@ -754,7 +797,7 @@ def subtree(nid: str, guard: frozenset) -> int: return 0 deeper = guard | {nid} total = 0 - for _, fn in children.get(nid, []): + for _, fn, _via in children.get(nid, []): cid = fn.get("node_id") if cid: total += 1 + subtree(cid, deeper) @@ -767,14 +810,22 @@ def subtree(nid: str, guard: frozenset) -> int: seen = {entry} cur = entry while len(spine) < max_hops: - kids = [fn for _, fn in sorted(children.get(cur, []), key=lambda pair: pair[0]) + kids = [(fn, via) + for _, fn, via in sorted(children.get(cur, []), key=lambda t: t[0]) if fn.get("node_id") not in seen and _story_fn_openable(fn)] if not kids: break - pick = max(kids, key=lambda fn: ( - 0 if _is_error_name(fn.get("name")) else 1, - subtree(fn.get("node_id"), frozenset()))) - nid = pick.get("node_id") + # A direct CALLS edge is the real control flow; ``indirect:may_invoke`` hops + # are duck-typed over-approximations (a session deserialize, a JSON dump that + # *might* run). Preferring direct keeps the spine on the dispatch chain + # (wsgi_app -> full_dispatch_request -> dispatch_request) instead of wandering + # into a serialization detour that only looks bigger. Error/teardown branches + # derank next, then the deepest subtree breaks the remaining tie. + pick = max(kids, key=lambda kv: ( + 1 if kv[1] == "direct" else 0, + 0 if _is_error_name(kv[0].get("name")) else 1, + subtree(kv[0].get("node_id"), frozenset()))) + nid = pick[0].get("node_id") cur = nid seen.add(nid) spine.append(nid) @@ -913,6 +964,7 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, entrypoints: list[dict] = [] requests: list[dict] = [] + used_ids: set[str] = set() try: by_handler = EntryPoints(store).by_handler() # Strongest anchor per handler, then a stable global order over handlers. @@ -922,7 +974,6 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, key=lambda kv: (_anchor_strength(kv[1]), kv[1].get("file") or "", kv[1].get("anchor_label") or "", kv[0])) - used_ids: set[str] = set() for handler_id, anchor in ordered: if len(entrypoints) >= max_entrypoints: break @@ -977,6 +1028,51 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, max_requests=8, max_core=32, max_hops=max(2, chain_depth), primary_family=primary_family) + # Promote each lifecycle root to an entrypoint. ``by_handler`` only recognises + # module-level public helpers, so a framework's real request driver -- a WSGI + # ``Flask.wsgi_app``, an event loop -- is *never* an anchored handler and would + # otherwise be a request whose entry is nowhere in the entrypoint set. These + # drivers are the truest "begin reading here" nodes, so they lead the list. This + # also gives a library with no anchored handler at all (itsdangerous) a real, + # source-backed entrypoint, which the code-understanding contract requires. + entry_node_ids = {entry["node_id"] for entry in entrypoints} + promoted: list[dict] = [] + for req in requests: + root = req.get("entry_node") + if not root or root in entry_node_ids: + continue + node = gl.nodes.get(root) + if node is None: + continue + nfile, nline = gl.loc(node)[0], gl.loc(node)[1] + if not nfile or not isinstance(nline, int) or nline <= 0: + continue + if _is_nonproduct_path(nfile): + continue + if primary_family is not None: + fam = _language_family(nfile) + if fam is not None and fam != primary_family: + continue + entry_node_ids.add(root) + label = gl.label(node) + eid = f"entry.{_slug(label)}" + if eid in used_ids: + eid = f"{eid}.{_slug(root)}" + used_ids.add(eid) + try: + efile = comp._relative_path(nfile) or nfile + except Exception: + efile = nfile + promoted.append({ + "id": eid, + "label": label, + "kind": "request-lifecycle", + "node_id": root, + "file": efile, + "line": nline, + }) + entrypoints = promoted + entrypoints + files: list[dict] = [] try: seen_paths: set[str] = set() From b1a7717e59f90eda97e976c3c3da05a43c222b4d Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 13:33:31 +0530 Subject: [PATCH 17/19] nav(export): rank lifecycle roots by control cone so the real dispatcher leads The code-understanding projection roots each guided request path at a top-of-stack driver and follows its execution story. Two candidate sources feed the root set: the planner's entry handlers and every in-degree-0 product callable. The old order kept all handlers ahead of the drivers and then truncated the combined list to a fixed cap. On a framework that emits many thin handlers this starved the real dispatcher out of the pass entirely: Click surfaces ~30 decorator handlers (`version_option`, `argument`, ...), each spinning a valid but peripheral story, which filled the cap before `Command.main` -- the actual in-degree-0 dispatcher with the widest control cone -- was ever considered. The bundle then led with `secho`/`pause` string helpers instead of the command lifecycle. Rank every candidate (handlers and de-trampolined drivers alike) by two-hop reach before applying the cap, so the widest-cone lifecycle always survives into the bounded story pass. Ordering here governs only which candidates the pass sees; `_lifecycle_projection` still re-ranks the survivors, so this does not change how kept requests are ordered relative to each other -- it only stops a high-reach driver from being truncated away. In-degree-0 trampolines are still descended to their single product callee first (`Flask.__call__` -> `wsgi_app`, `BaseCommand.__call__` -> `main`) so the story is named at the dispatcher, not the forwarder. Behavior on the previously-working cases is unchanged: Flask still leads with `wsgi_app -> full_dispatch_request -> dispatch_request -> ...`, json5 still promotes its `lib/parse.js` lifecycles, django's 40 entrypoints are intact. Click now leads with `main -> make_context -> parse_args -> iter_params_for_processing`, and `main` is promoted to the top request-lifecycle entrypoint. Cost is unchanged: the story pass still runs at most `cap` execution stories; the only added work is one two-hop reach count per handler candidate (microseconds), and the graph builder is untouched. Verified: 47 tests in nav/test_bundle.py pass; exports re-checked on clicklib/flasklib/json5. --- lachesis/nav/bundle.py | 175 ++++++++++++++++++++++++++++++++--------- 1 file changed, 137 insertions(+), 38 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index dcbd9866..add4156d 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -558,6 +558,35 @@ def _norm_node(gl, node: dict) -> dict: ) _CALL_EDGE_KINDS = ("CALLS", "INVOKES", "MAY_INVOKE") +# Modules that are real product code but *peripheral* to the request lifecycle a +# reader wants first: the command-line front door, generic string/util helpers, the +# in-tree test harness (``testing.py`` -- kept in the graph, but never the headline +# lifecycle). A framework's CLI command has a long, valid execution story, so pure +# spine length floats it above the web path; demoting these modules as lifecycle +# *roots* keeps them in the bundle while letting the dispatch spine lead. Matched on +# the file's basename stem so it stays language-agnostic (cli.py, cli.js, cli.ts). +_PERIPHERAL_MODULE_STEMS = frozenset({ + "cli", "__main__", "__main", "cmd", "cmdline", "commands", "command", + "utils", "util", "helpers", "helper", "testing", "compat", "_compat", +}) + + +def _is_peripheral_module_path(path: Optional[str]) -> bool: + """True when a file is product code but off the primary request lifecycle. + + A soft signal for *ranking* only -- never for inclusion. The stem set is generic + (a CLI front-door, string/util helpers, the test harness); a segment named + ``commands`` catches a management-command package regardless of file name. + """ + if not path: + return False + p = str(path).replace("\\", "/") + stem = p.rsplit("/", 1)[-1].rsplit(".", 1)[0].lower() + if stem in _PERIPHERAL_MODULE_STEMS: + return True + segments = p.lower().split("/") + return "commands" in segments[:-1] + def _is_error_name(name: Optional[str]) -> bool: n = str(name or "").lower() @@ -704,21 +733,33 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int, uncalled once tests are excluded. Truncated to ``cap`` so the story pass is bounded regardless of codebase size. """ - roots: list[str] = [] + # Both sources feed one ranked candidate pool. A planner handler is *not* an + # automatic front-of-line: a framework like Click emits dozens of thin decorator + # handlers (``version_option``, ``argument``) that each spin a valid but peripheral + # story, and if they were kept ahead of the drivers they would fill ``cap`` and + # starve the real dispatcher (``Command.main``, in-degree-0, widest cone) out of the + # pass entirely. Ranking every candidate by two-hop reach means the widest-cone + # lifecycle always survives the cap; the downstream story pass re-ranks the + # survivors, so this ordering governs only *which* candidates it gets to see. + candidates: list[tuple[int, str]] = [] seen: set[str] = set() + + def _consider(nid: str, node: Optional[dict]) -> None: + if not nid or nid in seen: + return + f = gl.loc(node)[0] if node is not None else None + if node is None or _is_nonproduct_path(f): + return # a test/example handler is not a product lifecycle root + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + return # a non-primary-language handler (bundled JS in a Python repo) + seen.add(nid) + candidates.append((_reach2(index, nid), nid)) + for hid in handler_ids: - if hid and hid not in seen: - seen.add(hid) - node = gl.nodes.get(hid) - if node is not None and _is_nonproduct_path(gl.loc(node)[0]): - continue # a test/example handler is not a product lifecycle root - if node is not None and primary_family is not None: - fam = _language_family(gl.loc(node)[0]) - if fam is not None and fam != primary_family: - continue # a non-primary-language handler (bundled JS in a Python repo) - roots.append(hid) + _consider(hid, gl.nodes.get(hid) if hid else None) - drivers: list[tuple[int, str]] = [] try: callable_nodes = list(index.nodes_of_kind("function", "method", "constructor")) except Exception: @@ -748,41 +789,75 @@ def _lifecycle_roots(index, gl, handler_ids: list[str], *, cap: int, except Exception: continue if inn == 0: - drivers.append((_reach2(index, nid), nid)) - drivers.sort(key=lambda pair: (-pair[0], pair[1])) - for _, nid in drivers: - # An in-degree-0 root is often a thin WSGI/entry trampoline; descend to the - # real orchestrator it forwards to so the lifecycle is named at the dispatcher. - nid = _descend_trampoline(index, gl, nid, primary_family=primary_family) - if nid not in seen: - seen.add(nid) - roots.append(nid) - return roots[:cap] + # An in-degree-0 root is often a thin WSGI/entry trampoline (Flask.__call__, + # Click's BaseCommand.__call__); descend to the real orchestrator it forwards + # to *before* ranking, so the driver is ordered by the dispatcher's own + # control cone (Click's main reaches far more than its one-line __call__) and + # the lifecycle is named at the dispatcher rather than the forwarder. + driver = _descend_trampoline(index, gl, nid, primary_family=primary_family) + _consider(driver, gl.nodes.get(driver)) + + candidates.sort(key=lambda pair: (-pair[0], pair[1])) + return [nid for _, nid in candidates][:cap] + + +def _hop_semantics(via: str, branch: dict) -> dict: + """Per-hop reader facts from an execution-story step: how it is reached and + whether it decides. ``reached_via`` names the call-seam boundary (a direct call + vs a dynamic dispatch the graph resolved), and the branch summary flags a hop + that forks control -- the decision points a newcomer traces. All derived from + real story structure; absent facts are simply omitted so hops stay compact. + """ + out: dict = {} + v = via or "" + if v == "entry": + out["reached_via"] = "entry" + elif v == "direct": + out["reached_via"] = "direct call" + elif v.startswith("indirect:"): + out["reached_via"] = f"dynamic dispatch ({v.split(':', 1)[1] or 'resolved'})" + elif v: + out["reached_via"] = v + count = branch.get("count") or 0 + if count: + out["decides"] = True + out["branch_count"] = count + kinds = branch.get("kinds") or [] + if kinds: + out["decision_kinds"] = kinds + return out -def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str]]: - """Linearize an execution story into (primary success spine, all functions). +def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str], dict]: + """Linearize an execution story into (primary success spine, all functions, meta). The story is a call tree keyed by (caller -> function). The spine walks from the - entry always choosing the deepest-subtree callee, deranking obvious error/ - teardown branches, so it follows the happy path (a WSGI entry down through + entry always choosing the direct-edge, deepest-subtree callee, deranking obvious + error/teardown branches, so it follows the happy path (a WSGI entry down through dispatch to the response) rather than wandering into a handler. Every consecutive pair on the spine is a real edge the story observed; cycles are cut by ``seen``. - Returns the ordered spine node ids and the flat set of every function id the - story touched (the raw material for the architecture core). + Returns the ordered spine node ids, the flat set of every function id the story + touched (the raw material for the architecture core), and a per-spine-node + semantics map (how each hop is reached, whether it branches). """ steps = story.get("steps") or [] entry = (story.get("entry") or {}).get("node_id") if not entry: - return [], [] + return [], [], {} children: dict[str, list[tuple[int, dict, str]]] = {} functions: dict[str, dict] = {} + branches: dict[str, dict] = {} for step in steps: fn = step.get("function") or {} fid = fn.get("node_id") if not fid: continue functions[fid] = fn + # Per-function control facts: how many decision points the body has and which + # control kinds -- surfaced on the hop as its decision signal. + rows = step.get("branches") or [] + kinds = sorted({r.get("control") for r in rows if r.get("control")}) + branches[fid] = {"count": step.get("branch_count") or 0, "kinds": kinds} caller = (step.get("caller") or {}).get("node_id") if caller: children.setdefault(caller, []).append( @@ -809,6 +884,7 @@ def subtree(nid: str, guard: frozenset) -> int: spine = [entry] seen = {entry} cur = entry + meta: dict[str, dict] = {entry: _hop_semantics("entry", branches.get(entry) or {})} while len(spine) < max_hops: kids = [(fn, via) for _, fn, via in sorted(children.get(cur, []), key=lambda t: t[0]) @@ -826,11 +902,12 @@ def subtree(nid: str, guard: frozenset) -> int: 0 if _is_error_name(kv[0].get("name")) else 1, subtree(kv[0].get("node_id"), frozenset()))) nid = pick[0].get("node_id") + meta[nid] = _hop_semantics(pick[1], branches.get(nid) or {}) cur = nid seen.add(nid) spine.append(nid) ordered_functions = [fid for fid in functions if _story_fn_openable(functions[fid])] - return spine, ordered_functions + return spine, ordered_functions, meta def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], *, @@ -856,7 +933,7 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], except Exception: return requests, core - ranked: list[tuple[int, int, str, list[str], list[str]]] = [] + ranked: list[tuple[int, int, int, int, str, list[str], list[str], dict]] = [] for root in roots: try: story = _call("execution_story", @@ -866,19 +943,33 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], continue if not isinstance(story, dict): continue - spine, functions = _story_spine(story, max_hops=max_hops) + spine, functions, meta = _story_spine(story, max_hops=max_hops) if len(spine) < 2: continue - ranked.append((len(spine), len(functions), root, spine, functions)) - - # Deepest, then broadest, wins attention; stable by root id for reproducibility. - ranked.sort(key=lambda row: (-row[0], -row[1], row[2])) + # A peripheral root (a CLI command, a util helper) still yields a long, valid + # story, so ranking on spine length alone floats it above the web request path + # a reader opened the bundle to see. Demote by the root's module so the primary + # dispatch lifecycle leads; the peripheral path is kept, just not first. + root_file = gl.loc(gl.nodes.get(root))[0] if gl.nodes.get(root) else None + demote = 1 if _is_peripheral_module_path(root_file) else 0 + # The dispatcher a reader wants first is the top-of-stack that drives the most + # code, not whichever helper happens to keep the longest in-library spine. The + # true lifecycle (Flask.wsgi_app, Click's Command.main -> invoke) exits to + # external user code quickly, so its openable spine is *short* even though its + # control cone is the widest; a string helper (secho -> echo -> isatty) stays + # in-library and spins a longer spine. Ranking by 2-hop reach first puts the + # real driver on top; spine length only breaks ties between comparable drivers. + reach = _reach2(index, root) + ranked.append((demote, -reach, len(spine), len(functions), root, spine, functions, meta)) + + # Primary lifecycles first (widest control cone), then deepest, then broadest. + ranked.sort(key=lambda row: (row[0], row[1], -row[2], -row[3], row[4])) node_ids = set(asm.nodes) covered: set[str] = set() core_ids: set[str] = set() used_ids: set[str] = set() - for _, _, root, spine, functions in ranked: + for _, _, _, _, root, spine, functions, meta in ranked: if len(requests) >= max_requests: break if root in covered: # a redundant suffix of a spine already shown @@ -891,7 +982,9 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], continue asm.add_node(_norm_node(gl, node), default_kind="function") node_ids.add(nid) - hops.append({"node_id": nid, "caption": gl.label(node)}) + hop = {"node_id": nid, "caption": gl.label(node)} + hop.update(meta.get(nid) or {}) + hops.append(hop) chain_ids.append(nid) if len(hops) < 2: continue @@ -1352,6 +1445,12 @@ def _finalize_requests(raw_requests: list[dict], node_map: dict, nid = hop.get("node_id") entry = {"id": f"{rid}:{i:02d}", "node_id": nid, "caption": hop.get("caption")} + # Carry the story-derived hop semantics (how this hop is reached, whether + # it forks control) through decoration so a reader sees the call-seam and + # decision points, not just an ordered list of names. + for key in ("reached_via", "decides", "branch_count", "decision_kinds"): + if hop.get(key) is not None: + entry[key] = hop[key] if i > 1: entry["edge_label"] = _edge_label( edges_by_pair, hops[i - 2].get("node_id"), nid) From 68818896c03ccc8873e6f908c8d04509bf543ab4 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 14:56:34 +0530 Subject: [PATCH 18/19] nav(export): route the lifecycle spine to its result terminus, name hops and areas The code-understanding projection was structurally usable but read coarsely: the Flask request spine stopped at a routing corner, hop captions were bare symbols, and every module of a flat single-package framework collapsed into one concept. This reworks the three, all on generic identifier/graph morphology -- an English verb/noun vocabulary and the real call graph -- never a per-framework symbol table. Spine reaches the response terminus. An execution story is a *tree*: each function is attached under its first-discovered caller, so a genuine callee can hang off an earlier parent than the one whose body makes the call. Flask's finalize_request (which builds the response) lands under handle_exception in the tree, not under full_dispatch_request whose call actually reaches it, so a walk over story-children alone could never route past dispatch_request to make_response. _story_spine now recovers the missing edges from the graph (_candidate_children): every genuine callee of the current node that the story itself visited becomes a candidate, carrying its story fn record and a via classified from the edge kind. It invents no nodes (only functions already in the story are admitted) and no edges the graph does not hold. The five-key hop ranker then does the rest generically -- a candidate whose subtree constructs the returned result (a construction verb over a result noun, and not a fallback/error name: make_response yes, the special-case make_default_options_ response no) is preferred -- so Flask now ends wsgi_app -> full_dispatch_request -> finalize_request -> make_response. Readable hop captions. Each hop keeps its exact symbol as caption (still greppable) and gains reads_as: a human phrase from the symbol's morphology (_readable_caption) -- the leading action verb past any modifier, rendered in the third person over the remaining object tokens (full_dispatch_request -> "dispatches the request", make_response -> "builds the response", unsign -> "verifies the signature on"). It degrades to the bare humanized noun phrase on a name with no known verb, so it only ever adds a hint, never hides the identifier. Module-centric concepts. Call-community clustering is too coarse for a flat package: Flask's whole tree came back as one 20-file community labelled off its first path (__init__.py). Concepts are now the module areas a newcomer would name -- one per product file, ranked by definition count alone, capped at _MAX_CONCEPTS, each labelled by its module stem (widened to "parent - stem" only to break a genuine collision like app vs sansio/app). Ranking on the definition count keeps this a single cheap node scan (django: 0.14s over 9,719 callable nodes). An earlier draft added an outgoing-call-degree tiebreaker that ran one graph edge-lookup per function; that scan measured 8.15s on django -- ~8s of avoidable per-function graph queries -- and it never changed an order in practice (a module's definition count already fully separates the areas), so it was dropped. Flask surfaces app, blueprints, cli, ctx, sessions, templating, config as distinct areas; json5 surfaces parse, stringify, cli; django surfaces models, options, query, widgets. _project_concepts still restricts every concept to product nodes in the final pool, so the areas stay honest to what shipped. This deliberately changes projection *output* (it is a comprehension-quality change, not behaviour-preserving); the graph builder and schema are untouched and the strict source-backed gating on every hop is unchanged. Removed the last framework-specific tokens from the error vocabulary (finalize_request/handle_http) that had both violated the generic-morphology rule and mis-deranked the response path. Verified: 47 tests in nav/test_bundle.py pass; exports re-checked on flasklib (spine now ends at make_response; 11 module concepts), clicklib (main -> make_context -> parse_args, unchanged), itsdangerous and json5 (spines unchanged), django (8 request spines sound, 4 module concepts, no regression). --- lachesis/nav/bundle.py | 322 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 287 insertions(+), 35 deletions(-) diff --git a/lachesis/nav/bundle.py b/lachesis/nav/bundle.py index add4156d..df1ffc3c 100644 --- a/lachesis/nav/bundle.py +++ b/lachesis/nav/bundle.py @@ -551,13 +551,136 @@ def _norm_node(gl, node: dict) -> dict: # The request lifecycle a reader wants is the *success* path; error, teardown and # logging branches are real but secondary, so we only derank them when choosing the -# primary hop -- never drop them. Substring match keeps this language-agnostic. -_LIFECYCLE_ERROR_TOKENS = ( - "exception", "error", "teardown", "cleanup", "abort", "log_", - "handle_http", "raise_", "rollback", "finalize_request", -) +# primary hop -- never drop them. Word-token match (not raw substring) over the +# identifier keeps this generic and framework-agnostic: it is a vocabulary of +# English failure/teardown verbs, never a hardcoded symbol from one library. +_LIFECYCLE_ERROR_TOKENS = frozenset({ + "exception", "error", "err", "teardown", "cleanup", "abort", "raise", + "rollback", "fail", "reject", "panic", "warn", "log", "logging", +}) _CALL_EDGE_KINDS = ("CALLS", "INVOKES", "MAY_INVOKE") +# A special-case/fallback branch is real but is not the lifecycle a reader opens the +# bundle to follow: an auto-generated default reply, a not-found placeholder, an +# unsupported-method stub. Deranked (never dropped) below error branches when picking +# the primary hop, so the spine stays on the ordinary request rather than diving into +# a corner case. Generic English morphology -- matches ``make_default_options_response`` +# or ``handle_not_found`` in any codebase, not a symbol from one framework. +_LIFECYCLE_FALLBACK_TOKENS = frozenset({ + "default", "fallback", "options", "notfound", "missing", "unsupported", + "unavailable", "placeholder", "noop", "stub", "unknown", +}) + +# A request lifecycle culminates in *constructing the thing it returns* -- a response, +# a rendered page, a serialized result. We recognise that terminus by morphology so the +# spine ends there rather than in a routing corner: a construction verb applied to a +# result noun. Generic across codebases (``make_response``, ``build_result``, +# ``render_page``, ``serialize_output``), never a hardcoded framework symbol. +_RESULT_CONSTRUCTION_VERBS = frozenset({ + "make", "build", "create", "construct", "render", "format", "compose", + "produce", "generate", "new", "serialize", "encode", "write", "emit", +}) +_RESULT_NOUNS = frozenset({ + "response", "reply", "result", "output", "answer", "payload", "body", + "page", "document", "content", "view", "html", "json", "template", +}) + + +def _identifier_tokens(name: Optional[str]) -> list[str]: + """Lowercased word tokens of an identifier, splitting snake_case and camelCase. + + ``full_dispatch_request`` -> ``[full, dispatch, request]``; ``makeResponse`` -> + ``[make, response]``; ``__call__`` -> ``[call]``; ``HTTPServer`` -> ``[http, + server]``. The atom every generic morphology check below reasons over, so a rule + keys off whole words rather than raw substrings (no ``err`` inside ``inherit``). + """ + return [t.lower() for t in re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]+|[a-z]+|\d+", + str(name or ""))] + + +def _is_fallback_name(name: Optional[str]) -> bool: + return bool(_LIFECYCLE_FALLBACK_TOKENS.intersection(_identifier_tokens(name))) + + +def _is_result_construction(name: Optional[str]) -> bool: + """True when an identifier reads as 'construct the returned result'. + + Requires both a construction verb and a result noun as whole tokens, and is not a + fallback/error name -- so ``make_response`` and ``render_page`` qualify while a + special-case ``make_default_options_response`` (fallback) and a plain + ``process_response`` (no construction verb) do not. + """ + toks = set(_identifier_tokens(name)) + if _LIFECYCLE_ERROR_TOKENS.intersection(toks) or _LIFECYCLE_FALLBACK_TOKENS.intersection(toks): + return False + return bool(_RESULT_CONSTRUCTION_VERBS.intersection(toks) and _RESULT_NOUNS.intersection(toks)) + + +# Generic leading-verb -> third-person phrase, so a hop caption reads as what the +# step *does* rather than as a bare symbol. Keyed off the identifier's action token, +# it renders any codebase's ``make_*``/``parse_*``/``dispatch_*`` the same way -- a +# vocabulary of English verbs, never a per-framework symbol table. +_VERB_READS_AS = { + "make": "builds", "build": "builds", "create": "creates", "construct": "constructs", + "new": "creates", "render": "renders", "format": "formats", "compose": "assembles", + "produce": "produces", "generate": "generates", "prepare": "prepares", "wrap": "wraps", + "get": "reads", "fetch": "fetches", "load": "loads", "read": "reads", "find": "finds", + "lookup": "looks up", "resolve": "resolves", "select": "selects", "match": "matches", + "search": "searches", "query": "queries", "collect": "collects", "gather": "gathers", + "dispatch": "dispatches", "route": "routes", "handle": "handles", "process": "processes", + "run": "runs", "execute": "runs", "exec": "runs", "invoke": "invokes", "call": "calls", + "apply": "applies", "perform": "performs", "iter": "iterates over", + "parse": "parses", "decode": "decodes", "deserialize": "deserializes", "unpack": "unpacks", + "encode": "encodes", "serialize": "serializes", "dump": "serializes", "pack": "packs", + "write": "writes", "save": "saves", "store": "stores", "persist": "persists", + "send": "sends", "emit": "emits", "flush": "flushes", "commit": "commits", + "validate": "validates", "check": "checks", "verify": "verifies", "ensure": "ensures", + "sign": "signs", "unsign": "verifies the signature on", "hash": "hashes", + "init": "initializes", "initialize": "initializes", "setup": "sets up", + "configure": "configures", "register": "registers", "bind": "binds", "connect": "connects", + "open": "opens", "close": "closes", "push": "pushes", "pop": "pops", + "add": "adds", "append": "appends", "remove": "removes", "delete": "deletes", + "update": "updates", "set": "sets", "reset": "resets", "clear": "clears", + "preprocess": "preprocesses", "postprocess": "post-processes", "finalize": "finalizes", + "convert": "converts", "transform": "transforms", "normalize": "normalizes", +} +# Modifier/adjective tokens that decorate an identifier without naming its action or +# object; dropped from a caption so ``full_dispatch_request`` reads "dispatches the +# request", not "dispatches the full request". +_CAPTION_FILLER = frozenset({ + "full", "do", "self", "the", "internal", "impl", "inner", "raw", "safe", + "unsafe", "sync", "async", "maybe", "try", "helper", "default", "real", +}) + + +def _readable_caption(name: Optional[str], *, is_entry: bool = False) -> str: + """A short human phrase for a hop: what the step does, from its name's morphology. + + Finds the leading action verb (past any modifier like ``full``/``do``) and renders + it in the third person over the remaining object tokens: ``make_response`` -> + "builds the response", ``full_dispatch_request`` -> "dispatches the request", + ``parse_args`` -> "parses the args". A name with no recognised verb reads as its + humanized noun phrase (an entry as the place to "start"). Never a framework table -- + the same rule renders any codebase, and it degrades to the bare symbol on anything + it cannot parse, so it only ever adds a hint, never hides the identifier. + """ + toks = _identifier_tokens(name) + if not toks: + return str(name or "step") + verb_i = None + for i, tok in enumerate(toks): + if tok in _VERB_READS_AS: + verb_i = i + break + if tok not in _CAPTION_FILLER: + break # a leading noun-style token: not a verb-first name + if verb_i is not None: + phrase = _VERB_READS_AS[toks[verb_i]] + obj = [t for t in toks[verb_i + 1:] if t not in _CAPTION_FILLER] + return f"{phrase} the {' '.join(obj)}" if obj else phrase + human = " ".join(t for t in toks if t not in _CAPTION_FILLER) or " ".join(toks) + return f"starts at {human}" if is_entry else human + # Modules that are real product code but *peripheral* to the request lifecycle a # reader wants first: the command-line front door, generic string/util helpers, the # in-tree test harness (``testing.py`` -- kept in the graph, but never the headline @@ -589,8 +712,38 @@ def _is_peripheral_module_path(path: Optional[str]) -> bool: def _is_error_name(name: Optional[str]) -> bool: - n = str(name or "").lower() - return any(tok in n for tok in _LIFECYCLE_ERROR_TOKENS) + return bool(_LIFECYCLE_ERROR_TOKENS.intersection(_identifier_tokens(name))) + + +# How many module areas the concept list surfaces. Concepts are the "areas" a reader +# would name (the request lifecycle, templates, sessions, the CLI); we bound them so a +# large tree stays legible while a small one is not padded. +_MAX_CONCEPTS = 12 + + +def _module_stem(path: Optional[str]) -> str: + """The bare module name of a source path: ``pkg/sessions.py`` -> ``sessions``.""" + base = str(path or "").replace("\\", "/").rsplit("/", 1)[-1] + return base.rsplit(".", 1)[0] or base + + +def _concept_label(path: Optional[str], stem_counts) -> str: + """A concept's display label from its module path. + + The module stem alone (``sessions``, ``templating``, ``cli``) is the area name a + reader recognises. Widen to ``parent · stem`` only to break a genuine collision -- + ``app.py`` and ``sansio/app.py`` both stem to ``app`` -- so labels stay short but + never ambiguous. Generic over any layout; never a per-framework name table. + """ + p = str(path or "").replace("\\", "/") + if p.startswith("src/"): + p = p[4:] + stem = _module_stem(p) + if stem_counts.get(stem, 0) > 1 and "/" in p: + parent = p.rsplit("/", 2)[-2] + if parent: + return f"{parent} · {stem}" + return stem def _story_fn_openable(fn: dict) -> bool: @@ -828,7 +981,7 @@ def _hop_semantics(via: str, branch: dict) -> dict: return out -def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str], dict]: +def _story_spine(story: dict, index, gl, *, max_hops: int) -> tuple[list[str], list[str], dict]: """Linearize an execution story into (primary success spine, all functions, meta). The story is a call tree keyed by (caller -> function). The spine walks from the @@ -839,6 +992,11 @@ def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str], d Returns the ordered spine node ids, the flat set of every function id the story touched (the raw material for the architecture core), and a per-spine-node semantics map (how each hop is reached, whether it branches). + + ``index``/``gl`` let the walk recover call edges the story *tree* attached to a + different parent (see ``_candidate_children``): the story visits each function + once, so a genuine callee can hang off an earlier caller than the one whose body + actually makes the call, and a tree-only walk could never reach it. """ steps = story.get("steps") or [] entry = (story.get("entry") or {}).get("node_id") @@ -863,6 +1021,45 @@ def _story_spine(story: dict, *, max_hops: int) -> tuple[list[str], list[str], d children.setdefault(caller, []).append( (step.get("sequence", 0), fn, step.get("via") or "")) + def _candidate_children(node_id: str) -> list[tuple[dict, str]]: + """Callees to consider when extending the spine from ``node_id``. + + The execution story is a *tree*: each function is attached under its + first-discovered caller, so a real callee can hang off a different parent + than the one whose body makes the call. Flask's ``finalize_request`` (which + builds the response) lands under ``handle_exception`` in the tree, not under + ``full_dispatch_request`` whose call actually reaches it -- so a walk over + story-children alone can never route the spine to the response terminus. + Recover the missing edges from the graph: every genuine callee of + ``node_id`` that the story itself visited becomes a candidate, carrying its + story fn record and a via classified from the edge kind. This invents no + nodes (only functions already in the story are admitted) and no edges the + graph does not hold; it merely lets the spine follow the real call an + earlier caller happened to be credited with in the tree. + """ + out: list[tuple[dict, str]] = [] + story_ids: set[str] = set() + for _seq, fn, via in sorted(children.get(node_id, []), key=lambda t: t[0]): + cid = fn.get("node_id") + if cid: + story_ids.add(cid) + out.append((fn, via)) + try: + direct = {t.get("id") for t in index.targets(node_id, _CALL_EDGE_KINDS[0]) + if t.get("id")} + callees = [t.get("id") for t in index.targets(node_id, *_CALL_EDGE_KINDS) + if t.get("id")] + except Exception: + return out + added: set[str] = set() + for t in callees: + if t in story_ids or t in added or t not in functions: + continue + added.add(t) + out.append((functions[t], + "direct" if t in direct else "indirect:may_invoke")) + return out + memo: dict[str, int] = {} def subtree(nid: str, guard: frozenset) -> int: @@ -881,25 +1078,56 @@ def subtree(nid: str, guard: frozenset) -> int: memo[nid] = total return total + reach_memo: dict[str, bool] = {} + + def reaches_result(nid: str, guard: frozenset) -> bool: + """Does this subtree build the value the request returns? A response, a + rendered page, a serialized result -- recognised by morphology (see + ``_is_result_construction``), so the spine can end at the response terminus + rather than in a routing corner. Bounded and cycle-guarded like ``subtree``. + """ + if nid in reach_memo: + return reach_memo[nid] + if nid in guard: + return False + if _is_result_construction((functions.get(nid) or {}).get("name")): + reach_memo[nid] = True + return True + deeper = guard | {nid} + found = any(cid and reaches_result(cid, deeper) + for _, fn, _via in children.get(nid, []) + for cid in (fn.get("node_id"),)) + reach_memo[nid] = found + return found + spine = [entry] seen = {entry} cur = entry meta: dict[str, dict] = {entry: _hop_semantics("entry", branches.get(entry) or {})} while len(spine) < max_hops: kids = [(fn, via) - for _, fn, via in sorted(children.get(cur, []), key=lambda t: t[0]) + for fn, via in _candidate_children(cur) if fn.get("node_id") not in seen and _story_fn_openable(fn)] if not kids: break - # A direct CALLS edge is the real control flow; ``indirect:may_invoke`` hops - # are duck-typed over-approximations (a session deserialize, a JSON dump that - # *might* run). Preferring direct keeps the spine on the dispatch chain - # (wsgi_app -> full_dispatch_request -> dispatch_request) instead of wandering - # into a serialization detour that only looks bigger. Error/teardown branches - # derank next, then the deepest subtree breaks the remaining tie. + # Rank each candidate hop, best first, by five generic signals: + # 1. a direct CALLS edge is the real control flow; ``indirect:may_invoke`` + # hops are duck-typed over-approximations (a session deserialize, a JSON + # dump that *might* run), so direct wins; + # 2. error/teardown branches derank (real, but not the success path); + # 3. special-case/fallback branches derank next (an auto OPTIONS reply, a + # not-found stub -- a corner, not the ordinary request); + # 4. a branch that reaches the response/result construction is preferred, so + # the spine ends where the request builds what it returns + # (full_dispatch_request -> finalize_request -> make_response) rather than + # tunnelling into the widest routing subtree and stopping at a corner; + # 5. the deepest subtree breaks any remaining tie. + # Every signal is morphology over the identifier, never a framework symbol. pick = max(kids, key=lambda kv: ( 1 if kv[1] == "direct" else 0, 0 if _is_error_name(kv[0].get("name")) else 1, + 0 if _is_fallback_name(kv[0].get("name")) else 1, + 1 if reaches_result(kv[0].get("node_id"), frozenset()) else 0, subtree(kv[0].get("node_id"), frozenset()))) nid = pick[0].get("node_id") meta[nid] = _hop_semantics(pick[1], branches.get(nid) or {}) @@ -943,7 +1171,7 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], continue if not isinstance(story, dict): continue - spine, functions, meta = _story_spine(story, max_hops=max_hops) + spine, functions, meta = _story_spine(story, index, gl, max_hops=max_hops) if len(spine) < 2: continue # A peripheral root (a CLI command, a util helper) still yields a long, valid @@ -982,7 +1210,13 @@ def _lifecycle_projection(asm: "_Assembler", index, gl, handler_ids: list[str], continue asm.add_node(_norm_node(gl, node), default_kind="function") node_ids.add(nid) - hop = {"node_id": nid, "caption": gl.label(node)} + label = gl.label(node) + # ``caption`` stays the exact symbol (a reader can grep it); ``reads_as`` + # adds a human phrase derived from the symbol's morphology, so the hop + # says what the step does ("dispatches the request") without hiding the + # identifier. First hop on the spine is the entry -- phrased as a start. + hop = {"node_id": nid, "caption": label, + "reads_as": _readable_caption(label, is_entry=not hops)} hop.update(meta.get(nid) or {}) hops.append(hop) chain_ids.append(nid) @@ -1181,28 +1415,44 @@ def _comprehension_projection(asm: "_Assembler", *, max_entrypoints: int, except Exception: files = [] + # Concepts are the module *areas* a newcomer would name: the request lifecycle, + # routing, request context, templates, sessions, the CLI. Call-community + # clustering is too coarse here -- a flat single-package framework (every file in + # one directory) collapses into one giant community, so the whole request path, + # templating and session code read as a single undifferentiated blob. Derive areas + # from the *modules* instead: one concept per product file, ranked by how much it + # defines (definition count, then path for a stable order), capped at + # ``_MAX_CONCEPTS``. Fully generic -- the busiest modules of any codebase are its + # areas, named by their own path, never a framework symbol table -- and it degrades + # to an empty list, never raises. Ranking on the definition count alone keeps this a + # single cheap node scan (no per-node graph query), so it stays bounded on a large + # tree where an edge lookup per function would dominate the export. concepts: list[dict] = [] try: - architecture = comp.architecture_map(max_communities=8, max_files_per_community=20) - for idx, community in enumerate(architecture.get("communities") or []): - # Filter non-product files *before* deriving the label and file set: a - # community that mixes json5's ``lib/*.js`` with the vendored TypeScript - # compiler under ``node_modules`` must be labelled ``lib``, not - # ``… · node_modules · typescript · lib`` off the first (vendored) path. - paths = [str(path) for path in community.get("files") or [] - if path and not _is_nonproduct_path(str(path))] - if not paths: + import collections as _collections + defs: "_collections.Counter" = _collections.Counter() + for node in index.nodes_of_kind("function", "method", "constructor"): + f = gl.loc(node)[0] + if not f or _is_nonproduct_path(f): continue - first = paths[0] - directory = first.rsplit("/", 1)[0] if "/" in first else first - if directory.startswith("src/"): - directory = directory[4:] - label = directory.replace("/", " · ") or first + if primary_family is not None: + fam = _language_family(f) + if fam is not None and fam != primary_family: + continue # keep the concept list in the repo's own language + try: + rel = comp._relative_path(f) or f + except Exception: + rel = f + defs[rel] += 1 + ranked_modules = sorted(defs.items(), key=lambda kv: (-kv[1], kv[0])) + top = ranked_modules[:_MAX_CONCEPTS] + stem_counts = _collections.Counter(_module_stem(rel) for rel, _ in top) + for rel, n in top: concepts.append({ - "id": f"concept.{_slug(community.get('id') or idx)}", - "label": label, - "description": f"Connected code area spanning {len(paths)} file(s).", - "file_paths": paths, + "id": f"concept.{_slug(rel)}", + "label": _concept_label(rel, stem_counts), + "description": f"The {_module_stem(rel)} module ({n} definition(s)).", + "file_paths": [rel], }) except Exception: concepts = [] @@ -1445,6 +1695,8 @@ def _finalize_requests(raw_requests: list[dict], node_map: dict, nid = hop.get("node_id") entry = {"id": f"{rid}:{i:02d}", "node_id": nid, "caption": hop.get("caption")} + if hop.get("reads_as"): + entry["reads_as"] = hop["reads_as"] # Carry the story-derived hop semantics (how this hop is reached, whether # it forks control) through decoration so a reader sees the call-seam and # decision points, not just an ordered list of names. From d96e11acc9606b539775d771285a904c81380470 Mon Sep 17 00:00:00 2001 From: Riyan Dhiman Date: Sun, 6 Sep 2026 15:13:50 +0530 Subject: [PATCH 19/19] release: cut 0.5.2 (version, changelog, server.json, crate) Roll the version surface 0.5.1 -> 0.5.2 for the code-understanding release. The tag-triggered release gates require every pin to agree: pyproject's version, both server.json pins (the server and the lachesis-cpg package), the native lifetime_kernel crate, and a matching "## [0.5.2]" CHANGELOG heading. Bring all of them to 0.5.2 in one commit so a single tag push does not trip the guard the way the 0.5.1 cut did when the first bump moved pyproject alone. The release carries the lachesis.yml project configuration with default exclusion of vendored/generated/test/build-config code from the build and the export, and the comprehension-quality rework of the 2.0 Explorer bundle: request lifecycles rooted at the real dispatcher and routed forward to their result terminus, human-readable reads_as hop captions, module-centric concepts, primary-language gating, and normalized node locations under an enforced code-understanding contract. Metadata and projection scoping only -- no graph-schema, query-engine or candidate-surface change, and the schema 1.0 security bundle is unchanged. Verified the release gate locally: pyproject, both server.json pins and the native crate all read 0.5.2, and CHANGELOG.md carries the "## [0.5.2]" heading. --- CHANGELOG.md | 49 +++++++++++++++++++++++++++++++ native/lifetime_kernel/Cargo.toml | 2 +- pyproject.toml | 2 +- server.json | 4 +-- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 954df86a..b40dda8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,55 @@ Lachesis is pre-1.0. Until 1.0 the graph schema, the query surface and the MCP t may change between minor versions; those changes are called out here explicitly rather than left for you to discover. +## [0.5.2] + +Code-understanding release. 0.5.1 gave the 2.0 Explorer bundle a comprehension +layer; this release makes that layer read like something a newcomer could follow and +scopes it to the code that actually ships. A project can now declare what is and is +not its own source, and the guided request walks, hop captions and concept areas are +reworked so the projection reads as prose over the real call graph rather than a wall +of symbols. Version, projection and build-scoping only: no graph-schema, query-engine +or candidate-surface changes, and the schema 1.0 security bundle is unchanged. + +### Added + +- **`lachesis.yml` project configuration.** A repository can declare its own build and + export knobs in a single `lachesis.yml` at the tree root, resolved from the analysis + root upward. Vendored, generated, build-config and test/example scaffolding are + excluded from the build and from the export projection by default, so the graph and + the comprehension surface describe the product code rather than its dependencies and + fixtures; the defaults are overridable per project. +- **Human-readable hop captions.** Each hop in a guided request walk keeps its exact + symbol (still greppable) and gains a `reads_as` phrase derived from the symbol's own + morphology — a leading action verb rendered over its object tokens — so a reader + follows the lifecycle in plain language without losing the identifier. +- **Module-centric concepts.** `graph.concepts` are now the module areas a newcomer + would name — one per product file, ranked by how much it defines and labelled by its + own module stem — replacing the single coarse call-community that a flat package + collapsed into. +- **Curated tour manifests and recorded symbol documentation** are consumed by the + export when present, and a bounded, source-backed core spine is exported for + exploration. + +### Changed + +- **Request lifecycles root at the real dispatcher and reach their result.** A guided + request now begins at the function that actually dispatches the work rather than the + thin entry trampoline, and the spine is routed forward — over edges recovered from + the call graph, not just the story tree — to the hop that constructs the returned + result. Lifecycle roots are ranked by their control cone so the genuine dispatcher + leads. Entrypoints, request roots and concepts are gated to the repository's primary + language so the projection stays in the repo's own idiom. This changes 2.0 + comprehension output by design; the strict source-backing on every emitted hop is + unchanged. + +### Fixed + +- Node locations are normalized (a null file, line or end-line reads as `""`/`0` + rather than a missing key), and the code-understanding contract is enforced — a + code-understanding projection with no openable entrypoints is rejected rather than + emitted empty. + ## [0.5.1] Explorer-bundle comprehension release. The graph-first 2.0 Explorer bundle already diff --git a/native/lifetime_kernel/Cargo.toml b/native/lifetime_kernel/Cargo.toml index 53cac828..0f217dec 100644 --- a/native/lifetime_kernel/Cargo.toml +++ b/native/lifetime_kernel/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lachesis-lifetime-kernel" -version = "0.5.1" +version = "0.5.2" edition = "2021" [lib] diff --git a/pyproject.toml b/pyproject.toml index 4d9c952b..21d1ddaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lachesis-cpg" -version = "0.5.1" +version = "0.5.2" description = "A compiler-precise code property graph with an embedded columnar store and a navigation layer for security reasoning over source code." readme = "README.md" requires-python = ">=3.10" diff --git a/server.json b/server.json index fffd7c71..3853b85e 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.UnboundCompute/lachesis", "title": "Lachesis", "description": "Compiler-precise code property graph for C, Python, and TypeScript, navigable over MCP.", - "version": "0.5.1", + "version": "0.5.2", "repository": { "url": "https://github.com/UnboundCompute/lachesis", "source": "github" @@ -13,7 +13,7 @@ "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "lachesis-cpg", - "version": "0.5.1", + "version": "0.5.2", "runtimeHint": "uvx", "runtimeArguments": [ { "type": "named", "name": "--from", "value": "lachesis-cpg" },