diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42a3ea98..432d4b79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,9 @@ jobs: # for scripts/ on the S2083 reasoning above — the engine does not model # the check — and kept on for src/. # - # tests/fixtures/ is excluded outright: it holds input corpora, not + # tests/fixtures/ is excluded from test analysis + # (`sonar.test.exclusions`; it sits under `sonar.tests`, so + # `sonar.exclusions` would not reach it): it holds input corpora, not # code. Its package.json files exist to be parsed by cgis, and a lock # file next to them (text:S8564) would be a fixture nothing reads. # @@ -230,7 +232,6 @@ jobs: -Dsonar.newCode.referenceBranch=main -Dsonar.sources=src,scripts,.github -Dsonar.tests=tests - -Dsonar.exclusions=tests/fixtures/** -Dsonar.test.exclusions=tests/fixtures/** -Dsonar.python.version=3.12 -Dsonar.python.coverage.reportPaths=coverage.xml diff --git a/CLAUDE.md b/CLAUDE.md index b3ea868b..d67fb0ad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,14 +46,15 @@ The pipeline runs in three phases: **Extract → Resolve → Store**. ``` IngestionPipeline (pipeline.py) - ├── BaseExtractor subclasses (extractors/) ← language-specific AST → Nodes+Edges - │ └── PythonExtractor ← only language implemented so far + ├── cgis.languages (registry) ← lazy: a pack is imported when its first file is met + │ ├── languages/python/ ← PythonExtractor + PACK + │ └── languages/typescript/ ← TypeScriptExtractor (.ts/.tsx) + PACK ├── ResolverEngine (resolver/engine.py) ← raw_call: targets → resolved FQN edges └── SQLiteStore (storage/sqlite_store.py) ← persists graph; WAL mode └── QueryEngine (query/engine.py) ← BFS traversals (impact / flow) ``` -**Node FQN format**: `module.ClassName.method_name` — fully dot-separated, derived from the file path (e.g., `src/cgis/pipeline.py` → `src.cgis.pipeline.IngestionPipeline.run`). Use `file_path_to_module_fqn(path)` from `extractors/python_extractor.py` to convert paths. `__init__.py` strips the `/__init__` suffix. +**Node FQN format**: `module.ClassName.method_name` — fully dot-separated, derived from the file path (e.g., `src/cgis/pipeline.py` → `src.cgis.pipeline.IngestionPipeline.run`). Use `cgis.languages.scope_of(path, source_root)` to convert paths (it picks the file's own pack; the Python rule lives in `languages/python/extractor.py`). `__init__.py` strips the `/__init__` suffix. **Raw call convention**: Extractors emit edges with `target = "raw_call:"`. The `ResolverEngine` then resolves these to actual FQNs or leaves them unresolved (keeps the `raw_call:` prefix in output). @@ -68,12 +69,18 @@ IngestionPipeline (pipeline.py) - `NodeType` — structural (`FILE`, `MODULE`, `CLASS`, `FUNCTION`, `METHOD`…), runtime (`API_ENDPOINT`, `DB_TABLE`…), semantic (`DOMAIN_CONCEPT`). - `EdgeType` — structural (`CONTAINS`, `DECLARES`, `IMPORTS`), behavioral (`CALLS`, `REFERENCES`), semantic (`HANDLES`, `PERSISTS`, `AUTHORIZES`). -## Adding a New Language Extractor - -1. Subclass `BaseExtractor` in `src/cgis/extractors/`. -2. Implement `parse(code: str, file_path: str) -> tuple[list[Node], list[Edge]]`. -3. Emit `CALLS` edges with `target="raw_call:"` — the resolver handles the rest. -4. Register the extractor in `cli.py`'s `extractors` dict with the file extension key. +## Adding a Language Pack + +1. Create `src/cgis/languages//` with an extractor subclassing `cgis.core.extractor.BaseExtractor` + (`parse(code, file_path) -> tuple[list[Node], list[Edge]]`; emit `CALLS` edges with + `target="raw_call:"`, set `Node.language = ""`). +2. In its `__init__.py`, define `PACK`, subclassing `cgis.core.extractor.LanguagePack` (mark methods `@override`) + (`name`, `extensions`, `code_fence`, `is_test`, `scope_of`, `extractor`). +3. Add one `Manifest` to `MANIFESTS` in `src/cgis/languages/__init__.py`, with a `load` function that + imports the pack statically inside it (lazy, yet visible to the review-fingerprint walk). Nothing + else names the pack: CLI, MCP server, guardian and scripts all go through the registry, and the + `surface-packs` forbidden-edge gate fails if a surface imports a pack directly. +4. A pack never imports another pack or `cgis.storage` (`cross-pack`, `pack-isolation` gates). ## Type Discipline diff --git a/docs/ontology/domains.yaml b/docs/ontology/domains.yaml index 0321fd3a..3d803f81 100644 --- a/docs/ontology/domains.yaml +++ b/docs/ontology/domains.yaml @@ -6,10 +6,10 @@ domains: description: "Source code parsing and AST extraction" heuristics: file_path_patterns: - - "*extractors/*" + - "*languages/*" fqn_patterns: - "*.Extractor*" - - "*extractors.*" + - "*languages.*" StorageLayer: description: "Database storage, schemas, and persistence" diff --git a/docs/ontology/patterns.yaml b/docs/ontology/patterns.yaml index 71d06329..822cca72 100644 --- a/docs/ontology/patterns.yaml +++ b/docs/ontology/patterns.yaml @@ -114,8 +114,10 @@ patterns: # measurement (see plan: docs/specs/plans/2026-06-10-motif-basis-part-b.md). project_domains: # Python graph (ingest root: src/ → FQNs carry the cgis. prefix) + # Re-pointed in language-packs step 1: the extractors moved into cgis.languages.; + # the contract (BaseExtractor) moved to cgis.core. - name: "extraction" - fqn_prefix: "cgis.extractors" + fqn_prefix: "cgis.languages" expected_pattern: layered_dag profile: python # Re-negotiated in #116: PythonExtractor was decomposed from a God Object into a diff --git a/docs/plans/2026-10-14-language-packs-step1.md b/docs/plans/2026-10-14-language-packs-step1.md new file mode 100644 index 00000000..452da218 --- /dev/null +++ b/docs/plans/2026-10-14-language-packs-step1.md @@ -0,0 +1,1031 @@ +# Language Packs — Step 1: Skeleton and Registry — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move the Python and TypeScript extractors into `cgis/languages//`, behind a lazy registry that replaces `extractors/registry.py`, without changing any stored graph. + +**Architecture:** `cgis/languages/__init__.py` is the registry. It holds one `Manifest` per pack (name, module path, extensions, the test files the walk never reads) and imports a pack module only when a file of that language turns up. Each pack module exposes `PACK`, an object satisfying the `LanguagePack` protocol: `code_fence`, `is_test`, `scope_of`, `extractor`. Surfaces (CLI, MCP server, guardian, `mermaid.py`, scripts) go through the registry, never a pack. The extractor contract (`BaseExtractor`, `ModuleNamer`) moves to `cgis.core.extractor`. + +**Tech Stack:** Python 3.12, tree-sitter, SQLite, pytest, ruff, mypy strict, interrogate. + +**Spec:** `docs/specs/2026-10-14-language-packs-design.md` (rev 3) — sections *Registry*, *The `LanguagePack` contract → Files and namespaces*, *Migration* row 1. Outline: `docs/plans/2026-10-14-language-packs.md` "Step 1". + +## Global Constraints + +- `make format && make lint && make type-check && make pytest && make doc-coverage` passes before every commit. Gate commands with `&&`, never with `;`, a bare newline after a heredoc, or a pipe on the gating command. +- mypy strict on `src/`; docstring coverage ≥ 90%; Sonar cognitive complexity ≤ 15 per function. +- **Behaviour unchanged:** `tests/fixtures/graph_identity/snapshot.json` is not modified (`git diff --exit-code main -- tests/fixtures/graph_identity/snapshot.json`), and `test_graph_identity.py` passes under both hash seeds. +- Moved modules are imported from their new paths. No compatibility shims, no re-export modules left at the old paths. +- Drift tolerances only go down (`docs/ontology/tolerances.lock`). The god-object baseline gets a renamed entry, never a new one. If a self-drift or god-object gate fails for any reason other than a renamed FQN, **stop and report**. Do not raise anything. +- Test-count floor: raise `.github/test-count-baseline` (now 3095) in the same PR if the suite passes 3245. +- Commits end with `Co-Authored-By: Claude Opus 5.5 `. The PR body ends with `🤖 Generated with [Claude Code](https://claude.com/claude-code)`. +- Work in the worktree `/home/zaebee/projects/ownima/cgb-step1` on branch `feat/language-packs-step1`. + +## Rulings (where this plan departs from the spec's text, and why) + +- **R1 — Two "is test" rules, not one.** The spec says a pack's `is_test` replaces both `core.paths.is_test_path` and the pipeline's `_TEST_FILE_PATTERN`. They are different rules: + - `_TEST_FILE_PATTERN` (`foo.test.ts`, `foo.spec.py`) makes the walk **skip** a file; + - `is_test_path` (`tests/`, `test_*.py`) **flags** an ingested node. + + Merging them would drop every `tests/` directory from the graph. So: + - the skip becomes `Manifest.never_ingested` plus the registry's `skips(filename)`; + - `LanguagePack.is_test` is the flag, and both packs delegate it to `is_test_path`; + - the `Node` validator keeps calling `is_test_path`, because core cannot import packs. + + The pack method gets its first consumer with Go (step 7): that extractor passes `is_test` explicitly, and the validator already honours an explicit value. *Cost if wrong:* one method rename in step 7. +- **R2 — TypeScript has two grammars.** `.ts` and `.tsx` need different parsers and different fences. So the pack answers per extension: + - `extractor(extension, source_roots)`, where the spec has `extractor(source_roots)`; + - `code_fence(path)`, where the spec has a `code_fence` attribute. + + *Cost if wrong:* a signature change when Go lands. Go ignores the argument. +- **R3 — `extractor_for(path, roots)` becomes `extractors(roots)`, a lazy `Mapping[str, BaseExtractor]`.** `IngestionPipeline` keeps its `Mapping` parameter. 34 tests inject exactly one language through it, and production callers pass the lazy mapping. Laziness comes from looking up by key: the mapping builds an extractor on first `__getitem__`, and `__contains__`/`__iter__` never build one. +- **R4 — The contract lives in `cgis.core.extractor`, not `cgis/languages/`.** The forbidden-edge gate's `_pack()` reads every `cgis.languages.` module as a pack. A contract module next to the packs would trip `cross-pack` from every pack. `base.py` imports only `cgis.core.models`, so `core-isolation` holds. +- **R5 — The CLI's Python fallback moves into the registry's `scope_of(path, root)`.** A path with no pack (e.g. extensionless `src/cgis/pipeline`) gets the Python rule, a slash-to-dot rewrite. This is the same kind of quirk the spec keeps in the registry for `classify` with no source. +- **R6 — `_python_ast.py` becomes `_syntax.py`, not `_ast.py`.** A module named `_ast` shadows the stdlib C module whenever its directory lands on `sys.path`. +- **R7 — Each pack module imports its extractor at top level.** The registry is what stays lazy: it imports a pack only when it meets one of that pack's files. Nothing imports inside a function. + +## Review Focus + +- **`Foo.TS` (uppercase extension).** Registry and pipeline must both say "not ours". Covered by the ported `TestCaseSensitivity` (Task 2). +- **A `.tsx` file must get the TSX grammar.** `"const a =
;"` parses cleanly with the `.tsx` extractor and with errors under `.ts` (Task 2, `test_tsx_gets_the_tsx_grammar`). +- **A Python-only repository must never load `tree_sitter_typescript`.** This includes the CLI path and the TS-only config readers, which must not touch the TS extractor when the repo has no package.json or tsconfig. Task 3's subprocess test has a positive control. +- **Every extracted node's `language` must name the pack that reads its file.** `mermaid.py` switches from the path suffix to `pack_for_node`, so this must hold (Task 2, `test_every_node_names_the_pack_that_reads_its_file`). A language no pack has (`"rust"`) maps to `None`, never to a default. +- **The skip rule.** `skips()` must equal the old `_TEST_FILE_PATTERN` on every name, including `a.TEST.tsx`, `a.spec.jsx` and `a.test.pyi` (Task 3, parametrised against the old regex copied into the test). + +--- + +### Task 1: Move the modules (no behaviour change) + +**Files:** +- Move: `src/cgis/extractors/base.py` → `src/cgis/core/extractor.py` +- Move: `src/cgis/extractors/python_extractor.py` → `src/cgis/languages/python/extractor.py` +- Move: `src/cgis/extractors/_python_{ast,annotations,classes,functions,imports,types}.py` → `src/cgis/languages/python/_{syntax,annotations,classes,functions,imports,types}.py` +- Move: `src/cgis/extractors/typescript_extractor.py` → `src/cgis/languages/typescript/extractor.py` +- Move: `src/cgis/extractors/_typescript_bindings.py` → `src/cgis/languages/typescript/_bindings.py` +- Modify (imports only): every importer in `src/`, `scripts/`, `tests/` +- Modify: `tests/self_parsing/test_architecture.py`, `tests/self_parsing/forbidden_edges_allowlist.json`, `tests/self_parsing/test_receiver_resolution.py:210-212`, `docs/ontology/patterns.yaml:117-132`, `docs/ontology/domains.yaml:5-12`, `docs/ontology/tolerances.lock` (only if the measurement allows lowering) + +`src/cgis/extractors/registry.py` stays where it is until Task 2. Its imports are rewritten here. + +**Interfaces:** +- Produces: + - `cgis.core.extractor.BaseExtractor` and `cgis.core.extractor.ModuleNamer`, unchanged; + - `cgis.languages.python.extractor.PythonExtractor` and `.file_path_to_module_fqn`; + - `cgis.languages.python._syntax.PYTHON_LANG`; + - `cgis.languages.typescript.extractor.TypeScriptExtractor`, `.file_path_to_module_fqn` and `.TYPESCRIPT_LANG`. + +- [ ] **Step 1: Record the before-numbers on main.** Later steps compare against these. + +```bash +S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1 && mkdir -p $S && uv run --project /home/zaebee/projects/ownima/codegraph-brain cgis ingest /home/zaebee/projects/ownima/codegraph-brain/src --output $S/self-main.db && uv run --project /home/zaebee/projects/ownima/codegraph-brain cgis drift --db $S/self-main.db --profile python > $S/drift-main.txt +grep -n "extraction" $S/drift-main.txt +``` + +Expected: an `extraction` line with a drift score at or below 0.40, plus its unresolved ratio. Write both numbers down; the Step 6 commit message carries them. + +- [ ] **Step 2: Make the self-parse guard able to fail.** Invariant 2 (`test_extractors_are_database_blind`) matches on `_clean(n.id) == "extractors"`. After the move nothing matches, and the test would pass vacuously. Add the guard first, while the constant still reads `"extractors"`. Edit `tests/self_parsing/test_architecture.py`. + +Add `_EXTRACTION = "extractors"` next to `_CGIS`, and at the start of `test_extractors_are_database_blind` add: + +```python + assert any(_clean(n.id) == _EXTRACTION for n in nodes), ( + f"no '{_EXTRACTION}' node in the self-parsed graph — the invariant would examine nothing" + ) +``` + +Then replace the two `"extractors"` literals in that test (the `_is_imports(...)` call and the `_clean(n.id) == "extractors"` check) with `_EXTRACTION`. + +Run: `uv run pytest tests/self_parsing/test_architecture.py -q -k database_blind` +Expected: PASS. The guard holds on today's tree. + +- [ ] **Step 3: Move the files with git.** + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && mkdir -p src/cgis/languages/python src/cgis/languages/typescript && git mv src/cgis/extractors/base.py src/cgis/core/extractor.py && git mv src/cgis/extractors/python_extractor.py src/cgis/languages/python/extractor.py && git mv src/cgis/extractors/_python_ast.py src/cgis/languages/python/_syntax.py && git mv src/cgis/extractors/_python_annotations.py src/cgis/languages/python/_annotations.py && git mv src/cgis/extractors/_python_classes.py src/cgis/languages/python/_classes.py && git mv src/cgis/extractors/_python_functions.py src/cgis/languages/python/_functions.py && git mv src/cgis/extractors/_python_imports.py src/cgis/languages/python/_imports.py && git mv src/cgis/extractors/_python_types.py src/cgis/languages/python/_types.py && git mv src/cgis/extractors/typescript_extractor.py src/cgis/languages/typescript/extractor.py && git mv src/cgis/extractors/_typescript_bindings.py src/cgis/languages/typescript/_bindings.py +``` + +- [ ] **Step 4: Rewrite the imports.** Only `from`/`import` lines get rewritten. A bare string can be synthetic: `tests/unit/test_python_extractor.py:226` expects `"src.cgis.extractors.base"`, which is what parsing a made-up path yields, and it must stay. Then the two files whose strings are real FQNs get rewritten in full. + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && uv run python - <<'PY' +import re +import subprocess +from pathlib import Path + +MOVES = [ + (r"cgis\.extractors\.base\b", "cgis.core.extractor"), + (r"cgis\.extractors\._python_ast\b", "cgis.languages.python._syntax"), + (r"cgis\.extractors\._python_(annotations|classes|functions|imports|types)\b", r"cgis.languages.python._\1"), + (r"cgis\.extractors\.python_extractor\b", "cgis.languages.python.extractor"), + (r"cgis\.extractors\._typescript_bindings\b", "cgis.languages.typescript._bindings"), + (r"cgis\.extractors\.typescript_extractor\b", "cgis.languages.typescript.extractor"), +] +IMPORT = re.compile(r"^\s*(from|import) cgis\.extractors\.") +FULL = {"tests/self_parsing/forbidden_edges_allowlist.json", "tests/self_parsing/test_receiver_resolution.py"} + + +def move(line: str) -> str: + for pattern, new in MOVES: + line = re.sub(pattern, new, line) + return line + + +files = subprocess.run(["git", "ls-files", "src", "scripts", "tests"], capture_output=True, text=True, check=True).stdout.split() +changed = 0 +for name in files: + if not name.endswith((".py", ".json")) or name.startswith("tests/fixtures/"): + continue + path = Path(name) + old = path.read_text(encoding="utf-8") + lines = old.splitlines(keepends=True) + new = "".join(move(l) if (name in FULL or IMPORT.match(l)) else l for l in lines) + if new != old: + path.write_text(new, encoding="utf-8") + changed += 1 +print("files changed:", changed) +PY +git grep -nE "^\s*(from|import) cgis\.extractors\.(base|python_extractor|typescript_extractor|_python_|_typescript_bindings)" -- src scripts tests +``` + +Expected: +- `files changed:` prints about 50; +- the `git grep` prints nothing (exit 1 is expected here); +- `git diff tests/self_parsing/forbidden_edges_allowlist.json tests/self_parsing/test_receiver_resolution.py` shows the keys and FQNs as `cgis.languages.python.extractor...` and `cgis.languages.typescript.extractor...`; +- `tests/unit/test_python_extractor.py:226` is untouched. + +Then list what is left: + +```bash +git grep -n "cgis\.extractors\|cgis/extractors" -- src scripts tests +``` + +Expected leftovers, all deliberate: +- docstrings and comments in `src/` that name an old path. Fix each one that describes where code lives now; leave the ones that record history (`#344`, `#506`); +- `src/cgis/extractors/registry.py`'s own module docstring; +- the importers of `cgis.extractors.registry` (Task 2 replaces them); +- `tests/self_parsing/test_architecture.py` `RULES` (Task 2); +- synthetic strings in `tests/unit/test_drift*.py`, `test_cli.py`, `test_gen_ideal_graph.py`, `test_python_extractor.py:23,226,230`, `test_review_fingerprint_closure.py:221` and `agent_ab_stubs.py:109`. These are made-up FQNs and module paths fed to parsers or scorers; no real module stands behind them. Leave them as they are. + +Anything else is a missed import: fix it. + +- [ ] **Step 5: Re-point the self-graph gates.** + +In `tests/self_parsing/test_architecture.py`: +- `_EXTRACTION = "languages"`. +- In `_FORBIDDEN_FROM_CORE`, replace `"extractors"` with `"languages"`. +- The module docstring says "Six invariants" and lists six. Make it "Seven invariants", add `7. Forbidden edges — exact allowlists per rule (language-packs spec).`, and change item 2 to `Extractor Boundary — cgis.languages.* never imports storage, query, api or pipeline.` +- In `_KNOWN_GOD_OBJECTS`, replace `"extractors.typescript_extractor.TypeScriptExtractor"` with `"languages.typescript.extractor.TypeScriptExtractor"`. Keep the comment. + +In `docs/ontology/patterns.yaml`, `extraction` domain: set `fqn_prefix: "cgis.languages"`. Add one comment line above it: `# Re-pointed in language-packs step 1: the extractors moved into cgis.languages.; the contract (BaseExtractor) moved to cgis.core.` + +In `docs/ontology/domains.yaml`, `ParserLayer`: +- `file_path_patterns: ["*languages/*"]`; +- `fqn_patterns: ["*.Extractor*", "*languages.*"]`. + +- [ ] **Step 6: Measure the re-pointed domain.** + +```bash +S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1 && uv run cgis ingest /home/zaebee/projects/ownima/cgb-step1/src --output $S/self-branch.db && uv run cgis drift --db $S/self-branch.db --profile python > $S/drift-branch.txt +grep -n "extraction" $S/drift-main.txt $S/drift-branch.txt +``` + +Run both commands from the worktree. + +Expected: an `extraction` line in both files. +- If the branch drift is ≤ 0.40 and the unresolved ratio is ≤ 0.37, keep both lock values. +- If drift fell by more than 0.05, lower `drift_tolerance` to `measured + 0.03`, rounded up to two places, in both `patterns.yaml` and `tolerances.lock`. +- **If drift is > 0.40 or the unresolved ratio is > 0.37, stop and report the numbers.** The ratchet forbids raising either. + +- [ ] **Step 7: Run the full gate and the identity check.** + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1 && make format > /dev/null && make lint > $S/lint.log 2>&1 && make type-check > $S/mypy.log 2>&1 && make pytest > $S/pytest.log 2>&1 && make doc-coverage > $S/doc.log 2>&1 && git diff --exit-code main -- tests/fixtures/graph_identity/snapshot.json && echo GATE-OK +tail -3 $S/pytest.log +``` + +Expected: `GATE-OK`, with the same passed count as main plus the one Step 2 guard. + +If `test_forbidden_edges_match_the_allowlist_exactly` fails, compare its "Actual" output with the allowlist: +- only renamed keys, same counts: correct the JSON; +- any count change: stop and investigate. + +- [ ] **Step 8: Commit.** + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && git add -A && git commit -q -F - <<'MSG' && git log --oneline -1 +refactor(languages): move the extractors into cgis.languages., the contract into core + +Pure move (git mv) plus import rewrites; the identity snapshot is unchanged. +BaseExtractor/ModuleNamer -> cgis.core.extractor; _python_ast -> _syntax (an `_ast` +module shadows the stdlib one). Self-graph gates re-pointed: Invariant 2 now has a +guard that fails when it examines no node; extraction domain -> cgis.languages +(drift -> , unresolved -> ). + +Co-Authored-By: Claude Opus 5.5 +MSG +``` + +Fill the ``/`` placeholders in the message from Step 6's output before running this. + +--- + +### Task 2: The lazy registry and the two packs + +**Files:** +- Create: `src/cgis/languages/__init__.py` (registry) +- Create: `src/cgis/languages/python/__init__.py`, `src/cgis/languages/typescript/__init__.py` (packs) +- Create: `tests/unit/test_languages.py` (ports `tests/unit/test_extractor_registry.py` and adds the new tests) +- Delete: `src/cgis/extractors/registry.py`, `tests/unit/test_extractor_registry.py`, and the now-empty `src/cgis/extractors/` +- Modify: + - `src/cgis/cli.py:39-40,205-208,838-848` + - `src/cgis/api/mcp_server.py:24-25,60-64,472` + - `src/cgis/api/auto_refresh.py:26,81` + - `src/cgis/guardian/collector.py:16,129,175-176,214-219,365` + - `src/cgis/guardian/martian.py:23` + - `src/cgis/query/render/mermaid.py:6,77-79` + - `scripts/graph_identity.py:21,123` + - `scripts/generate_health.py:18,24,80` + - `scripts/guardian_martian.py:38,301-304` + - the five tests that call `build_extractors` + - `tests/self_parsing/test_architecture.py` (`RULES`) + - `tests/self_parsing/forbidden_edges_allowlist.json` + +**Interfaces:** +- Consumes (Task 1): `cgis.core.extractor.BaseExtractor`, `PythonExtractor`, `TypeScriptExtractor`, both `file_path_to_module_fqn`, `PYTHON_LANG`, `TYPESCRIPT_LANG`, `cgis.core.paths.is_test_path`. +- Produces, in `cgis.languages`: + - `LanguagePack` (Protocol) + - `Manifest` + - `MANIFESTS: tuple[Manifest, ...]` + - `is_supported(path: str) -> bool` + - `pack_for_path(path: str) -> LanguagePack | None` + - `pack_named(name: str) -> LanguagePack | None` + - `pack_for_node(node: Node) -> LanguagePack | None` + - `scope_of(path: str, source_root: str | None) -> str` + - `skips(filename: str) -> bool` + - `extractors(source_roots: list[str]) -> Mapping[str, BaseExtractor]` + + Each pack module also exposes `PACK`. + +- [ ] **Step 1: Write the failing tests.** Create `tests/unit/test_languages.py`: + +```python +"""Tests for the language registry (#344, language-packs step 1). + +The registry exists because several sites had to agree about what a file +extension means and did not. The tests that matter hold those sites together — +above all `TestModuleFqnRoundTrip`, which ingests real source and asserts the +registry derives the ids the extractors actually emitted. +""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +from cgis.core.models import Node, NodeType +from cgis.core.paths import is_test_path +from cgis.languages import ( + MANIFESTS, + Manifest, + extractors, + is_supported, + pack_for_node, + pack_for_path, + pack_named, + scope_of, +) +from cgis.pipeline import IngestionPipeline + +TS_SOURCE = """ +export function helper(value: number): number { + return value * 2; +} +""" + +TSX_SOURCE = """ +export const Widget = () => { + return null; +}; +""" + +PY_SOURCE = """ +def helper(value: int) -> int: + return value * 2 +""" + + +def _ingest(tmp_path: Path, files: dict[str, str]) -> list[Node]: + for relative, source in files.items(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + nodes, _, _ = IngestionPipeline(extractors([])).run(str(tmp_path)) + return nodes + + +class TestPackForPath: + """The lookup every caller must branch on rather than defaulting.""" + + @pytest.mark.parametrize( + ("path", "name"), + [ + ("src/cgis/pipeline.py", "python"), + ("src/app/handler.ts", "typescript"), + ("src/app/Widget.tsx", "typescript"), + ("src/app/types.d.ts", "typescript"), + ("src\\app\\handler.ts", "typescript"), + ], + ) + def test_recognises_supported_extensions(self, path: str, name: str) -> None: + pack = pack_for_path(path) + assert pack is not None + assert pack.name == name + assert is_supported(path) + + @pytest.mark.parametrize("path", ["README.md", "src/app/old.js", "src/app/old.jsx", "Makefile"]) + def test_returns_none_for_anything_without_an_extractor(self, path: str) -> None: + """.js and .jsx especially: the TS FQN helper strips them, but nothing parses them.""" + assert pack_for_path(path) is None + assert not is_supported(path) + + +class TestCaseSensitivity: + """The registry must agree with the ingest pipeline, including where both say no.""" + + def test_an_uppercase_extension_is_unsupported_because_ingest_skips_it( + self, tmp_path: Path + ) -> None: + """`Foo.TS` is never parsed; calling it supported would recreate #344's silent miss.""" + nodes = _ingest(tmp_path, {"Foo.TS": TS_SOURCE}) + assert [n for n in nodes if n.type is NodeType.FILE] == [] + assert pack_for_path("Foo.TS") is None + + +class TestPacks: + """Each pack is what its manifest declares, and answers the step-1 contract.""" + + @pytest.mark.parametrize("manifest", MANIFESTS, ids=lambda m: m.name) + def test_a_pack_is_what_its_manifest_says(self, manifest: Manifest) -> None: + pack = pack_named(manifest.name) + assert pack is not None + assert (pack.name, pack.extensions) == (manifest.name, manifest.extensions) + + def test_an_unknown_language_has_no_pack(self) -> None: + assert pack_named("rust") is None + node = Node( + id="m.f", type=NodeType.FUNCTION, name="f", file_path="m.rs", + start_line=1, end_line=1, language="rust", + ) # fmt: skip + assert pack_for_node(node) is None + + @pytest.mark.parametrize( + ("path", "fence"), [("a.py", "python"), ("a.ts", "ts"), ("a.tsx", "tsx")] + ) + def test_fence_matches_the_file(self, path: str, fence: str) -> None: + pack = pack_for_path(path) + assert pack is not None + assert pack.code_fence(path) == fence + + @pytest.mark.parametrize( + "path", + ["tests/a.py", "src/test_a.py", "src/a.py", "tests/a.ts", "src/a.test.ts", "contest/a.ts"], + ) + def test_is_test_is_the_core_rule_for_both_packs(self, path: str) -> None: + """Ruling R1: the flag stays `is_test_path`; the Node validator still calls it.""" + for manifest in MANIFESTS: + pack = pack_named(manifest.name) + assert pack is not None + assert pack.is_test(path) is is_test_path(path) + + def test_every_node_names_the_pack_that_reads_its_file(self, tmp_path: Path) -> None: + """`mermaid.py` finds the pack by `Node.language`; that must agree with the path.""" + nodes = _ingest( + tmp_path, {"pkg/mod.py": PY_SOURCE, "app/h.ts": TS_SOURCE, "app/W.tsx": TSX_SOURCE} + ) + extracted = [n for n in nodes if n.file_path in {"pkg/mod.py", "app/h.ts", "app/W.tsx"}] + assert {n.file_path for n in extracted} == {"pkg/mod.py", "app/h.ts", "app/W.tsx"} + for node in extracted: + assert pack_for_node(node) is pack_for_path(node.file_path), node.id + + +class TestExtractors: + """The pipeline's mapping comes from the registry, so it cannot drift from it.""" + + def test_covers_every_registered_extension(self) -> None: + assert set(extractors([])) == {ext for m in MANIFESTS for ext in m.extensions} + + def test_source_roots_reach_the_extractors(self) -> None: + """A root that does not reach the extractor silently changes every FQN.""" + nodes, _ = extractors(["src"])[".ts"].parse(TS_SOURCE, "src/app/handler.ts") + assert nodes[0].id == "app.handler" + + def test_tsx_gets_the_tsx_grammar(self) -> None: + """Ruling R2: one pack, two grammars, chosen by extension.""" + jsx = b"const a =
;\n" + found = extractors([]) + assert not found[".tsx"]._parser.parse(jsx).root_node.has_error # noqa: SLF001 + assert found[".ts"]._parser.parse(jsx).root_node.has_error # noqa: SLF001 + + def test_membership_and_iteration_build_nothing(self) -> None: + """Run in a fresh interpreter: this process has long since loaded both grammars.""" + script = ( + "import sys\n" + "from cgis.languages import extractors\n" + "m = extractors([])\n" + "assert '.ts' in m and '.tsx' in m and len(list(m)) == 3\n" + "print(sorted(x for x in sys.modules if x.startswith('tree_sitter_')))\n" + ) + out = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=False + ) + assert out.returncode == 0, out.stderr + assert out.stdout.strip() == "[]" + + +class TestModuleFqnRoundTrip: + """The acceptance test of #344: derive the id the extractor really emitted.""" + + @staticmethod + def _files(tmp_path: Path, files: dict[str, str]) -> dict[str, str]: + nodes = _ingest(tmp_path, files) + return {n.file_path: n.id for n in nodes if n.type is NodeType.FILE} + + def test_typescript(self, tmp_path: Path) -> None: + ingested = self._files( + tmp_path, + {"app/handler.ts": TS_SOURCE, "app/Widget.tsx": TSX_SOURCE, "components/index.tsx": TSX_SOURCE}, + ) + assert ingested, "ingest produced no FILE nodes — the fixture is wrong, not the code" + for relative, node_id in ingested.items(): + assert scope_of(relative, None) == node_id + + def test_barrel_file_collapses_like_the_extractor_does(self, tmp_path: Path) -> None: + ingested = self._files(tmp_path, {"components/index.tsx": TSX_SOURCE}) + assert ingested["components/index.tsx"] == "components" + + def test_python(self, tmp_path: Path) -> None: + ingested = self._files(tmp_path, {"pkg/mod.py": PY_SOURCE, "pkg/__init__.py": ""}) + assert ingested + for relative, node_id in ingested.items(): + assert scope_of(relative, None) == node_id + + def test_the_python_rule_would_have_failed_this(self, tmp_path: Path) -> None: + """Pins the #344 defect: the Python rule leaves `.ts` on.""" + ingested = self._files(tmp_path, {"app/handler.ts": TS_SOURCE}) + python = pack_named("python") + assert python is not None + assert python.scope_of("app/handler.ts", None) == "app.handler.ts" + assert python.scope_of("app/handler.ts", None) != ingested["app/handler.ts"] + + def test_a_path_no_pack_reads_gets_the_python_rule(self) -> None: + """Ruling R5: the CLI's extensionless fallback, kept in one place.""" + assert scope_of("src/cgis/pipeline", None) == "src.cgis.pipeline" +``` + +- [ ] **Step 2: Run them and watch them fail.** + +Run: `uv run pytest tests/unit/test_languages.py -q` +Expected: collection error, `ImportError: cannot import name 'MANIFESTS' from 'cgis.languages'` (`cgis.languages` is still a namespace package). + +- [ ] **Step 3: Write the registry.** Create `src/cgis/languages/__init__.py`: + +```python +"""The language registry: which files cgis reads, and which pack reads them (#344, #146). + +One manifest per pack declares everything needed before a file is met — name, +module, extensions, the test files the walk never reads — so nothing imports a +pack until a file of its language turns up. Surfaces and `query` import this +module, never `cgis.languages.` (the `surface-packs` gate). + +Adding a language is a manifest here plus a package beside this file that +exposes `PACK`. + +Case-sensitive on purpose: `IngestionPipeline._get_extractor` matches +`filename.endswith(ext)` against these lowercase extensions, so `Foo.TS` is +never parsed. Calling it supported here would derive an FQN that matches no +node — the silent miss #344 removed. `TestCaseSensitivity` pins the agreement. +""" + +import importlib +import re +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from functools import cache +from pathlib import PurePosixPath +from typing import Protocol + +from cgis.core.extractor import BaseExtractor +from cgis.core.models import Node + + +class LanguagePack(Protocol): + """What one language contributes (spec: The `LanguagePack` contract; step 1 of it).""" + + name: str + extensions: tuple[str, ...] + + def code_fence(self, path: str) -> str: + """Markdown fence label for this file's full-text context.""" + ... + + def is_test(self, path: str) -> bool: + """Whether this file is test code (the `Node.is_test` flag, not the walk's skip).""" + ... + + def scope_of(self, path: str, source_root: str | None) -> str: + """The namespace unit this file's definitions live in — its module FQN for py/ts.""" + ... + + def extractor(self, extension: str, source_roots: list[str]) -> BaseExtractor: + """A new extractor for files of `extension`.""" + ... + + +@dataclass(frozen=True) +class Manifest: + """What the registry knows about a pack without importing it.""" + + name: str #: equals `Node.language` for everything the pack emits + module: str #: imported on first use; must define `PACK` + extensions: tuple[str, ...] #: matched against the path suffix, dot included + never_ingested: re.Pattern[str] #: this language's test files the walk skips outright + + +MANIFESTS: tuple[Manifest, ...] = ( + Manifest( + name="python", + module="cgis.languages.python", + extensions=(".py",), + never_ingested=re.compile(r"\.(test|spec)\.py$", re.IGNORECASE), + ), + Manifest( + name="typescript", + module="cgis.languages.typescript", + extensions=(".ts", ".tsx"), + # .js/.jsx have no pack, but the walk always skipped their test files; + # kept so `skips` stays exactly the pattern it replaced. + never_ingested=re.compile(r"\.(test|spec)\.(ts|tsx|js|jsx)$", re.IGNORECASE), + ), +) + +_BY_EXTENSION: dict[str, Manifest] = {ext: m for m in MANIFESTS for ext in m.extensions} +_BY_NAME: dict[str, Manifest] = {m.name: m for m in MANIFESTS} + + +def _suffix(path: str) -> str: + return PurePosixPath(path.replace("\\", "/")).suffix + + +@cache +def _load(name: str) -> LanguagePack: + """Import a pack's module and return its `PACK` — once per process.""" + pack: LanguagePack = importlib.import_module(_BY_NAME[name].module).PACK + return pack + + +def is_supported(path: str) -> bool: + """Whether some pack reads this path. Imports no pack.""" + return _suffix(path) in _BY_EXTENSION + + +def pack_for_path(path: str) -> LanguagePack | None: + """The pack for a path's extension, or None — callers branch, never default.""" + manifest = _BY_EXTENSION.get(_suffix(path)) + return None if manifest is None else _load(manifest.name) + + +def pack_named(name: str) -> LanguagePack | None: + """The pack whose `Node.language` is `name`, or None.""" + return _load(name) if name in _BY_NAME else None + + +def pack_for_node(node: Node) -> LanguagePack | None: + """The pack that emitted `node`, by `Node.language` — never by re-sniffing the path.""" + return pack_named(node.language) + + +def scope_of(path: str, source_root: str | None) -> str: + """A path's module FQN by its own pack. + + A path no pack reads (an extensionless `src/cgis/pipeline`) gets the Python + rule, which is then a plain slash-to-dot rewrite — the fallback the CLI has + always used, kept here so it lives in one place. + """ + pack = pack_for_path(path) or _load("python") + return pack.scope_of(path, source_root) + + +def skips(filename: str) -> bool: + """Whether the walk never reads this file (`foo.test.ts`, `foo.spec.py`).""" + return any(m.never_ingested.search(filename) for m in MANIFESTS) + + +class _LazyExtractors(Mapping[str, BaseExtractor]): + """Extension → extractor, each built (and its grammar loaded) on first lookup.""" + + def __init__(self, source_roots: list[str]) -> None: + self._roots = list(source_roots) + self._built: dict[str, BaseExtractor] = {} + + def __getitem__(self, extension: str) -> BaseExtractor: + if extension not in self._built: + manifest = _BY_EXTENSION[extension] # KeyError for an unknown one, as a dict + self._built[extension] = _load(manifest.name).extractor(extension, self._roots) + return self._built[extension] + + def __contains__(self, extension: object) -> bool: + return extension in _BY_EXTENSION # Mapping's default would build it + + def __iter__(self) -> Iterator[str]: + return iter(_BY_EXTENSION) + + def __len__(self) -> int: + return len(_BY_EXTENSION) + + +def extractors(source_roots: list[str]) -> Mapping[str, BaseExtractor]: + """The extension → extractor mapping `IngestionPipeline` takes, built lazily.""" + return _LazyExtractors(source_roots) +``` + +- [ ] **Step 4: Write the two packs.** Create `src/cgis/languages/python/__init__.py`: + +```python +"""The Python language pack: tree-sitter-python, module FQNs from file paths.""" + +from cgis.core.extractor import BaseExtractor +from cgis.core.paths import is_test_path +from cgis.languages.python._syntax import PYTHON_LANG +from cgis.languages.python.extractor import PythonExtractor, file_path_to_module_fqn + + +class PythonPack: + """`LanguagePack` for `.py`.""" + + name: str = PYTHON_LANG + extensions: tuple[str, ...] = (".py",) + + def code_fence(self, path: str) -> str: + """Every Python file fences as `python`.""" + return "python" + + def is_test(self, path: str) -> bool: + """The core rule (`tests/`, `test_*.py`, `conftest.py`, …) — ruling R1.""" + return is_test_path(path) + + def scope_of(self, path: str, source_root: str | None) -> str: + """The module FQN the extractor gives this file's FILE node.""" + return file_path_to_module_fqn(path, source_root) + + def extractor(self, extension: str, source_roots: list[str]) -> BaseExtractor: + """One grammar for the one extension.""" + return PythonExtractor(source_roots=source_roots) + + +PACK = PythonPack() +``` + +Create `src/cgis/languages/typescript/__init__.py`: + +```python +"""The TypeScript language pack: `.ts` and `.tsx`, one grammar each.""" + +from cgis.core.extractor import BaseExtractor +from cgis.core.paths import is_test_path +from cgis.languages.typescript.extractor import ( + TYPESCRIPT_LANG, + TypeScriptExtractor, + file_path_to_module_fqn, +) + + +class TypeScriptPack: + """`LanguagePack` for `.ts`/`.tsx` — the extension picks the grammar (ruling R2).""" + + name: str = TYPESCRIPT_LANG + extensions: tuple[str, ...] = (".ts", ".tsx") + + def code_fence(self, path: str) -> str: + """`tsx` for `.tsx`, `ts` for the rest.""" + return "tsx" if path.endswith(".tsx") else "ts" + + def is_test(self, path: str) -> bool: + """The core rule — `tests/` segments; `foo.test.ts` never reaches a node (R1).""" + return is_test_path(path) + + def scope_of(self, path: str, source_root: str | None) -> str: + """The module FQN the extractor gives this file's FILE node (`/index` collapses).""" + return file_path_to_module_fqn(path, source_root) + + def extractor(self, extension: str, source_roots: list[str]) -> BaseExtractor: + """The TSX grammar for `.tsx`, the TypeScript one otherwise.""" + return TypeScriptExtractor(tsx=extension == ".tsx", source_roots=source_roots) + + +PACK = TypeScriptPack() +``` + +If `PYTHON_LANG` or `TYPESCRIPT_LANG` is not importable from those modules, check with `git grep -n "PYTHON_LANG =\|TYPESCRIPT_LANG ="`. Import it from where it is defined, and ledger the path. + +- [ ] **Step 5: Run the new tests.** + +Run: `uv run pytest tests/unit/test_languages.py -q` +Expected: all pass. If `test_an_unknown_language_has_no_pack` fails while building the `Node`, read `Node`'s required fields in `src/cgis/core/models.py` and adjust the constructor arguments only. + +- [ ] **Step 6: Switch every consumer to the registry.** + - **`src/cgis/cli.py`:** + - Replace lines 39-40 with `from cgis.languages import extractors, is_supported, scope_of`. + - In `ingest`, replace `extractors = build_extractors(roots)` and `IngestionPipeline(extractors, ...)` with `pipeline = IngestionPipeline(extractors(roots), domains_config=domains)`. + - At about line 842, keep the `if` and replace the `language`/`to_fqn` lines with `target = scope_of(normalized, None)`. The comment becomes: `# Per-language, because the rules disagree (#344); a path no pack reads gets the Python slash-to-dot rewrite (registry scope_of).` + - Confirm `git grep -n "file_path_to_module_fqn\|language_for\|build_extractors" src/cgis/cli.py` prints nothing. + - **`src/cgis/api/mcp_server.py`:** + - Delete lines 24-25 and the `_EXTRACTORS` dict at 60-64. + - Add `from cgis.languages import extractors`. + - At line 472, use `pipeline = IngestionPipeline(extractors([]))`. + - Confirm that `git grep -n "_EXTRACTORS" src/cgis/api/mcp_server.py` prints nothing. + - **`src/cgis/api/auto_refresh.py`:** `from cgis.languages import extractors`; `IngestionPipeline(extractors(source_roots), domains_config=domains)`. + - **`src/cgis/guardian/collector.py`:** + - `from cgis.languages import is_supported, pack_for_path`. + - At 175-176: `pack = pack_for_path(rel_path)` then `fence = pack.code_fence(rel_path) if pack else ""`. + - At 214-219: `pack = pack_for_path(rel_path)`; keep the existing None branch; then `module_fqn = pack.scope_of(rel_path, self.source_root)`. + - **`src/cgis/guardian/martian.py`:** `from cgis.languages import is_supported`. + - **`src/cgis/query/render/mermaid.py`:** + - `from cgis.languages import pack_for_node`. + - At 75-79: `pack = pack_for_node(node)`, `if pack is not None:`, `path_fqn = pack.scope_of(node.file_path, None)`. + - Comment: `# The path FQN by the node's own pack (#504): the Python rule keeps .ts on.` + - **`scripts/graph_identity.py`:** `from cgis.languages import extractors`; `IngestionPipeline(extractors([]))`. + - **`scripts/generate_health.py`:** + - Delete the `PythonExtractor` import and `_EXTRACTORS`. + - Add `from cgis.languages import extractors`; `IngestionPipeline(extractors([]))`. + - It ingests `src/cgis`, which holds no `.ts` files: `git ls-files src | grep -cE "\.tsx?$"` prints 0. So all languages ingest the same graph as Python alone did. + - **`scripts/guardian_martian.py`:** + - `from cgis.languages import is_supported, pack_for_path`. + - At 301-304: `pack = pack_for_path(path)`; keep the None branch; `store.get_node(pack.scope_of(path, SOURCE_ROOT))`. + - **Tests:** + - Run `git grep -ln "build_extractors" -- tests`. + - In each listed file, import `from cgis.languages import extractors` and replace `build_extractors(` with `extractors(`. Rename any local variable named `extractors` that would now shadow the import. + - Run `git grep -n "language_for\|LANGUAGES\b\|cgis.extractors.registry" -- tests src scripts`. Expected: only `tests/unit/agent_ab_stubs.py:109`, which is fake tool output; leave it. + - **Delete:** `git rm -q src/cgis/extractors/registry.py tests/unit/test_extractor_registry.py`, then `/bin/rm -rf src/cgis/extractors` (only `__pycache__` remains). + +- [ ] **Step 7: Close the gate's `cgis.extractors` entries.** In `tests/self_parsing/test_architecture.py` `RULES`: + - `pack-isolation` sources become `("cgis.languages",)`; + - `surface-packs` targets become `("cgis.languages",)` and its `exempt_modules` becomes `frozenset({"cgis.languages"})`. + + In `forbidden_edges_allowlist.json`, `"surface-packs"` becomes `{}`. + +Run: `uv run pytest tests/self_parsing/test_architecture.py -q` +Expected: PASS. The surface-to-pack edges are gone because every surface now imports `cgis.languages`. If the test reports any `surface-packs` or `cross-pack` entry, a consumer still imports a pack module: fix the import, not the allowlist. + +- [ ] **Step 8: Full gate, identity check, commit.** + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1 && make format > /dev/null && make lint > $S/lint.log 2>&1 && make type-check > $S/mypy.log 2>&1 && make pytest > $S/pytest.log 2>&1 && make doc-coverage > $S/doc.log 2>&1 && git diff --exit-code main -- tests/fixtures/graph_identity/snapshot.json && git add -A && git commit -q -F - <<'MSG' && git log --oneline -1 +feat(languages): lazy registry and python/typescript packs replace extractors/registry + +cgis.languages holds one manifest per pack and imports a pack only when a file of +its language is met; extractors(roots) is a Mapping that builds each extractor on +first lookup. CLI, MCP server, auto-refresh, guardian, mermaid and the scripts go +through the registry; the surface-packs allowlist is empty. Rulings R1-R5 of the +step-1 plan: a pack's is_test is the node flag (the walk's skip is the manifest's +never_ingested), TS answers per extension, the contract lives in core. + +Co-Authored-By: Claude Opus 5.5 +MSG +``` + +Expected: the commit lands, so every gate passed. If anything failed, read the log, fix it, and rerun the whole line. + +--- + +### Task 3: The pipeline asks the registry, and Python-only ingests stay TypeScript-free + +**Files:** +- Modify: `src/cgis/pipeline.py:447-456` (`_TEST_FILE_PATTERN`, `_get_extractor`) +- Modify: `src/cgis/workspaces.py:111-121,132,179`, `src/cgis/tsconfig_paths.py:81-91,113` +- Test: `tests/unit/test_languages.py` (append) + +**Interfaces:** +- Consumes (Task 2): `cgis.languages.skips`, `cgis.languages.extractors`. + +- [ ] **Step 1: Write the failing tests.** Append to `tests/unit/test_languages.py`: + +```python +import re + +from cgis.languages import skips + +# The pipeline's pattern before step 1, copied so `skips` is held to it exactly. +_OLD_SKIP = re.compile(r"\.(test|spec)\.(py|ts|tsx|js|jsx)$", re.IGNORECASE) + + +@pytest.mark.parametrize( + "name", + [ + "a.test.py", "a.spec.ts", "a.TEST.tsx", "a.spec.jsx", "a.test.js", "a.test.TS", + "a_test.py", "test_a.py", "a.py", "a.ts", "contest.ts", "a.test.pyi", "a.spec.d.ts", + ], +) # fmt: skip +def test_skips_is_the_pipeline_pattern_it_replaced(name: str) -> None: + assert skips(name) is bool(_OLD_SKIP.search(name)) + + +def _cli_ingest_loads(tmp_path: Path, files: dict[str, str]) -> list[str]: + """Grammar modules a fresh `cgis ingest` of `files` imported, in a clean interpreter.""" + repo = tmp_path / "repo" + for relative, source in files.items(): + (repo / relative).parent.mkdir(parents=True, exist_ok=True) + (repo / relative).write_text(source, encoding="utf-8") + script = ( + "import sys\n" + "from typer.testing import CliRunner\n" + "from cgis.cli import app\n" + f"r = CliRunner().invoke(app, ['ingest', {str(repo)!r}, '--output', {str(tmp_path / 'g.db')!r}])\n" + "assert r.exit_code == 0, r.output\n" + "import json\n" + "print(json.dumps(sorted(m for m in sys.modules if m.startswith('tree_sitter_'))))\n" + ) + out = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=False) + assert out.returncode == 0, out.stderr + loaded: list[str] = json.loads(out.stdout.strip().splitlines()[-1]) # last line: ours + return loaded + + +def test_a_python_only_ingest_never_loads_the_typescript_grammar(tmp_path: Path) -> None: + """Spec, Registry: loading every grammar eagerly is what the registry exists to avoid.""" + assert _cli_ingest_loads(tmp_path, {"pkg/a.py": PY_SOURCE}) == ["tree_sitter_python"] + + +def test_a_typescript_file_does_load_it(tmp_path: Path) -> None: + """The control: without it, the test above would pass on a probe that sees nothing.""" + loaded = _cli_ingest_loads(tmp_path, {"pkg/a.py": PY_SOURCE, "app/h.ts": TS_SOURCE}) + assert "tree_sitter_typescript" in loaded +``` + +Move the `re` and `json` imports and `skips` to the file's import block when you paste this. ruff will flag them otherwise. The pipeline logs through structlog, which may write to the real stdout around `CliRunner`. That is why the probe reads only its own last line. + +- [ ] **Step 2: Run them and watch the right ones fail.** + +Run: `uv run pytest tests/unit/test_languages.py -q -k "skips or grammar or does_load"` +Expected: +- `test_skips_*` PASS. The registry already has `skips`. This test pins equality before the pipeline starts relying on it. +- `test_a_typescript_file_does_load_it` PASS. +- `test_a_python_only_ingest_never_loads_the_typescript_grammar` FAILS, with `['tree_sitter_python', 'tree_sitter_typescript']`. Two causes: `_get_extractor` iterates `.items()`, and `WorkspacePackages`/`TsconfigPaths` call `extractors.get(".ts")` in `__init__`. + +If it fails with any other module list, find the importer with `uv run python -X importtime -c "from cgis.cli import app" 2>&1 | grep tree_sitter_typescript` and add that site to Step 3. + +- [ ] **Step 3: Make the pipeline and the TS config readers lazy.** + +In `src/cgis/pipeline.py`: +- delete `_TEST_FILE_PATTERN`; +- add `from cgis.languages import skips` to the imports; +- delete `import re` if nothing else in the file uses it (`grep -n "re\." src/cgis/pipeline.py`); +- replace `_get_extractor` with: + +```python + def _get_extractor(self, filename: str) -> BaseExtractor | None: + """Return the registered extractor for the given filename, or None. + + Matches on keys and fetches only the hit: the registry's mapping builds an + extractor, grammar and all, on first lookup, so walking its values would load + every language for a repository that has one. + """ + if skips(filename): + return None + for ext in self._extractors: + if filename.endswith(ext): + return self._extractors[ext] + return None +``` + +In `src/cgis/workspaces.py` `WorkspacePackages`: +- `__init__` stores `self._extractors = extractors` and no longer computes `self._extractor`. Keep the docstring and add one sentence: `The extractor is looked up on first use, so a repository with no package.json never builds it.` +- Add `from functools import cached_property`, and a property: + +```python + @cached_property + def _extractor(self) -> ModuleNamer | None: + """The `.ts` extractor if it can name modules — fetched only when a manifest needs it.""" + extractor = self._extractors.get(".ts") or self._extractors.get(".tsx") + return extractor if isinstance(extractor, ModuleNamer) else None +``` + +- In the method at :179 that returns the package map, change the guard to `if not self._claims or self._extractor is None:`. With no claims the loop already returns `{}`, so the output is unchanged. +- `note()` keeps `if self._extractor is None: return`. It only runs for a `package.json`. + +In `src/cgis/tsconfig_paths.py` `TsconfigPaths`, make the same change: +- store `self._extractors`; +- a `cached_property` `_namer` with the same body; +- `aliases()` guard: `if not self._configs or self._namer is None: return {}`. + +- [ ] **Step 4: Run the tests.** + +Run: `uv run pytest tests/unit/test_languages.py tests/unit/test_pipeline.py tests/unit/test_workspaces.py tests/unit/test_tsconfig_paths.py tests/unit/test_configs_above_root.py -q` +Expected: all pass. If one of those files does not exist, `ls tests/unit | grep -i "workspace\|tsconfig\|configs_above"` gives the right names. + +Mutation check: put `.items()` back in `_get_extractor` for one run. Expected: `test_a_python_only_ingest_never_loads_the_typescript_grammar` fails. Restore it, and confirm `git diff --stat` shows only the intended files. + +- [ ] **Step 5: Full gate, identity check, commit.** + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1 && make format > /dev/null && make lint > $S/lint.log 2>&1 && make type-check > $S/mypy.log 2>&1 && make pytest > $S/pytest.log 2>&1 && make doc-coverage > $S/doc.log 2>&1 && git diff --exit-code main -- tests/fixtures/graph_identity/snapshot.json && git add -A && git commit -q -F - <<'MSG' && git log --oneline -1 +perf(pipeline): a Python-only ingest no longer loads the TypeScript grammar + +The walk's test-file skip comes from the registry manifests (skips), the +pipeline fetches only the extractor a file needs, and the workspace/tsconfig +readers ask for the TS extractor only when a package.json or tsconfig exists. +Pinned by a fresh-interpreter CLI ingest, with a TS control. + +Co-Authored-By: Claude Opus 5.5 +MSG +``` + +--- + +### Task 4: Documentation and the small debts + +**Files:** +- Modify: `CLAUDE.md` (Architecture block, the FQN-format paragraph, "Adding a New Language Extractor") +- Modify: `docs/specs/2026-10-14-language-packs-design.md:313` (the CLI line flagged in the #596 review) +- Modify: `.github/workflows/ci.yml` (drop the no-op `-Dsonar.exclusions=tests/fixtures/**`) + +- [ ] **Step 1: CLAUDE.md.** + - In the Architecture block, replace `├── BaseExtractor subclasses (extractors/) ← language-specific AST → Nodes+Edges` and `│ └── PythonExtractor ← only language implemented so far` with: + +``` + ├── cgis.languages (registry) ← lazy: a pack is imported when its first file is met + │ ├── languages/python/ ← PythonExtractor + PACK + │ └── languages/typescript/ ← TypeScriptExtractor (.ts/.tsx) + PACK +``` + + - In the FQN-format paragraph, change the helper reference to `cgis.languages.scope_of(path, source_root)` (or a pack's `scope_of`). Change the example path `extractors/python_extractor.py` to `languages/python/extractor.py`. + - Replace the "Adding a New Language Extractor" section with: + +```markdown +## Adding a Language Pack + +1. Create `src/cgis/languages//` with an extractor subclassing `cgis.core.extractor.BaseExtractor` + (`parse(code, file_path) -> tuple[list[Node], list[Edge]]`; emit `CALLS` edges with + `target="raw_call:"`, set `Node.language = ""`). +2. In its `__init__.py`, define `PACK`, satisfying `cgis.languages.LanguagePack` + (`name`, `extensions`, `code_fence`, `is_test`, `scope_of`, `extractor`). +3. Add one `Manifest` to `MANIFESTS` in `src/cgis/languages/__init__.py`. Nothing else names the pack: + CLI, MCP server, guardian and scripts all go through the registry, and the `surface-packs` + forbidden-edge gate fails if a surface imports a pack directly. +4. A pack never imports another pack or `cgis.storage` (`cross-pack`, `pack-isolation` gates). +``` + +- [ ] **Step 2: The spec line.** In `docs/specs/2026-10-14-language-packs-design.md:313`, change `` `scripts/graph_identity.py ` diffs two ingests `` to `` `scripts/graph_identity.py diff ` diffs two ingests (`snapshot ` records one) ``. + +- [ ] **Step 3: ci.yml.** + - Delete the line `-Dsonar.exclusions=tests/fixtures/**`. `tests/fixtures` sits under `sonar.tests`, so only `sonar.test.exclusions` applies to it (#596 review). + - In the comment above, change "is excluded outright" to "is excluded from test analysis (`sonar.test.exclusions`; it sits under `sonar.tests`)". + - Run `python3 -c "import yaml; yaml.safe_load(open('.github/workflows/ci.yml'))"`. + +- [ ] **Step 4: Gate and commit.** + +```bash +cd /home/zaebee/projects/ownima/cgb-step1 && S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1 && make format > /dev/null && make lint > $S/lint.log 2>&1 && make type-check > $S/mypy.log 2>&1 && make pytest > $S/pytest.log 2>&1 && make doc-coverage > $S/doc.log 2>&1 && git add -A && git commit -q -F - <<'MSG' && git log --oneline -1 +docs: CLAUDE.md describes language packs; spec CLI line; drop a no-op Sonar flag + +Co-Authored-By: Claude Opus 5.5 +MSG +``` + +--- + +### Task 5: Identity report, review, PR + +- [ ] **Step 1: Ingest the same inputs with main's cgis and the branch's cgis**, the two-extractors-one-tree comparison the spec requires. + +```bash +S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1/identity && MAIN=/home/zaebee/projects/ownima/codegraph-brain && BR=/home/zaebee/projects/ownima/cgb-step1 && /bin/rm -rf $S && mkdir -p $S/cgis $S/oa && git -C $MAIN rev-parse --short HEAD && git -C $MAIN archive 715cd9a src | tar -x -C $S/cgis && git -C /home/zaebee/projects/ownima/owner-api archive 8de2245 ownima-backend | tar -x -C $S/oa && uv run --project $MAIN cgis ingest $S/cgis/src --output $S/cgis-main.db > /dev/null && uv run --project $BR cgis ingest $S/cgis/src --output $S/cgis-branch.db > /dev/null && uv run --project $MAIN cgis ingest $S/oa/ownima-backend --output $S/oa-main.db > /dev/null && uv run --project $BR cgis ingest $S/oa/ownima-backend --output $S/oa-branch.db > /dev/null && echo INGESTED +``` + +Expected: +- the first line is main's HEAD and equals `git rev-parse --short main`; if it doesn't, pull main first; +- then `INGESTED`. + +Each `--project` runs that checkout's editable cgis, so the two databases come from two different extractors reading one tree. + +- [ ] **Step 2: Diff them.** + +```bash +S=/tmp/claude-1001/-home-zaebee-projects-ownima-codegraph-brain/97c475b5-6bc7-4231-a643-ff81f3f651ec/scratchpad/step1/identity && cd /home/zaebee/projects/ownima/cgb-step1 && uv run python scripts/graph_identity.py diff $S/cgis-main.db $S/cgis/src $S/cgis-branch.db $S/cgis/src && uv run python scripts/graph_identity.py diff $S/oa-main.db $S/oa/ownima-backend $S/oa-branch.db $S/oa/ownima-backend +``` + +Expected: `identical` twice, exit 0. Any diff line is a behaviour change: stop and investigate before the PR. + +- [ ] **Step 3: Independent review.** Dispatch a reviewer with execution tools on the most capable model. Give it the branch diff against `main`, this plan, the spec, the Rulings section and the Review Focus list. Fix Critical/Important findings test-first, then rerun the full gate and Step 2. + +- [ ] **Step 4: Open the PR.** Push `feat/language-packs-step1`. Title: `refactor(languages): step 1 — lazy registry, python/typescript packs`. + +The body carries: +- what moved, with the module map from Task 1; +- the Rulings R1–R7, one line each; +- the identity report from Step 2, both repos, with main's and the branch's SHAs; +- the extraction-domain numbers before and after (Task 1 Step 6); +- the `surface-packs` allowlist going from 3 entries to 0; +- the Python-only acceptance test and its control. + +End the body with `🤖 Generated with [Claude Code](https://claude.com/claude-code)`. Then check the Gemini and Guardian comments and the Sonar result, and reply in every thread. diff --git a/docs/specs/2026-10-14-language-packs-design.md b/docs/specs/2026-10-14-language-packs-design.md index 3552f9b4..410a33b3 100644 --- a/docs/specs/2026-10-14-language-packs-design.md +++ b/docs/specs/2026-10-14-language-packs-design.md @@ -310,8 +310,8 @@ cgis reading refactored source. from step 0 on. **The Python/TS snapshot never changes, step 7 included.** Go gets a corpus and snapshot of its own, so the pilot cannot quietly move the existing languages. -- `scripts/graph_identity.py ` diffs two - ingests. Every PR in steps 1–5 reports it in its description, for cgis +- `scripts/graph_identity.py diff ` diffs two + ingests (`snapshot ` records one). Every PR in steps 1–5 reports it in its description, for cgis `src/` at 715cd9a and owner-api @ 8de2245, with `main`'s cgis and the branch's cgis over the same input. diff --git a/scripts/generate_health.py b/scripts/generate_health.py index 78df3b7c..58b21408 100644 --- a/scripts/generate_health.py +++ b/scripts/generate_health.py @@ -15,13 +15,12 @@ from dataclasses import dataclass from pathlib import Path -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore _SRC = Path("src/cgis") _OUTPUT = Path("docs/architecture/health_badge.json") -_EXTRACTORS = {".py": PythonExtractor()} @dataclass @@ -77,7 +76,7 @@ def generate_health(db: str | None = None) -> None: with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: db_path = tmp.name try: - pipeline = IngestionPipeline(_EXTRACTORS) + pipeline = IngestionPipeline(extractors([])) with SQLiteStore(db_path) as store: pipeline.run(str(_SRC), store=store) result = _compute_health(db_path) diff --git a/scripts/graph_identity.py b/scripts/graph_identity.py index 04089412..8f59c355 100644 --- a/scripts/graph_identity.py +++ b/scripts/graph_identity.py @@ -18,7 +18,7 @@ from pathlib import Path from typing import cast -from cgis.extractors.registry import build_extractors +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore @@ -120,7 +120,7 @@ def diff(before: Mapping[str, object], after: Mapping[str, object]) -> list[str] def _ingest(root: Path, db: Path) -> None: """One full ingest of `root` into `db`, as `cgis ingest` does it, ingest state included.""" with SQLiteStore(str(db)) as store: - pipeline = IngestionPipeline(build_extractors([])) + pipeline = IngestionPipeline(extractors([])) pipeline.run(str(root), store=store) store.record_ingest(str(root), pipeline.observed_mtimes) store.record_ingest_options([], None) diff --git a/scripts/guardian_martian.py b/scripts/guardian_martian.py index 11a7b9bd..dcade6c1 100644 --- a/scripts/guardian_martian.py +++ b/scripts/guardian_martian.py @@ -35,7 +35,6 @@ from pydantic import ValidationError -from cgis.extractors.registry import is_supported, language_for from cgis.guardian.calibrate import ( DEFAULT_JUDGE_CONCURRENCY, assign_matches, @@ -81,6 +80,7 @@ build_skeptic_provider, temperature_setting, ) +from cgis.languages import is_supported, pack_for_path from cgis.storage.sqlite_store import SQLiteStore CORPUS_DIR = Path("benchmarks/martian") @@ -298,10 +298,10 @@ def graph_alignment(db_path: Path, changed_files: Sequence[str]) -> tuple[int, i with SQLiteStore(str(db_path)) as store: found = 0 for path in supported: - language = language_for(path) - if language is None: # pragma: no cover - filtered above + pack = pack_for_path(path) + if pack is None: # pragma: no cover - filtered above continue - if store.get_node(language.file_path_to_module_fqn(path, SOURCE_ROOT)): + if store.get_node(pack.scope_of(path, SOURCE_ROOT)): found += 1 return found, len(supported) diff --git a/src/cgis/api/auto_refresh.py b/src/cgis/api/auto_refresh.py index 6556a6c4..f5d3d190 100644 --- a/src/cgis/api/auto_refresh.py +++ b/src/cgis/api/auto_refresh.py @@ -23,7 +23,7 @@ import structlog from cgis.core.freshness import FreshnessState -from cgis.extractors.registry import build_extractors +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore @@ -78,7 +78,7 @@ def _refresh(db_path: str) -> bool: ) return False started = time.perf_counter() - pipeline = IngestionPipeline(build_extractors(source_roots), domains_config=domains) + pipeline = IngestionPipeline(extractors(source_roots), domains_config=domains) nodes, _raw, _resolved = pipeline.run(root, store=store) if nodes: store.record_ingest(root, pipeline.observed_mtimes) diff --git a/src/cgis/api/mcp_server.py b/src/cgis/api/mcp_server.py index 2f3fc9bc..d58d6873 100644 --- a/src/cgis/api/mcp_server.py +++ b/src/cgis/api/mcp_server.py @@ -21,8 +21,7 @@ from cgis.core.coverage import TraversalCoverage from cgis.core.freshness import Freshness, FreshnessState from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.python_extractor import PythonExtractor -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.query.analysis.overview import DEFAULT_DEPTH, DEFAULT_LIMIT, build_overview from cgis.query.analysis.suggest_service import report_to_dict, suggest_packages @@ -57,11 +56,6 @@ mcp = CgisServer("cgis-code-graph", instructions=SERVER_INSTRUCTIONS, version=__version__) -_EXTRACTORS = { - ".py": PythonExtractor(), - ".ts": TypeScriptExtractor(), - ".tsx": TypeScriptExtractor(tsx=True), -} _DEFAULT_DB = "graph.db" # Agents choose arguments from the JSON schema, and the SDK does not copy @@ -469,7 +463,7 @@ def cgis_ingest( logger.warning("MCP ingest refused db_path", db=db_path) return refusal - pipeline = IngestionPipeline(_EXTRACTORS) + pipeline = IngestionPipeline(extractors([])) try: with SQLiteStore(db_path) as store: # A rebuild replaces the stored graph in the transaction that writes the diff --git a/src/cgis/cli.py b/src/cgis/cli.py index 2a466364..796f3e78 100644 --- a/src/cgis/cli.py +++ b/src/cgis/cli.py @@ -36,9 +36,8 @@ from cgis.core.coverage import TraversalCoverage from cgis.core.freshness import FreshnessState from cgis.core.models import VIRTUAL_FILE_PATH, Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import file_path_to_module_fqn -from cgis.extractors.registry import build_extractors, is_supported, language_for from cgis.guardian.metrics import load_reviews, rate_review +from cgis.languages import extractors, is_supported, scope_of from cgis.pipeline import IngestionPipeline from cgis.query.analysis.analyzer import AnalyzerEngine from cgis.query.analysis.anomaly import AnomalyType, ArchitecturalAnomaly @@ -203,9 +202,7 @@ def ingest( Scan a repository, extract code structure, and resolve semantic links. """ roots = source_roots or [] - extractors = build_extractors(roots) - - pipeline = IngestionPipeline(extractors, domains_config=domains) + pipeline = IngestionPipeline(extractors(roots), domains_config=domains) if domains and not Path(domains).is_file(): console.print( @@ -836,16 +833,13 @@ def structure( raise typer.Exit(code=1) _warn_if_not_fresh(db) - # Normalize file path → FQN. Per-language, because the helpers disagree: + # Normalize file path → FQN. Per-language, because the rules disagree: # the Python one strips ".py" and collapses "/__init__", so it would turn # "src/app/foo.ts" into "src.app.foo.ts" — an id no extractor emits (#344). + # A path no pack reads gets the Python slash-to-dot rewrite (`scope_of`). if "/" in target or "\\" in target or is_supported(target): normalized = target.replace("\\", "/").removeprefix("./") - language = language_for(normalized) - # An extensionless path ("src/cgis/pipeline") has no language; the - # Python helper is then just a slash-to-dot rewrite, which is correct. - to_fqn = language.file_path_to_module_fqn if language else file_path_to_module_fqn - target = to_fqn(normalized, None) + target = scope_of(normalized, None) console.print(f"[dim]→ FQN: {escape(target)}[/dim]") with SQLiteStore(db) as store: diff --git a/src/cgis/extractors/base.py b/src/cgis/core/extractor.py similarity index 66% rename from src/cgis/extractors/base.py rename to src/cgis/core/extractor.py index 37680977..7b589bbd 100644 --- a/src/cgis/extractors/base.py +++ b/src/cgis/core/extractor.py @@ -1,4 +1,8 @@ -"""Base Extractor Interface""" +"""The extractor contract: `BaseExtractor`, `ModuleNamer`, and the `LanguagePack` a pack exposes. + +In core, not beside the packs: the registry imports each pack and each pack +subclasses the protocol, so the protocol living in the registry would be a cycle. +""" from abc import ABC, abstractmethod from typing import Protocol, runtime_checkable @@ -56,3 +60,26 @@ def parse(self, code: str, file_path: str) -> tuple[list[Node], list[Edge]]: Returns: A tuple containing (list_of_nodes, list_of_edges). """ + + +class LanguagePack(Protocol): + """What one language contributes (spec: The `LanguagePack` contract; step 1 of it).""" + + name: str + extensions: tuple[str, ...] + + def code_fence(self, path: str) -> str: + """Markdown fence label for this file's full-text context.""" + ... + + def is_test(self, path: str) -> bool: + """Whether this file is test code (the `Node.is_test` flag, not the walk's skip).""" + ... + + def scope_of(self, path: str, source_root: str | None) -> str: + """The namespace unit this file's definitions live in — its module FQN for py/ts.""" + ... + + def extractor(self, extension: str, source_roots: list[str]) -> BaseExtractor: + """A new extractor for files of `extension`.""" + ... diff --git a/src/cgis/extractors/.gitkeep b/src/cgis/extractors/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cgis/extractors/registry.py b/src/cgis/extractors/registry.py deleted file mode 100644 index 200b1179..00000000 --- a/src/cgis/extractors/registry.py +++ /dev/null @@ -1,93 +0,0 @@ -"""One place that says which extensions cgis understands, and how (#344). - -Before this module the answer was spread over four sites that had to agree and -did not: `cli.py` registered the extractors, `ContextCollector` filtered changed -files by `.py`, fenced every file as ```python, and derived FQNs with the -*Python* helper. The last of those is the one that bites — the Python helper -strips `.py` and collapses `/__init__`, so handing it `src/app/foo.ts` yields -`src.app.foo.ts`, an id no extractor ever emitted. `GraphContextCollector.sections` then finds -nothing, logs at debug, and the review completes looking normal with no graph -context at all. - -Adding a language should touch this file and nothing else. -""" - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import PurePosixPath - -from cgis.extractors.base import BaseExtractor -from cgis.extractors.python_extractor import PythonExtractor -from cgis.extractors.python_extractor import file_path_to_module_fqn as python_module_fqn -from cgis.extractors.typescript_extractor import TypeScriptExtractor -from cgis.extractors.typescript_extractor import file_path_to_module_fqn as typescript_module_fqn - - -@dataclass(frozen=True) -class Language: - """Everything the ingest pipeline and the reviewer need for one extension.""" - - #: Matched against `PurePosixPath(path).suffix`, so it includes the dot. - extension: str - build_extractor: Callable[[list[str]], BaseExtractor] - #: MUST be the same function the extractor uses to build its FILE node id. - #: `test_module_fqn_round_trips_against_ingested_nodes` holds them together. - file_path_to_module_fqn: Callable[[str, str | None], str] - #: Markdown fence label for full-file context. - code_fence: str - - -LANGUAGES: tuple[Language, ...] = ( - Language( - extension=".py", - build_extractor=lambda roots: PythonExtractor(source_roots=roots), - file_path_to_module_fqn=python_module_fqn, - code_fence="python", - ), - Language( - extension=".ts", - build_extractor=lambda roots: TypeScriptExtractor(source_roots=roots), - file_path_to_module_fqn=typescript_module_fqn, - code_fence="ts", - ), - Language( - extension=".tsx", - build_extractor=lambda roots: TypeScriptExtractor(tsx=True, source_roots=roots), - file_path_to_module_fqn=typescript_module_fqn, - code_fence="tsx", - ), -) - -# `.js`/`.jsx` are deliberately absent. The TypeScript FQN helper strips them, -# but `LANGUAGES` registers no extractor for them, so the graph holds no JS -# nodes. Listing them here would let the collector derive a plausible FQN that -# matches nothing — reintroducing the silent-miss path from the other direction. - -_BY_EXTENSION: dict[str, Language] = {lang.extension: lang for lang in LANGUAGES} - - -def language_for(file_path: str) -> Language | None: - """The Language for a path's extension, or None when cgis cannot parse it. - - None is the "not our file" answer, and every caller must branch on it rather - than falling back to a default: a default is precisely how a TypeScript path - used to acquire a Python FQN. - - Case-sensitive on purpose, and it must stay that way while - `IngestionPipeline._get_extractor` is — it matches `filename.endswith(ext)` - against these same lowercase keys, so `Foo.TS` is never parsed and the graph - holds no node for it. Lowercasing here alone would call such a file - supported, derive an FQN for it, and miss silently — the exact failure this - module exists to remove. `TestCaseSensitivity` pins the agreement. - """ - return _BY_EXTENSION.get(PurePosixPath(file_path.replace("\\", "/")).suffix) - - -def is_supported(file_path: str) -> bool: - """Whether cgis has an extractor for this path's extension.""" - return language_for(file_path) is not None - - -def build_extractors(source_roots: list[str]) -> dict[str, BaseExtractor]: - """The extension → extractor mapping `IngestionPipeline` takes.""" - return {lang.extension: lang.build_extractor(source_roots) for lang in LANGUAGES} diff --git a/src/cgis/guardian/collector.py b/src/cgis/guardian/collector.py index f126149d..9ed43ada 100644 --- a/src/cgis/guardian/collector.py +++ b/src/cgis/guardian/collector.py @@ -13,8 +13,8 @@ import structlog -from cgis.extractors.registry import is_supported, language_for from cgis.guardian.chunker import Chunk +from cgis.languages import is_supported, pack_for_path from cgis.query.drift.drift import DriftScorer from cgis.query.drift.fingerprint import FingerprintExtractor from cgis.query.drift.quotient import build_quotient, quotient_call_tally @@ -172,8 +172,8 @@ def collect_full_files(self, files: list[str] | None = None) -> str: used += size # Per-language fence: the label was hardcoded "python", so a TS # file reached the model announced as Python (#344). - language = language_for(rel_path) - fence = language.code_fence if language else "" + pack = pack_for_path(rel_path) + fence = pack.code_fence(rel_path) if pack else "" sections.append(f"#### `{rel_path}`\n```{fence}\n{text}\n```") return "\n\n".join(sections + omitted) @@ -211,12 +211,12 @@ def sections(self, changed_files: list[str], *, flow: bool) -> tuple[list[str], with SQLiteStore(str(self.db_path)) as store: engine = QueryEngine(store) for rel_path in changed_files: - language = language_for(rel_path) - if language is None: + pack = pack_for_path(rel_path) + if pack is None: # Callers filter, so this means one stopped doing so. log.warning("No extractor for changed file.", path=rel_path) continue - module_fqn = language.file_path_to_module_fqn(rel_path, self.source_root) + module_fqn = pack.scope_of(rel_path, self.source_root) nodes, edges = engine.get_impact_graph(module_fqn, max_depth=2) title = "Impact graph" if not nodes and flow: diff --git a/src/cgis/guardian/martian.py b/src/cgis/guardian/martian.py index eb043cd2..ee9065fc 100644 --- a/src/cgis/guardian/martian.py +++ b/src/cgis/guardian/martian.py @@ -20,11 +20,11 @@ from pydantic import BaseModel, Field, model_validator -from cgis.extractors.registry import is_supported from cgis.guardian.calibrate import JudgePair, assign_from_grid from cgis.guardian.collector import FeaturesSource from cgis.guardian.findings import Finding from cgis.guardian.runner import TemperatureSource +from cgis.languages import is_supported #: Severity vocabulary of the upstream corpus. Four values, not three — the #: spec's §"Ground truth format" omits `Critical` (12 of the 173 comments). diff --git a/src/cgis/import_names.py b/src/cgis/import_names.py index d0985d90..661607b0 100644 --- a/src/cgis/import_names.py +++ b/src/cgis/import_names.py @@ -16,7 +16,7 @@ import structlog -from cgis.extractors.base import BaseExtractor +from cgis.core.extractor import BaseExtractor from cgis.npm_dependencies import NpmDependencies from cgis.python_dependencies import declared_import_roots from cgis.tsconfig_paths import TSCONFIG, TsconfigPaths diff --git a/src/cgis/languages/__init__.py b/src/cgis/languages/__init__.py new file mode 100644 index 00000000..95fe94ba --- /dev/null +++ b/src/cgis/languages/__init__.py @@ -0,0 +1,144 @@ +"""The language registry: which files cgis reads, and which pack reads them (#344, #146). + +One manifest per pack declares everything needed before a file is met — name, +module, extensions, the test files the walk never reads — so nothing imports a +pack until a file of its language turns up. Surfaces and `query` import this +module, never `cgis.languages.` (the `surface-packs` gate). + +Adding a language is a manifest here plus a package beside this file that +exposes `PACK`. Each manifest loads its pack with a static import inside a +function — lazy, yet visible to the review-fingerprint walk, which cannot follow +`importlib` (`test_no_dynamic_imports_in_the_review_path`). + +Case-sensitive on purpose: `IngestionPipeline._get_extractor` matches +`filename.endswith(ext)` against these lowercase extensions, so `Foo.TS` is +never parsed. Calling it supported here would derive an FQN that matches no +node — the silent miss #344 removed. `TestCaseSensitivity` pins the agreement. +""" + +import re +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from functools import cache +from pathlib import PurePosixPath + +from cgis.core.extractor import BaseExtractor, LanguagePack +from cgis.core.models import Node + + +@dataclass(frozen=True) +class Manifest: + """What the registry knows about a pack without importing it.""" + + name: str #: equals `Node.language` for everything the pack emits + load: Callable[[], LanguagePack] #: imports the pack; called once, on first use + extensions: tuple[str, ...] #: matched against the path suffix, dot included + never_ingested: re.Pattern[str] #: this language's test files the walk skips outright + + +def _python() -> LanguagePack: + from cgis.languages.python import PACK # noqa: PLC0415 — on the first .py file + + return PACK + + +def _typescript() -> LanguagePack: + from cgis.languages.typescript import PACK # noqa: PLC0415 — on the first .ts/.tsx file + + return PACK + + +MANIFESTS: tuple[Manifest, ...] = ( + Manifest( + name="python", + load=_python, + extensions=(".py",), + never_ingested=re.compile(r"\.(test|spec)\.py$", re.IGNORECASE), + ), + Manifest( + name="typescript", + load=_typescript, + extensions=(".ts", ".tsx"), + # .js/.jsx have no pack, but the walk always skipped their test files; + # kept so `skips` stays exactly the pattern it replaced. + never_ingested=re.compile(r"\.(test|spec)\.(ts|tsx|js|jsx)$", re.IGNORECASE), + ), +) + +_BY_EXTENSION: dict[str, Manifest] = {ext: m for m in MANIFESTS for ext in m.extensions} +_BY_NAME: dict[str, Manifest] = {m.name: m for m in MANIFESTS} + + +def _suffix(path: str) -> str: + return PurePosixPath(path.replace("\\", "/")).suffix + + +@cache +def _load(name: str) -> LanguagePack: + """Import a pack and return its `PACK` — once per process.""" + return _BY_NAME[name].load() + + +def is_supported(path: str) -> bool: + """Whether some pack reads this path. Imports no pack.""" + return _suffix(path) in _BY_EXTENSION + + +def pack_for_path(path: str) -> LanguagePack | None: + """The pack for a path's extension, or None — callers branch, never default.""" + manifest = _BY_EXTENSION.get(_suffix(path)) + return None if manifest is None else _load(manifest.name) + + +def pack_named(name: str) -> LanguagePack | None: + """The pack whose `Node.language` is `name`, or None.""" + return _load(name) if name in _BY_NAME else None + + +def pack_for_node(node: Node) -> LanguagePack | None: + """The pack that emitted `node`, by `Node.language` — never by re-sniffing the path.""" + return pack_named(node.language) + + +def scope_of(path: str, source_root: str | None) -> str: + """A path's module FQN by its own pack. + + A path no pack reads (an extensionless `src/cgis/pipeline`) gets the Python + rule, which is then a plain slash-to-dot rewrite — the fallback the CLI has + always used, kept here so it lives in one place. + """ + pack = pack_for_path(path) or _load("python") + return pack.scope_of(path, source_root) + + +def skips(filename: str) -> bool: + """Whether the walk never reads this file (`foo.test.ts`, `foo.spec.py`).""" + return any(m.never_ingested.search(filename) for m in MANIFESTS) + + +class _LazyExtractors(Mapping[str, BaseExtractor]): + """Extension → extractor, each built (and its grammar loaded) on first lookup.""" + + def __init__(self, source_roots: list[str]) -> None: + self._roots = list(source_roots) + self._built: dict[str, BaseExtractor] = {} + + def __getitem__(self, extension: str) -> BaseExtractor: + if extension not in self._built: + manifest = _BY_EXTENSION[extension] # KeyError for an unknown one, as a dict + self._built[extension] = _load(manifest.name).extractor(extension, self._roots) + return self._built[extension] + + def __contains__(self, extension: object) -> bool: + return extension in _BY_EXTENSION # Mapping's default would build it + + def __iter__(self) -> Iterator[str]: + return iter(_BY_EXTENSION) + + def __len__(self) -> int: + return len(_BY_EXTENSION) + + +def extractors(source_roots: list[str]) -> Mapping[str, BaseExtractor]: + """The extension → extractor mapping `IngestionPipeline` takes, built lazily.""" + return _LazyExtractors(source_roots) diff --git a/src/cgis/languages/python/__init__.py b/src/cgis/languages/python/__init__.py new file mode 100644 index 00000000..134422ab --- /dev/null +++ b/src/cgis/languages/python/__init__.py @@ -0,0 +1,38 @@ +"""The Python language pack: tree-sitter-python, module FQNs from file paths.""" + +from typing import override + +from cgis.core.extractor import BaseExtractor, LanguagePack +from cgis.core.paths import is_test_path +from cgis.languages.python._syntax import PYTHON_LANG +from cgis.languages.python.extractor import PythonExtractor, file_path_to_module_fqn + + +class PythonPack(LanguagePack): + """`LanguagePack` for `.py`.""" + + name: str = PYTHON_LANG + extensions: tuple[str, ...] = (".py",) + + @override + def code_fence(self, path: str) -> str: + """Every Python file fences as `python`.""" + return "python" + + @override + def is_test(self, path: str) -> bool: + """The core rule (`tests/`, `test_*.py`, `conftest.py`, …) — ruling R1.""" + return is_test_path(path) + + @override + def scope_of(self, path: str, source_root: str | None) -> str: + """The module FQN the extractor gives this file's FILE node.""" + return file_path_to_module_fqn(path, source_root) + + @override + def extractor(self, extension: str, source_roots: list[str]) -> BaseExtractor: + """One grammar for the one extension.""" + return PythonExtractor(source_roots=source_roots) + + +PACK = PythonPack() diff --git a/src/cgis/extractors/_python_annotations.py b/src/cgis/languages/python/_annotations.py similarity index 97% rename from src/cgis/extractors/_python_annotations.py rename to src/cgis/languages/python/_annotations.py index f3f2097b..3e9f1a70 100644 --- a/src/cgis/extractors/_python_annotations.py +++ b/src/cgis/languages/python/_annotations.py @@ -1,6 +1,6 @@ """Type-name collection from annotation subtrees (spec D9). -`clean_python_type_string` in `_python_types.py` answers "what type is this +`clean_python_type_string` in `_types.py` answers "what type is this variable" — it reduces `list[Node]` to `list`, which is correct for `local_types` (the receiver of `x.append()` really is a list). A reference edge asks the other question: which types does this annotation *name*. Measured on @@ -13,7 +13,7 @@ _STRING_PARSER = Parser(Language(tspython.language())) -# Mirrors `TypeResolver._GENERIC_WRAPPERS` in `_python_types.py` (kept as a +# Mirrors `TypeResolver._GENERIC_WRAPPERS` in `_types.py` (kept as a # separate constant rather than importing a private class attribute across # modules): these typing constructs are never themselves a type reference. # `Annotated[T, ...]` additionally carries non-type metadata (typically a diff --git a/src/cgis/extractors/_python_classes.py b/src/cgis/languages/python/_classes.py similarity index 99% rename from src/cgis/extractors/_python_classes.py rename to src/cgis/languages/python/_classes.py index 90a9ab0c..9d381848 100644 --- a/src/cgis/extractors/_python_classes.py +++ b/src/cgis/languages/python/_classes.py @@ -10,7 +10,7 @@ from tree_sitter import Node as BaseNode from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors._python_ast import ( +from cgis.languages.python._syntax import ( PYTHON_LANG, extract_node_name, get_id, diff --git a/src/cgis/extractors/_python_functions.py b/src/cgis/languages/python/_functions.py similarity index 99% rename from src/cgis/extractors/_python_functions.py rename to src/cgis/languages/python/_functions.py index ea72b3ca..c35cfa2e 100644 --- a/src/cgis/extractors/_python_functions.py +++ b/src/cgis/languages/python/_functions.py @@ -14,8 +14,8 @@ from tree_sitter import Node as BaseNode from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors._python_annotations import _WRAPPER_NAMES, collect_type_names -from cgis.extractors._python_ast import ( +from cgis.languages.python._annotations import _WRAPPER_NAMES, collect_type_names +from cgis.languages.python._syntax import ( PYTHON_LANG, assigned_attr_name, extract_node_name, @@ -23,7 +23,7 @@ get_identifier, is_method, ) -from cgis.extractors._python_types import TypeResolver +from cgis.languages.python._types import TypeResolver def _descendants_of_type(node: BaseNode, node_type: str) -> list[BaseNode]: diff --git a/src/cgis/extractors/_python_imports.py b/src/cgis/languages/python/_imports.py similarity index 99% rename from src/cgis/extractors/_python_imports.py rename to src/cgis/languages/python/_imports.py index 7a90dc85..3d5199e4 100644 --- a/src/cgis/extractors/_python_imports.py +++ b/src/cgis/languages/python/_imports.py @@ -7,7 +7,7 @@ from tree_sitter import Node as BaseNode from cgis.core.models import Edge, EdgeType -from cgis.extractors._python_ast import resolve_relative_module +from cgis.languages.python._syntax import resolve_relative_module def _has_wildcard(node: BaseNode) -> bool: diff --git a/src/cgis/extractors/_python_ast.py b/src/cgis/languages/python/_syntax.py similarity index 100% rename from src/cgis/extractors/_python_ast.py rename to src/cgis/languages/python/_syntax.py diff --git a/src/cgis/extractors/_python_types.py b/src/cgis/languages/python/_types.py similarity index 98% rename from src/cgis/extractors/_python_types.py rename to src/cgis/languages/python/_types.py index 7a21da8b..99550bb0 100644 --- a/src/cgis/extractors/_python_types.py +++ b/src/cgis/languages/python/_types.py @@ -2,7 +2,7 @@ from collections.abc import Callable -from cgis.extractors._python_ast import file_path_to_module_fqn +from cgis.languages.python._syntax import file_path_to_module_fqn class TypeResolver: diff --git a/src/cgis/extractors/python_extractor.py b/src/cgis/languages/python/extractor.py similarity index 98% rename from src/cgis/extractors/python_extractor.py rename to src/cgis/languages/python/extractor.py index 9a892b37..fd3527b8 100644 --- a/src/cgis/extractors/python_extractor.py +++ b/src/cgis/languages/python/extractor.py @@ -6,25 +6,25 @@ from tree_sitter import Language, Parser from tree_sitter import Node as BaseNode +from cgis.core.extractor import BaseExtractor from cgis.core.models import Edge, Node, NodeType -from cgis.extractors._python_ast import ( - enclosing_class_fqn, - extract_decorator_names, - is_module_level_assignment, - is_name_load, -) -from cgis.extractors._python_ast import file_path_to_module_fqn as _file_path_to_module_fqn -from cgis.extractors._python_classes import ClassHandler -from cgis.extractors._python_functions import ( +from cgis.languages.python._classes import ClassHandler +from cgis.languages.python._functions import ( FunctionHandler, collect_context_manager_type, collect_return_annotation, emit_annotation_edges, emit_name_reference_edges, ) -from cgis.extractors._python_imports import ImportHandler -from cgis.extractors._python_types import TypeResolver -from cgis.extractors.base import BaseExtractor +from cgis.languages.python._imports import ImportHandler +from cgis.languages.python._syntax import ( + enclosing_class_fqn, + extract_decorator_names, + is_module_level_assignment, + is_name_load, +) +from cgis.languages.python._syntax import file_path_to_module_fqn as _file_path_to_module_fqn +from cgis.languages.python._types import TypeResolver def file_path_to_module_fqn(file_path: str, source_root: str | None = None) -> str: diff --git a/src/cgis/languages/typescript/__init__.py b/src/cgis/languages/typescript/__init__.py new file mode 100644 index 00000000..599c483e --- /dev/null +++ b/src/cgis/languages/typescript/__init__.py @@ -0,0 +1,41 @@ +"""The TypeScript language pack: `.ts` and `.tsx`, one grammar each.""" + +from typing import override + +from cgis.core.extractor import BaseExtractor, LanguagePack +from cgis.core.paths import is_test_path +from cgis.languages.typescript.extractor import ( + TYPESCRIPT_LANG, + TypeScriptExtractor, + file_path_to_module_fqn, +) + + +class TypeScriptPack(LanguagePack): + """`LanguagePack` for `.ts`/`.tsx` — the extension picks the grammar (ruling R2).""" + + name: str = TYPESCRIPT_LANG + extensions: tuple[str, ...] = (".ts", ".tsx") + + @override + def code_fence(self, path: str) -> str: + """`tsx` for `.tsx`, `ts` for the rest.""" + return "tsx" if path.endswith(".tsx") else "ts" + + @override + def is_test(self, path: str) -> bool: + """The core rule — `tests/` segments; `foo.test.ts` never reaches a node (R1).""" + return is_test_path(path) + + @override + def scope_of(self, path: str, source_root: str | None) -> str: + """The module FQN the extractor gives this file's FILE node (`/index` collapses).""" + return file_path_to_module_fqn(path, source_root) + + @override + def extractor(self, extension: str, source_roots: list[str]) -> BaseExtractor: + """The TSX grammar for `.tsx`, the TypeScript one otherwise.""" + return TypeScriptExtractor(tsx=extension == ".tsx", source_roots=source_roots) + + +PACK = TypeScriptPack() diff --git a/src/cgis/extractors/_typescript_bindings.py b/src/cgis/languages/typescript/_bindings.py similarity index 100% rename from src/cgis/extractors/_typescript_bindings.py rename to src/cgis/languages/typescript/_bindings.py diff --git a/src/cgis/extractors/typescript_extractor.py b/src/cgis/languages/typescript/extractor.py similarity index 99% rename from src/cgis/extractors/typescript_extractor.py rename to src/cgis/languages/typescript/extractor.py index 39957e86..7a833645 100644 --- a/src/cgis/extractors/typescript_extractor.py +++ b/src/cgis/languages/typescript/extractor.py @@ -4,10 +4,10 @@ from tree_sitter import Language, Parser from tree_sitter import Node as TSNode +from cgis.core.extractor import BaseExtractor from cgis.core.js_globals import specifier_fqn from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors._typescript_bindings import bare_imports, shadowed_globals -from cgis.extractors.base import BaseExtractor +from cgis.languages.typescript._bindings import bare_imports, shadowed_globals _RAW_CALL_PREFIX = "raw_call:" #: `Node.language` for everything this extractor emits, `.ts` and `.tsx` alike; diff --git a/src/cgis/pipeline.py b/src/cgis/pipeline.py index d99b5815..22b6639e 100644 --- a/src/cgis/pipeline.py +++ b/src/cgis/pipeline.py @@ -3,7 +3,6 @@ import hashlib import json import os -import re from collections.abc import Mapping from pathlib import Path from typing import TYPE_CHECKING @@ -12,11 +11,12 @@ from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn +from cgis.core.extractor import BaseExtractor from cgis.core.generated import is_generated_source from cgis.core.models import Edge, EdgeType, Node, NodeType from cgis.core.paths import EXCLUDED_DIRS -from cgis.extractors.base import BaseExtractor from cgis.import_names import ImportNameCollector +from cgis.languages import skips from cgis.resolver.engine import ResolverEngine from cgis.resolver.symbols import dispatch_edges from cgis.resolver.uplift import SemanticUpliftEngine @@ -444,13 +444,16 @@ def _persist_incremental( for stale_path in stale_files: logger.info("Removed stale file from graph", file_path=stale_path) - _TEST_FILE_PATTERN = re.compile(r"\.(test|spec)\.(py|ts|tsx|js|jsx)$", re.IGNORECASE) - def _get_extractor(self, filename: str) -> BaseExtractor | None: - """Return the registered extractor for the given filename, or None.""" - if self._TEST_FILE_PATTERN.search(filename): + """Return the registered extractor for the given filename, or None. + + Matches on keys and fetches only the hit: the registry's mapping builds an + extractor, grammar and all, on first lookup, so walking its values would load + every language for a repository that has one. + """ + if skips(filename): return None - for ext, extractor in self._extractors.items(): + for ext in self._extractors: if filename.endswith(ext): - return extractor + return self._extractors[ext] return None diff --git a/src/cgis/query/render/mermaid.py b/src/cgis/query/render/mermaid.py index 961577ff..bfab3dec 100644 --- a/src/cgis/query/render/mermaid.py +++ b/src/cgis/query/render/mermaid.py @@ -3,7 +3,7 @@ import re from cgis.core.models import VIRTUAL_FILE_PATH, Edge, Node, NodeNamespace, NodeType -from cgis.extractors.registry import language_for +from cgis.languages import pack_for_path _RAW_CALL_PREFIX = "raw_call:" _UNRESOLVED_STYLE = ":::unresolvedNode" @@ -72,11 +72,14 @@ def _node_slug(node: Node) -> str: :func:`_fqn_slug`. Collisions are disambiguated later by :class:`_IdAllocator`. """ if node.namespace == NodeNamespace.INTERNAL and node.file_path != VIRTUAL_FILE_PATH: - # The path FQN by the file's own language: the Python helper keeps `.ts` - # on, so every TypeScript node used to miss and fall back (#504). - language = language_for(node.file_path) - if language is not None: - path_fqn = language.file_path_to_module_fqn(node.file_path, None) + # The path FQN by the pack that reads the file: the Python rule keeps `.ts` + # on, so every TypeScript node used to miss and fall back (#504). By path, + # not `Node.language`: a directory row (`cgis structure`) and a TS node + # stored before #513 both carry the "python" default, and neither should + # get the Python rule. + pack = pack_for_path(node.file_path) + if pack is not None: + path_fqn = pack.scope_of(node.file_path, None) slug = _internal_node_slug(node.id, path_fqn) if slug is not None: return slug diff --git a/src/cgis/resolver/js_builtins.py b/src/cgis/resolver/js_builtins.py index 59962b71..28322bfb 100644 --- a/src/cgis/resolver/js_builtins.py +++ b/src/cgis/resolver/js_builtins.py @@ -70,7 +70,7 @@ ) #: Extensions whose calls are JavaScript at runtime. Matches the extractors -#: registered in `extractors/registry.py` for `.ts`/`.tsx`. +#: registered in `cgis.languages` for `.ts`/`.tsx`. _JS_SUFFIXES: tuple[str, ...] = (".ts", ".tsx") diff --git a/src/cgis/tsconfig_paths.py b/src/cgis/tsconfig_paths.py index 63788d3f..51cd179d 100644 --- a/src/cgis/tsconfig_paths.py +++ b/src/cgis/tsconfig_paths.py @@ -23,11 +23,12 @@ import re from collections.abc import Mapping from dataclasses import dataclass, replace +from functools import cached_property from pathlib import Path import structlog -from cgis.extractors.base import BaseExtractor, ModuleNamer +from cgis.core.extractor import BaseExtractor, ModuleNamer logger = structlog.getLogger(__name__) @@ -83,13 +84,19 @@ def __init__(self, workspace_root: Path, extractors: Mapping[str, BaseExtractor] The extractor is the one registered for `.ts`, chosen by extension so this module imports no language's extractor (#506). With none configured, or one - that cannot name modules, every config is ignored. + that cannot name modules, every config is ignored. It is looked up on first + use, so a repository with no tsconfig.json never builds it. """ self._root = workspace_root - extractor = extractors.get(".ts") or extractors.get(".tsx") - self._namer = extractor if isinstance(extractor, ModuleNamer) else None + self._extractors = extractors self._configs: set[Path] = set() + @cached_property + def _namer(self) -> ModuleNamer | None: + """The `.ts` extractor if it can name modules — fetched only when a config needs it.""" + extractor = self._extractors.get(".ts") or self._extractors.get(".tsx") + return extractor if isinstance(extractor, ModuleNamer) else None + def note(self, config: Path) -> None: """Record a `tsconfig.json` the walk passed; it is read in `aliases`.""" self._configs |= {config} @@ -110,7 +117,7 @@ def aliases( `"extends": "@x/tsconfig/base.json"` finds a shared config that is not installed into `node_modules`. """ - if self._namer is None: + if not self._configs or self._namer is None: return {} reader = _Reader(self._root, self._namer, dict(package_directories or {})) scopes: dict[str, dict[str, list[str]]] = {} diff --git a/src/cgis/workspaces.py b/src/cgis/workspaces.py index 802cbdb5..1f39cfb1 100644 --- a/src/cgis/workspaces.py +++ b/src/cgis/workspaces.py @@ -8,12 +8,13 @@ import json from collections.abc import Iterable, Mapping +from functools import cached_property from pathlib import Path import structlog import yaml -from cgis.extractors.base import BaseExtractor, ModuleNamer +from cgis.core.extractor import BaseExtractor, ModuleNamer logger = structlog.getLogger(__name__) @@ -114,13 +115,20 @@ def __init__(self, workspace_root: Path, extractors: Mapping[str, BaseExtractor] The extractor is the one registered for `.ts`, chosen by extension rather than by class, so this module imports no language's extractor (#506). With none configured, or one that cannot name modules, every manifest is ignored. + The extractor is looked up on first use, so a repository with no + package.json never builds it. """ self._root = workspace_root - extractor = extractors.get(".ts") or extractors.get(".tsx") - self._extractor = extractor if isinstance(extractor, ModuleNamer) else None + self._extractors = extractors # package name as written -> every directory that claims it self._claims: dict[str, set[Path]] = {} + @cached_property + def _extractor(self) -> ModuleNamer | None: + """The `.ts` extractor if it can name modules — fetched only when a manifest needs it.""" + extractor = self._extractors.get(".ts") or self._extractors.get(".tsx") + return extractor if isinstance(extractor, ModuleNamer) else None + def note(self, manifest: Path) -> None: """Record which directory a `package.json` names. @@ -176,7 +184,7 @@ def unambiguous(self) -> dict[str, str]: directory FQN comes from the extractor's own `module_fqn`, so it is spelled exactly as that package's node ids are. """ - if self._extractor is None: + if not self._claims or self._extractor is None: return {} claims: dict[str, set[str]] = {} for name, found in sorted(self._claims.items()): diff --git a/tests/self_parsing/conftest.py b/tests/self_parsing/conftest.py index fddcfd62..d7150c77 100644 --- a/tests/self_parsing/conftest.py +++ b/tests/self_parsing/conftest.py @@ -6,8 +6,8 @@ import pytest from cgis.core.models import Edge, Node -from cgis.extractors.python_extractor import PythonExtractor -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages.python.extractor import PythonExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/self_parsing/forbidden_edges_allowlist.json b/tests/self_parsing/forbidden_edges_allowlist.json index 9c4150a0..3e056584 100644 --- a/tests/self_parsing/forbidden_edges_allowlist.json +++ b/tests/self_parsing/forbidden_edges_allowlist.json @@ -5,11 +5,7 @@ "resolver-storage": { "cgis.resolver.uplift -> cgis.storage.sqlite_store": 8 }, - "surface-packs": { - "cgis.api.mcp_server -> cgis.extractors.python_extractor": 3, - "cgis.api.mcp_server -> cgis.extractors.typescript_extractor": 4, - "cgis.cli -> cgis.extractors.python_extractor": 2 - }, + "surface-packs": {}, "surface-storage": { "cgis.api.auto_refresh -> cgis.storage.sqlite_store": 7, "cgis.api.mcp_server -> cgis.storage.sqlite_store": 20, diff --git a/tests/self_parsing/test_annotation_calibration.py b/tests/self_parsing/test_annotation_calibration.py index bff2ad6b..01751c85 100644 --- a/tests/self_parsing/test_annotation_calibration.py +++ b/tests/self_parsing/test_annotation_calibration.py @@ -6,7 +6,7 @@ """ from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.resolver.engine import ResolverEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/self_parsing/test_architecture.py b/tests/self_parsing/test_architecture.py index c860d52f..00e2bb3b 100644 --- a/tests/self_parsing/test_architecture.py +++ b/tests/self_parsing/test_architecture.py @@ -1,12 +1,13 @@ """Architectural guardrails: hexagonal boundary invariants verified via the self-parsed graph. -Six invariants are enforced: +Seven invariants are enforced: 1. Pure Domain Isolation — cgis.core.models never imports other cgis.* subpackages. - 2. Extractor Boundary — cgis.extractors.* never imports storage, query, or api layers. + 2. Extractor Boundary — cgis.languages.* never imports storage, query, api or pipeline. 3. Storage Boundary — cgis.storage.* never imports the api layer. 4. Model Immutability — Node and Edge declare frozen=True. 5. No Circular Dependencies — AnalyzerEngine finds zero cycles in cgis itself. 6. God Object Baseline — number of God Objects does not grow beyond known set. + 7. Forbidden edges — exact allowlists per rule (language-packs spec). """ import json @@ -22,8 +23,9 @@ # --------------------------------------------------------------------------- _CGIS = "cgis" +_EXTRACTION = "languages" _FORBIDDEN_API = frozenset({"api"}) -_FORBIDDEN_FROM_CORE = frozenset({"extractors", "storage", "query", "api", "resolver", "pipeline"}) +_FORBIDDEN_FROM_CORE = frozenset({"languages", "storage", "query", "api", "resolver", "pipeline"}) _FORBIDDEN_FROM_EXTRACTORS = frozenset({"storage", "query", "api", "pipeline"}) _CORE_NOT_FOUND = "ARCHITECTURAL VIOLATION — 'core.models' node not found in the parsed graph" @@ -80,15 +82,18 @@ def test_extractors_are_database_blind( ) -> None: """Extractors must only do AST parsing — zero knowledge of storage, query, or api.""" _, nodes, edges = graph_data + assert any(_clean(n.id) == _EXTRACTION for n in nodes), ( + f"no '{_EXTRACTION}' node in the self-parsed graph — the invariant would examine nothing" + ) violations = set() for e in edges: - is_imports = _is_imports(_clean(e.source), "extractors", e.type) + is_imports = _is_imports(_clean(e.source), _EXTRACTION, e.type) is_forbidden = _is_forbidden(e.target, _CGIS, _FORBIDDEN_FROM_EXTRACTORS) if is_imports and is_forbidden: violations.add(f"{e.source} -> {e.target}") for n in nodes: - if _clean(n.id) == "extractors": + if _clean(n.id) == _EXTRACTION: import_map = n.metadata.get("import_map") or {} for val in import_map.values(): if _is_forbidden(val, _CGIS, _FORBIDDEN_FROM_EXTRACTORS): @@ -184,7 +189,7 @@ def test_analyzer_report_is_deterministic( _KNOWN_GOD_OBJECTS = frozenset( { "storage.sqlite_store.SQLiteStore", # data-access layer — many CRUD methods by design - "extractors.typescript_extractor.TypeScriptExtractor", # complex AST visitor + "languages.typescript.extractor.TypeScriptExtractor", # complex AST visitor } ) @@ -234,12 +239,11 @@ class _Rule: RULES: tuple[_Rule, ...] = ( _Rule("cross-pack", ("cgis.languages",), ("cgis.languages",), exempt_modules=frozenset({"cgis.languages"}), cross_pack=True), - _Rule("pack-isolation", ("cgis.languages", "cgis.extractors"), + _Rule("pack-isolation", ("cgis.languages",), ("cgis.storage", "cgis.query", "cgis.api", "cgis.guardian", "cgis.pipeline")), _Rule("resolver-storage", ("cgis.resolver",), ("cgis.storage",)), _Rule("surface-packs", ("cgis.query", "cgis.guardian", "cgis.cli", "cgis.api"), - ("cgis.languages", "cgis.extractors"), - exempt_modules=frozenset({"cgis.languages", "cgis.extractors.registry"})), + ("cgis.languages",), exempt_modules=frozenset({"cgis.languages"})), _Rule("surface-storage", ("cgis.cli", "cgis.api", "cgis.guardian"), ("cgis.storage",)), _Rule("core-isolation", ("cgis.core",), ("cgis",), exempt_prefixes=("cgis.core",)), ) # fmt: skip diff --git a/tests/self_parsing/test_init_ontology_roundtrip.py b/tests/self_parsing/test_init_ontology_roundtrip.py index d221652d..27041dc8 100644 --- a/tests/self_parsing/test_init_ontology_roundtrip.py +++ b/tests/self_parsing/test_init_ontology_roundtrip.py @@ -19,7 +19,7 @@ def test_self_graph_proposal_round_trips( - every domain report has status != "empty" - every domain report satisfies drift_score <= tolerance + 1e-9 - sanity: some discovered fqn_prefix contains "query" - - sanity: some discovered fqn_prefix contains "extractors" + - sanity: some discovered fqn_prefix contains "languages" """ store, _, _ = root_graph_data db_path = store.db_path @@ -50,6 +50,6 @@ def test_self_graph_proposal_round_trips( assert any("query" in p for p in discovered_prefixes), ( f"No 'query' prefix found; discovered: {discovered_prefixes}" ) - assert any("extractors" in p for p in discovered_prefixes), ( - f"No 'extractors' prefix found; discovered: {discovered_prefixes}" + assert any("languages" in p for p in discovered_prefixes), ( + f"No 'languages' prefix found; discovered: {discovered_prefixes}" ) diff --git a/tests/self_parsing/test_receiver_resolution.py b/tests/self_parsing/test_receiver_resolution.py index 07567369..607f67ca 100644 --- a/tests/self_parsing/test_receiver_resolution.py +++ b/tests/self_parsing/test_receiver_resolution.py @@ -207,9 +207,9 @@ def test_named_receiver_calls_resolve_to_their_exact_targets( "cgis.resolver.engine.ResolverEngine._resolved_call_edge", "cgis.resolver.symbols.SymbolResolver.resolve_self_call", ), - ("cgis.extractors.python_extractor.PythonExtractor.parse", "tree_sitter.Parser.parse"), + ("cgis.languages.python.extractor.PythonExtractor.parse", "tree_sitter.Parser.parse"), ( - "cgis.extractors.typescript_extractor.TypeScriptExtractor.parse", + "cgis.languages.typescript.extractor.TypeScriptExtractor.parse", "tree_sitter.Parser.parse", ), } diff --git a/tests/self_parsing/test_self_parse.py b/tests/self_parsing/test_self_parse.py index ffc8310a..8584b05f 100644 --- a/tests/self_parsing/test_self_parse.py +++ b/tests/self_parsing/test_self_parse.py @@ -15,7 +15,7 @@ import pytest from cgis.core.models import VIRTUAL_FILE_PATH, Edge, EdgeType, Node, NodeNamespace -from cgis.extractors.python_extractor import file_path_to_module_fqn +from cgis.languages.python.extractor import file_path_to_module_fqn from cgis.storage.sqlite_store import SQLiteStore SRC_DIR = Path(__file__).parent.parent.parent / "src" / "cgis" @@ -67,9 +67,9 @@ def _fqn(relative: str, *parts: str) -> str: _fqn("resolver/symbols.py", "SymbolResolver"), _fqn("resolver/symbols.py", "SymbolResolver", "resolve_global_call"), # extractor - _fqn("extractors/python_extractor.py", "PythonExtractor"), - _fqn("extractors/python_extractor.py", "PythonExtractor", "parse"), - _fqn("extractors/python_extractor.py", "file_path_to_module_fqn"), + _fqn("languages/python/extractor.py", "PythonExtractor"), + _fqn("languages/python/extractor.py", "PythonExtractor", "parse"), + _fqn("languages/python/extractor.py", "file_path_to_module_fqn"), ] # fmt: on diff --git a/tests/self_parsing/test_self_parse_ts.py b/tests/self_parsing/test_self_parse_ts.py index caf111b8..ba24dc4b 100644 --- a/tests/self_parsing/test_self_parse_ts.py +++ b/tests/self_parsing/test_self_parse_ts.py @@ -10,7 +10,7 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.typescript_extractor import file_path_to_module_fqn +from cgis.languages.typescript.extractor import file_path_to_module_fqn from cgis.resolver.js_builtins import JS_BUILTINS_ROOT, JS_GLOBALS from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/review_path_inventory.txt b/tests/unit/review_path_inventory.txt index 76b2cde8..38cd96ca 100644 --- a/tests/unit/review_path_inventory.txt +++ b/tests/unit/review_path_inventory.txt @@ -1,14 +1,10 @@ src/cgis/__init__.py +src/cgis/core/coverage.py +src/cgis/core/extractor.py +src/cgis/core/freshness.py +src/cgis/core/js_globals.py src/cgis/core/models.py -src/cgis/extractors/_python_ast.py -src/cgis/extractors/_python_classes.py -src/cgis/extractors/_python_functions.py -src/cgis/extractors/_python_imports.py -src/cgis/extractors/_python_types.py -src/cgis/extractors/base.py -src/cgis/extractors/python_extractor.py -src/cgis/extractors/registry.py -src/cgis/extractors/typescript_extractor.py +src/cgis/core/paths.py src/cgis/guardian/__init__.py src/cgis/guardian/axes.py src/cgis/guardian/chunked.py @@ -16,6 +12,7 @@ src/cgis/guardian/chunker.py src/cgis/guardian/collector.py src/cgis/guardian/core.py src/cgis/guardian/diff_index.py +src/cgis/guardian/evidence.py src/cgis/guardian/findings.py src/cgis/guardian/github_poster.py src/cgis/guardian/metrics.py @@ -29,13 +26,27 @@ src/cgis/guardian/recording.py src/cgis/guardian/render.py src/cgis/guardian/runner.py src/cgis/guardian/skeptic.py +src/cgis/languages/__init__.py +src/cgis/languages/python/__init__.py +src/cgis/languages/python/_annotations.py +src/cgis/languages/python/_classes.py +src/cgis/languages/python/_functions.py +src/cgis/languages/python/_imports.py +src/cgis/languages/python/_syntax.py +src/cgis/languages/python/_types.py +src/cgis/languages/python/extractor.py +src/cgis/languages/typescript/__init__.py +src/cgis/languages/typescript/_bindings.py +src/cgis/languages/typescript/extractor.py src/cgis/query/analysis/__init__.py src/cgis/query/analysis/health.py src/cgis/query/drift/__init__.py src/cgis/query/drift/_scc.py +src/cgis/query/drift/catalog.py src/cgis/query/drift/drift.py src/cgis/query/drift/fingerprint.py src/cgis/query/drift/quotient.py +src/cgis/query/drift/scoring.py src/cgis/query/drift/triads.py src/cgis/query/engine.py src/cgis/query/render/__init__.py diff --git a/tests/unit/test_agent_ab_task.py b/tests/unit/test_agent_ab_task.py index 42d4ae20..30ede3eb 100644 --- a/tests/unit/test_agent_ab_task.py +++ b/tests/unit/test_agent_ab_task.py @@ -1,6 +1,7 @@ """Agent A/B tasks and answer scoring (#543).""" import json +import subprocess from pathlib import Path import pytest @@ -50,12 +51,19 @@ def test_shipped_tasks_load_with_unique_ids() -> None: def test_shipped_cgis_keys_name_files_that_exist() -> None: - """A key naming a moved file can never be met, so every run would score a miss.""" + """A key naming a moved file can never be met, so every run would score a miss. + + Checked in the tree the task runs on — `agent_ab.py` checks out `task.sha` — not + in HEAD: a later move (language-packs step 1) must not fail a task pinned before it. + """ for task in load_tasks(TASKS_DIR): if task.repo != "cgis": continue for path in task.gold.files + task.gold.allowed_files: - assert (REPO_ROOT / path).is_file(), f"{task.id}: {path}" + found = subprocess.run( + ["git", "cat-file", "-e", f"{task.sha}:{path}"], cwd=REPO_ROOT, check=False + ) + assert found.returncode == 0, f"{task.id}: {path} not in {task.sha[:7]}" def test_duplicate_ids_are_refused(tmp_path: Path) -> None: diff --git a/tests/unit/test_annotation_edges.py b/tests/unit/test_annotation_edges.py index fdfc59b6..c15eaed7 100644 --- a/tests/unit/test_annotation_edges.py +++ b/tests/unit/test_annotation_edges.py @@ -1,7 +1,7 @@ """Annotation positions produce one raw_dep edge per named type (spec D4, D9).""" from cgis.core.models import Edge, EdgeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.resolver.engine import ResolverEngine _RAW_DEP = "raw_dep:" diff --git a/tests/unit/test_audit.py b/tests/unit/test_audit.py index 4b3575b1..00093c5e 100644 --- a/tests/unit/test_audit.py +++ b/tests/unit/test_audit.py @@ -5,7 +5,7 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.context.audit import NoAuditSourcesError, audit_reachability from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_class_cohesion.py b/tests/unit/test_class_cohesion.py index dfad27e7..af17d2c8 100644 --- a/tests/unit/test_class_cohesion.py +++ b/tests/unit/test_class_cohesion.py @@ -14,7 +14,7 @@ from cgis.cli import app from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.render.metrics import ClassCohesion, DuckDBAnalyzer, class_cohesion_metrics from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 60c3be1a..3188aa26 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -22,7 +22,7 @@ from cgis.cli import _drift_status_label, app from cgis.core.freshness import FreshnessState from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.python_extractor import file_path_to_module_fqn +from cgis.languages.python.extractor import file_path_to_module_fqn from cgis.query.engine import QueryEngine from cgis.resolver.engine import ResolverEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_configs_above_root.py b/tests/unit/test_configs_above_root.py index 31bebbdd..63a4043f 100644 --- a/tests/unit/test_configs_above_root.py +++ b/tests/unit/test_configs_above_root.py @@ -18,8 +18,8 @@ import pytest from cgis.core.models import EdgeType, NodeNamespace -from cgis.extractors.typescript_extractor import TypeScriptExtractor from cgis.import_names import ImportNameCollector +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.pipeline import IngestionPipeline from cgis.workspaces import pnpm_member_names diff --git a/tests/unit/test_context_service.py b/tests/unit/test_context_service.py index f4b04a64..d94cdfcf 100644 --- a/tests/unit/test_context_service.py +++ b/tests/unit/test_context_service.py @@ -5,7 +5,7 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.context.context_service import build_context from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_di_acceptance.py b/tests/unit/test_di_acceptance.py index dea937e5..62efdac8 100644 --- a/tests/unit/test_di_acceptance.py +++ b/tests/unit/test_di_acceptance.py @@ -10,7 +10,7 @@ import pytest -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.engine import QueryEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_extractor_registry.py b/tests/unit/test_extractor_registry.py deleted file mode 100644 index bfb4ce59..00000000 --- a/tests/unit/test_extractor_registry.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Tests for the language registry (#344). - -The registry exists because four sites had to agree about what a file extension -means and did not. The tests that matter here are therefore the ones that hold -those sites together — above all `TestModuleFqnRoundTrip`, which ingests real -source and asserts the registry derives the ids the extractors actually emitted. -A test asserting `"src.app.foo"` by hand would keep passing while the two FQN -helpers drifted apart again, which is exactly how the bug arrived. -""" - -from pathlib import Path - -import pytest - -from cgis.core.models import NodeType -from cgis.extractors.python_extractor import file_path_to_module_fqn as python_fqn -from cgis.extractors.registry import ( - LANGUAGES, - build_extractors, - is_supported, - language_for, -) -from cgis.pipeline import IngestionPipeline - -TS_SOURCE = """ -export function helper(value: number): number { - return value * 2; -} -""" - -TSX_SOURCE = """ -export const Widget = () => { - return null; -}; -""" - -PY_SOURCE = """ -def helper(value: int) -> int: - return value * 2 -""" - - -class TestLanguageFor: - """The lookup every caller must branch on rather than defaulting.""" - - @pytest.mark.parametrize( - ("path", "extension"), - [ - ("src/cgis/pipeline.py", ".py"), - ("src/app/handler.ts", ".ts"), - ("src/app/Widget.tsx", ".tsx"), - ("src/app/types.d.ts", ".ts"), - ("src\\app\\handler.ts", ".ts"), - ], - ) - def test_recognises_supported_extensions(self, path: str, extension: str) -> None: - language = language_for(path) - assert language is not None - assert language.extension == extension - - @pytest.mark.parametrize("path", ["README.md", "src/app/old.js", "src/app/old.jsx", "Makefile"]) - def test_returns_none_for_anything_without_an_extractor(self, path: str) -> None: - """.js and .jsx especially: the TS FQN helper strips them, but nothing parses them. - - Registering them would let the collector derive a plausible FQN for a - file the graph has no nodes for — the silent miss (#344) from the other - direction. - """ - assert language_for(path) is None - assert not is_supported(path) - - -class TestCaseSensitivity: - """The registry must agree with the ingest pipeline, including where both say no.""" - - def test_an_uppercase_extension_is_unsupported_because_ingest_skips_it( - self, tmp_path: Path - ) -> None: - """Deliberate, and verified by execution rather than asserted from the code. - - `IngestionPipeline._get_extractor` matches with `filename.endswith(ext)` - against lowercase keys, so `Foo.TS` is never parsed and the graph holds - no node for it. Lowercasing the lookup here would therefore *create* the - bug #344 removed: the collector would call the file supported, derive - `foo` as its FQN, miss in the graph, log at debug and produce a review - with no context and no complaint. - - Case-insensitivity is defensible, but only if the pipeline and the - registry gain it together — never here alone. - """ - (tmp_path / "Foo.TS").write_text(TS_SOURCE, encoding="utf-8") - nodes, _, _ = IngestionPipeline(build_extractors([])).run(str(tmp_path)) - assert [n for n in nodes if n.type is NodeType.FILE] == [] - assert language_for("Foo.TS") is None - - -class TestCodeFence: - """Full-file context announces the language it is actually showing.""" - - @pytest.mark.parametrize( - ("path", "fence"), - [("a.py", "python"), ("a.ts", "ts"), ("a.tsx", "tsx")], - ) - def test_fence_matches_the_language(self, path: str, fence: str) -> None: - language = language_for(path) - assert language is not None - assert language.code_fence == fence - - -class TestBuildExtractors: - """The pipeline's mapping comes from the registry, so it cannot drift from it.""" - - def test_covers_every_registered_language(self) -> None: - extractors = build_extractors([]) - assert set(extractors) == {lang.extension for lang in LANGUAGES} - - def test_source_roots_reach_the_extractors(self) -> None: - """A root that does not reach the extractor silently changes every FQN.""" - extractors = build_extractors(["src"]) - nodes, _ = extractors[".ts"].parse(TS_SOURCE, "src/app/handler.ts") - assert nodes[0].id == "app.handler" - - -class TestModuleFqnRoundTrip: - """The acceptance test of #344: derive the id the extractor really emitted. - - `GraphContextCollector.sections` looks a changed file up in `graph.db` by the FQN the - registry derives. If that string is not byte-identical to the node id the - extractor wrote, the lookup misses, the collector logs at debug and - continues, and the review completes with no graph context and no complaint. - Comparing against ingested nodes is the only form of this test that can - catch the two helpers diverging. - """ - - @staticmethod - def _ingest(tmp_path: Path, files: dict[str, str]) -> dict[str, str]: - """Write files, ingest them, and return {relative path: FILE node id}.""" - for relative, source in files.items(): - path = tmp_path / relative - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(source, encoding="utf-8") - nodes, _, _ = IngestionPipeline(build_extractors([])).run(str(tmp_path)) - # file_path is already relative to the ingested root, and no source_root - # is configured — so the ids below are derived from these exact strings. - return {n.file_path: n.id for n in nodes if n.type is NodeType.FILE} - - def test_typescript(self, tmp_path: Path) -> None: - ingested = self._ingest( - tmp_path, - { - "app/handler.ts": TS_SOURCE, - "app/Widget.tsx": TSX_SOURCE, - "components/index.tsx": TSX_SOURCE, - }, - ) - assert ingested, "ingest produced no FILE nodes — the fixture is wrong, not the code" - for relative, node_id in ingested.items(): - language = language_for(relative) - assert language is not None - assert language.file_path_to_module_fqn(relative, None) == node_id - - def test_barrel_file_collapses_like_the_extractor_does(self, tmp_path: Path) -> None: - """`/index` for TS, not `/__init__` — the Python helper collapses neither.""" - ingested = self._ingest(tmp_path, {"components/index.tsx": TSX_SOURCE}) - assert ingested["components/index.tsx"] == "components" - - def test_python(self, tmp_path: Path) -> None: - ingested = self._ingest(tmp_path, {"pkg/mod.py": PY_SOURCE, "pkg/__init__.py": ""}) - for relative, node_id in ingested.items(): - language = language_for(relative) - assert language is not None - assert language.file_path_to_module_fqn(relative, None) == node_id - - def test_the_python_helper_would_have_failed_this(self, tmp_path: Path) -> None: - """Pins the actual defect, so a regression is legible rather than a diff. - - Feeding a `.ts` path to the Python helper leaves the extension on and - yields an id no node carries. This is what the collector did on every - TypeScript file before #344. - """ - ingested = self._ingest(tmp_path, {"app/handler.ts": TS_SOURCE}) - node_id = ingested["app/handler.ts"] - assert python_fqn("app/handler.ts", None) == "app.handler.ts" - assert python_fqn("app/handler.ts", None) != node_id diff --git a/tests/unit/test_fingerprint.py b/tests/unit/test_fingerprint.py index b4e28aeb..d8485c12 100644 --- a/tests/unit/test_fingerprint.py +++ b/tests/unit/test_fingerprint.py @@ -6,7 +6,7 @@ from conftest import module_with_funcs from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.drift.fingerprint import ( FingerprintExtractor, diff --git a/tests/unit/test_freshness.py b/tests/unit/test_freshness.py index fc684b86..456db2ae 100644 --- a/tests/unit/test_freshness.py +++ b/tests/unit/test_freshness.py @@ -11,7 +11,7 @@ from cgis.core.freshness import Freshness, FreshnessState from cgis.core.models import Edge, Node, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_graph_identity_dump.py b/tests/unit/test_graph_identity_dump.py index 459c90d5..393f7c1c 100644 --- a/tests/unit/test_graph_identity_dump.py +++ b/tests/unit/test_graph_identity_dump.py @@ -12,7 +12,7 @@ import graph_identity as gi # noqa: E402 -from cgis.extractors.registry import build_extractors # noqa: E402 +from cgis.languages import extractors # noqa: E402 from cgis.pipeline import IngestionPipeline # noqa: E402 from cgis.storage.sqlite_store import SQLiteStore # noqa: E402 @@ -20,7 +20,7 @@ def _ingest(root: Path, db: Path) -> None: """As `cgis ingest` does it, ingest state included.""" with SQLiteStore(str(db)) as store: - pipeline = IngestionPipeline(build_extractors([])) + pipeline = IngestionPipeline(extractors([])) pipeline.run(str(root), store=store) store.record_ingest(str(root), pipeline.observed_mtimes) store.record_ingest_options([], None) diff --git a/tests/unit/test_guardian_collector.py b/tests/unit/test_guardian_collector.py index 95fc5a2a..794bd97f 100644 --- a/tests/unit/test_guardian_collector.py +++ b/tests/unit/test_guardian_collector.py @@ -7,9 +7,9 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.registry import build_extractors from cgis.guardian.chunker import Chunk from cgis.guardian.collector import ContextCollector, parse_features +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore @@ -470,7 +470,7 @@ def _ingest_ts_project(tmp_path: Path) -> Path: (tmp_path / "app" / "main.ts").write_text(_TS_MAIN, encoding="utf-8") db = tmp_path / "graph.db" with SQLiteStore(str(db)) as store: - IngestionPipeline(build_extractors([])).run(str(tmp_path), store=store) + IngestionPipeline(extractors([])).run(str(tmp_path), store=store) return db diff --git a/tests/unit/test_guardian_martian_script.py b/tests/unit/test_guardian_martian_script.py index bd1a14c2..3570574d 100644 --- a/tests/unit/test_guardian_martian_script.py +++ b/tests/unit/test_guardian_martian_script.py @@ -17,10 +17,10 @@ import pytest from pydantic import BaseModel -from cgis.extractors.registry import build_extractors from cgis.guardian.martian import SliceCounts from cgis.guardian.providers.base import BaseProvider, ProviderUsage from cgis.guardian.providers.mistral import MistralProvider +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore @@ -354,7 +354,7 @@ def _db(self, tmp_path: Path, files: dict[str, str]) -> Path: (tree / rel).write_text(src, encoding="utf-8") db = tmp_path / "g.db" with SQLiteStore(str(db)) as store: - IngestionPipeline(build_extractors([])).run(str(tree), store=store) + IngestionPipeline(extractors([])).run(str(tree), store=store) return db def test_counts_the_changed_files_the_graph_knows(self, tmp_path: Path) -> None: diff --git a/tests/unit/test_import_acceptance.py b/tests/unit/test_import_acceptance.py index 0bd1bb7a..e58affe4 100644 --- a/tests/unit/test_import_acceptance.py +++ b/tests/unit/test_import_acceptance.py @@ -10,7 +10,7 @@ import pytest from cgis.core.models import EdgeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.engine import QueryEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_inject_architecture_graph.py b/tests/unit/test_inject_architecture_graph.py index eb41a7df..ec656cbf 100644 --- a/tests/unit/test_inject_architecture_graph.py +++ b/tests/unit/test_inject_architecture_graph.py @@ -11,7 +11,7 @@ import pytest -from cgis.extractors.registry import build_extractors +from cgis.languages import extractors from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore @@ -112,7 +112,7 @@ def _graph_db(tmp_path: Path) -> str: db = str(tmp_path / "graph.db") # Autodoc ingests ./src, so stored paths lack the prefix that --path-prefix restores. with SQLiteStore(db) as store: - IngestionPipeline(build_extractors([])).run(str(tmp_path / "src"), store=store) + IngestionPipeline(extractors([])).run(str(tmp_path / "src"), store=store) return db diff --git a/tests/unit/test_languages.py b/tests/unit/test_languages.py new file mode 100644 index 00000000..77b2330f --- /dev/null +++ b/tests/unit/test_languages.py @@ -0,0 +1,281 @@ +"""Tests for the language registry (#344, language-packs step 1). + +The registry exists because several sites had to agree about what a file +extension means and did not. The tests that matter hold those sites together — +above all `TestModuleFqnRoundTrip`, which ingests real source and asserts the +registry derives the ids the extractors actually emitted. +""" + +import json +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +from cgis.core.models import Node, NodeType +from cgis.core.paths import is_test_path +from cgis.languages import ( + MANIFESTS, + Manifest, + extractors, + is_supported, + pack_for_node, + pack_for_path, + pack_named, + scope_of, + skips, +) +from cgis.pipeline import IngestionPipeline + +TS_SOURCE = """ +export function helper(value: number): number { + return value * 2; +} +""" + +TSX_SOURCE = """ +export const Widget = () => { + return null; +}; +""" + +PY_SOURCE = """ +def helper(value: int) -> int: + return value * 2 +""" + + +def _ingest(tmp_path: Path, files: dict[str, str]) -> list[Node]: + for relative, source in files.items(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + nodes, _, _ = IngestionPipeline(extractors([])).run(str(tmp_path)) + return nodes + + +class TestPackForPath: + """The lookup every caller must branch on rather than defaulting.""" + + @pytest.mark.parametrize( + ("path", "name"), + [ + ("src/cgis/pipeline.py", "python"), + ("src/app/handler.ts", "typescript"), + ("src/app/Widget.tsx", "typescript"), + ("src/app/types.d.ts", "typescript"), + ("src\\app\\handler.ts", "typescript"), + ], + ) + def test_recognises_supported_extensions(self, path: str, name: str) -> None: + pack = pack_for_path(path) + assert pack is not None + assert pack.name == name + assert is_supported(path) + + @pytest.mark.parametrize("path", ["README.md", "src/app/old.js", "src/app/old.jsx", "Makefile"]) + def test_returns_none_for_anything_without_an_extractor(self, path: str) -> None: + """.js and .jsx especially: the TS FQN helper strips them, but nothing parses them.""" + assert pack_for_path(path) is None + assert not is_supported(path) + + +class TestCaseSensitivity: + """The registry must agree with the ingest pipeline, including where both say no.""" + + def test_an_uppercase_extension_is_unsupported_because_ingest_skips_it( + self, tmp_path: Path + ) -> None: + """`Foo.TS` is never parsed; calling it supported would recreate #344's silent miss.""" + nodes = _ingest(tmp_path, {"Foo.TS": TS_SOURCE}) + assert [n for n in nodes if n.type is NodeType.FILE] == [] + assert pack_for_path("Foo.TS") is None + + +class TestPacks: + """Each pack is what its manifest declares, and answers the step-1 contract.""" + + @pytest.mark.parametrize("manifest", MANIFESTS, ids=lambda m: m.name) + def test_a_pack_is_what_its_manifest_says(self, manifest: Manifest) -> None: + pack = pack_named(manifest.name) + assert pack is not None + assert (pack.name, pack.extensions) == (manifest.name, manifest.extensions) + + def test_an_unknown_language_has_no_pack(self) -> None: + assert pack_named("rust") is None + node = Node( + id="m.f", type=NodeType.FUNCTION, name="f", file_path="m.rs", + start_line=1, end_line=1, language="rust", + ) # fmt: skip + assert pack_for_node(node) is None + + @pytest.mark.parametrize( + ("path", "fence"), [("a.py", "python"), ("a.ts", "ts"), ("a.tsx", "tsx")] + ) + def test_fence_matches_the_file(self, path: str, fence: str) -> None: + pack = pack_for_path(path) + assert pack is not None + assert pack.code_fence(path) == fence + + @pytest.mark.parametrize( + "path", + ["tests/a.py", "src/test_a.py", "src/a.py", "tests/a.ts", "src/a.test.ts", "contest/a.ts"], + ) + def test_is_test_is_the_core_rule_for_both_packs(self, path: str) -> None: + """Ruling R1: the flag stays `is_test_path`; the Node validator still calls it.""" + for manifest in MANIFESTS: + pack = pack_named(manifest.name) + assert pack is not None + assert pack.is_test(path) is is_test_path(path) + + def test_every_node_names_the_pack_that_reads_its_file(self, tmp_path: Path) -> None: + """`pack_for_node` (by `Node.language`) and `pack_for_path` agree on extracted nodes. + + Synthesized rows do not carry a language (the "python" default), which is why + `mermaid.py` asks by path — `test_a_package_row_keeps_its_full_slug`. + """ + files = {"pkg/mod.py": PY_SOURCE, "app/h.ts": TS_SOURCE, "app/W.tsx": TSX_SOURCE} + nodes = _ingest(tmp_path, files) + extracted = [n for n in nodes if n.file_path in files] + assert {n.file_path for n in extracted} == set(files) + for node in extracted: + assert pack_for_node(node) is pack_for_path(node.file_path), node.id + + +class TestExtractors: + """The pipeline's mapping comes from the registry, so it cannot drift from it.""" + + def test_covers_every_registered_extension(self) -> None: + assert set(extractors([])) == {ext for m in MANIFESTS for ext in m.extensions} + + def test_source_roots_reach_the_extractors(self) -> None: + """A root that does not reach the extractor silently changes every FQN.""" + nodes, _ = extractors(["src"])[".ts"].parse(TS_SOURCE, "src/app/handler.ts") + assert nodes[0].id == "app.handler" + + def test_tsx_gets_the_tsx_grammar(self) -> None: + """Ruling R2: one pack, two grammars, chosen by extension.""" + jsx = b"const a =
;\n" + found = extractors([]) + assert not found[".tsx"]._parser.parse(jsx).root_node.has_error # noqa: SLF001 + assert found[".ts"]._parser.parse(jsx).root_node.has_error # noqa: SLF001 + + def test_membership_and_iteration_build_nothing(self) -> None: + """Run in a fresh interpreter: this process has long since loaded both grammars.""" + script = ( + "import sys\n" + "from cgis.languages import extractors\n" + "m = extractors([])\n" + "assert '.ts' in m and '.tsx' in m and len(list(m)) == 3\n" + "print(sorted(x for x in sys.modules if x.startswith('tree_sitter_')))\n" + ) + out = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=False + ) + assert out.returncode == 0, out.stderr + assert out.stdout.strip() == "[]" + + +class TestModuleFqnRoundTrip: + """The acceptance test of #344: derive the id the extractor really emitted.""" + + @staticmethod + def _files(tmp_path: Path, files: dict[str, str]) -> dict[str, str]: + nodes = _ingest(tmp_path, files) + return {n.file_path: n.id for n in nodes if n.type is NodeType.FILE} + + def test_typescript(self, tmp_path: Path) -> None: + ingested = self._files( + tmp_path, + { + "app/handler.ts": TS_SOURCE, + "app/Widget.tsx": TSX_SOURCE, + "components/index.tsx": TSX_SOURCE, + }, + ) + assert ingested, "ingest produced no FILE nodes — the fixture is wrong, not the code" + for relative, node_id in ingested.items(): + assert scope_of(relative, None) == node_id + + def test_barrel_file_collapses_like_the_extractor_does(self, tmp_path: Path) -> None: + ingested = self._files(tmp_path, {"components/index.tsx": TSX_SOURCE}) + assert ingested["components/index.tsx"] == "components" + + def test_python(self, tmp_path: Path) -> None: + ingested = self._files(tmp_path, {"pkg/mod.py": PY_SOURCE, "pkg/__init__.py": ""}) + assert ingested + for relative, node_id in ingested.items(): + assert scope_of(relative, None) == node_id + + def test_the_python_rule_would_have_failed_this(self, tmp_path: Path) -> None: + """Pins the #344 defect: the Python rule leaves `.ts` on.""" + ingested = self._files(tmp_path, {"app/handler.ts": TS_SOURCE}) + python = pack_named("python") + assert python is not None + assert python.scope_of("app/handler.ts", None) == "app.handler.ts" + assert python.scope_of("app/handler.ts", None) != ingested["app/handler.ts"] + + def test_a_path_no_pack_reads_gets_the_python_rule(self) -> None: + """Ruling R5: the CLI's extensionless fallback, kept in one place.""" + assert scope_of("src/cgis/pipeline", None) == "src.cgis.pipeline" + + +# The pipeline's pattern before step 1, copied so `skips` is held to it exactly. +_OLD_SKIP = re.compile(r"\.(test|spec)\.(py|ts|tsx|js|jsx)$", re.IGNORECASE) + + +@pytest.mark.parametrize( + "name", + [ + "a.test.py", "a.spec.ts", "a.TEST.tsx", "a.spec.jsx", "a.test.js", "a.test.TS", + "a_test.py", "test_a.py", "a.py", "a.ts", "contest.ts", "a.test.pyi", "a.spec.d.ts", + ], +) # fmt: skip +def test_skips_is_the_pipeline_pattern_it_replaced(name: str) -> None: + assert skips(name) is bool(_OLD_SKIP.search(name)) + + +def _cli_ingest_loads(tmp_path: Path, files: dict[str, str]) -> list[str]: + """Grammar packages a fresh `cgis ingest` of `files` imported, in a clean interpreter. + + The pipeline logs through structlog, which may write to the real stdout around + `CliRunner`; the probe reads only its own last line. + """ + repo = tmp_path / "repo" + for relative, source in files.items(): + (repo / relative).parent.mkdir(parents=True, exist_ok=True) + (repo / relative).write_text(source, encoding="utf-8") + script = ( + "import json, sys\n" + "from typer.testing import CliRunner\n" + "from cgis.cli import app\n" + f"argv = ['ingest', {str(repo)!r}, '--output', {str(tmp_path / 'g.db')!r}]\n" + "r = CliRunner().invoke(app, argv)\n" + "assert r.exit_code == 0, r.output\n" + "loaded = {m.split('.')[0] for m in sys.modules if m.startswith('tree_sitter_')}\n" + "print(json.dumps(sorted(loaded)))\n" + ) + out = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=False + ) + assert out.returncode == 0, out.stderr + loaded: list[str] = json.loads(out.stdout.strip().splitlines()[-1]) + return loaded + + +def test_a_python_only_ingest_never_loads_the_typescript_grammar(tmp_path: Path) -> None: + """Spec, Registry: loading every grammar eagerly is what the registry exists to avoid. + + The README matters: a file no pack reads is what makes a value-walking lookup + build every extractor on its way to "no match". + """ + files = {"pkg/a.py": PY_SOURCE, "README.md": "# demo\n"} + assert _cli_ingest_loads(tmp_path, files) == ["tree_sitter_python"] + + +def test_a_typescript_file_does_load_it(tmp_path: Path) -> None: + """The control: without it, the test above would pass on a probe that sees nothing.""" + loaded = _cli_ingest_loads(tmp_path, {"pkg/a.py": PY_SOURCE, "app/h.ts": TS_SOURCE}) + assert "tree_sitter_typescript" in loaded diff --git a/tests/unit/test_mermaid.py b/tests/unit/test_mermaid.py index 64a941a8..74b0a438 100644 --- a/tests/unit/test_mermaid.py +++ b/tests/unit/test_mermaid.py @@ -423,3 +423,25 @@ def test_node_slug_of_a_typescript_node_peels_its_own_module_prefix() -> None: """ node = _make_node_in_file("apps.web.components.Button.render", "apps/web/components/Button.tsx") assert _node_slug(node) == "Button_render" + + +def test_a_package_row_keeps_its_full_slug() -> None: + """A directory row (`cgis structure`) has no file a pack reads: no path slug (step 1). + + It carries `Node.language`'s default, "python"; resolving its pack from that + peeled the id down to the bare stem and renamed every package id in the diagram. + """ + node = Node( + id="cgis.query", type=NodeType.MODULE, name="query", file_path="cgis/query", + start_line=0, end_line=0, + ) # fmt: skip + assert _node_slug(node) == "cgis_query" + + +def test_a_tsx_node_from_an_old_database_still_slugs_by_its_path() -> None: + """Before #513 the TS extractor left `language` at its "python" default.""" + node = Node( + id="components.Button", type=NodeType.FUNCTION, name="Button", + file_path="components/Button.tsx", start_line=1, end_line=2, + ) # fmt: skip + assert _node_slug(node) == "Button" diff --git a/tests/unit/test_metrics_resolution.py b/tests/unit/test_metrics_resolution.py index da88991b..2a836064 100644 --- a/tests/unit/test_metrics_resolution.py +++ b/tests/unit/test_metrics_resolution.py @@ -13,7 +13,7 @@ from cgis.cli import app from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.render.metrics import DuckDBAnalyzer, resolution_metric from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_name_load_positions.py b/tests/unit/test_name_load_positions.py index 862c50f8..42e7af15 100644 --- a/tests/unit/test_name_load_positions.py +++ b/tests/unit/test_name_load_positions.py @@ -12,7 +12,7 @@ import pytest from cgis.core.models import EdgeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor # (label, snippet, is Widget a use of the class?) _POSITIONS: list[tuple[str, str, bool]] = [ diff --git a/tests/unit/test_npm_dependencies.py b/tests/unit/test_npm_dependencies.py index 73a3f5d5..defb007f 100644 --- a/tests/unit/test_npm_dependencies.py +++ b/tests/unit/test_npm_dependencies.py @@ -17,7 +17,7 @@ from pathlib import Path from cgis.core.models import Edge, EdgeType, Node, NodeNamespace -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.npm_dependencies import NpmDependencies from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py index 2d23e01a..3d8c20e7 100644 --- a/tests/unit/test_pipeline.py +++ b/tests/unit/test_pipeline.py @@ -6,7 +6,7 @@ import pytest from cgis.core.models import EdgeType, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.resolver.engine import ResolverEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_pipeline_incremental_symbols.py b/tests/unit/test_pipeline_incremental_symbols.py index 1abcb985..5ae882a0 100644 --- a/tests/unit/test_pipeline_incremental_symbols.py +++ b/tests/unit/test_pipeline_incremental_symbols.py @@ -14,7 +14,7 @@ import pytest from cgis.core.models import VIRTUAL_FILE_PATH, Edge, Node -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_pipeline_ts_workspaces.py b/tests/unit/test_pipeline_ts_workspaces.py index 2922c2f2..2f06275b 100644 --- a/tests/unit/test_pipeline_ts_workspaces.py +++ b/tests/unit/test_pipeline_ts_workspaces.py @@ -15,7 +15,7 @@ from pathlib import Path from cgis.core.models import EdgeType -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_python_annotations.py b/tests/unit/test_python_annotations.py index 959cc9dd..7747f3f3 100644 --- a/tests/unit/test_python_annotations.py +++ b/tests/unit/test_python_annotations.py @@ -4,8 +4,8 @@ from tree_sitter import Language, Parser from tree_sitter import Node as BaseNode -from cgis.extractors._python_annotations import collect_type_names -from cgis.extractors._python_ast import enclosing_class_fqn +from cgis.languages.python._annotations import collect_type_names +from cgis.languages.python._syntax import enclosing_class_fqn _PARSER = Parser(Language(tspython.language())) diff --git a/tests/unit/test_python_dependencies.py b/tests/unit/test_python_dependencies.py index bdc58f9d..202d2ca7 100644 --- a/tests/unit/test_python_dependencies.py +++ b/tests/unit/test_python_dependencies.py @@ -8,7 +8,7 @@ from pathlib import Path from cgis.core.models import Node, NodeNamespace -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.python_dependencies import declared_import_roots, import_name from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_python_extractor.py b/tests/unit/test_python_extractor.py index d3bc57ac..b759b178 100644 --- a/tests/unit/test_python_extractor.py +++ b/tests/unit/test_python_extractor.py @@ -3,8 +3,8 @@ import pytest from cgis.core.models import Edge, EdgeType, NodeType -from cgis.extractors._python_ast import resolve_relative_module -from cgis.extractors.python_extractor import PythonExtractor, file_path_to_module_fqn +from cgis.languages.python._syntax import resolve_relative_module +from cgis.languages.python.extractor import PythonExtractor, file_path_to_module_fqn @pytest.fixture diff --git a/tests/unit/test_resolver.py b/tests/unit/test_resolver.py index f41aef2a..6a780a32 100644 --- a/tests/unit/test_resolver.py +++ b/tests/unit/test_resolver.py @@ -3,7 +3,7 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.resolver.engine import ResolverEngine diff --git a/tests/unit/test_resolver_js_builtins.py b/tests/unit/test_resolver_js_builtins.py index f86c2fe9..7ce6c2b5 100644 --- a/tests/unit/test_resolver_js_builtins.py +++ b/tests/unit/test_resolver_js_builtins.py @@ -3,8 +3,8 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import PythonExtractor -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages.python.extractor import PythonExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.resolver.engine import ResolverEngine diff --git a/tests/unit/test_resolver_language_scope.py b/tests/unit/test_resolver_language_scope.py index c16c6a72..56cceabb 100644 --- a/tests/unit/test_resolver_language_scope.py +++ b/tests/unit/test_resolver_language_scope.py @@ -5,9 +5,9 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace -from cgis.extractors.python_extractor import PythonExtractor -from cgis.extractors.registry import build_extractors -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages import extractors +from cgis.languages.python.extractor import PythonExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.pipeline import IngestionPipeline from cgis.resolver.engine import ResolverEngine from cgis.resolver.indices import IndexBuilder @@ -108,7 +108,7 @@ def test_incremental_ts_change_does_not_downgrade_python_stdlib(tmp_path: Path) (tmp_path / "tt").mkdir() (tmp_path / "pp" / "mod.py").write_text(_PY_JSON, encoding="utf-8") (tmp_path / "tt" / "util.ts").write_text(_TS_JSON, encoding="utf-8") - pipeline = IngestionPipeline(build_extractors([])) + pipeline = IngestionPipeline(extractors([])) db = str(tmp_path / "g.db") def namespace() -> NodeNamespace: diff --git a/tests/unit/test_resolver_phantom_internal.py b/tests/unit/test_resolver_phantom_internal.py index 5ebb0170..264f2fbc 100644 --- a/tests/unit/test_resolver_phantom_internal.py +++ b/tests/unit/test_resolver_phantom_internal.py @@ -16,7 +16,7 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.resolver.engine import ResolverEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_resolver_ts_declarations.py b/tests/unit/test_resolver_ts_declarations.py index 94d7ed30..7f301bf7 100644 --- a/tests/unit/test_resolver_ts_declarations.py +++ b/tests/unit/test_resolver_ts_declarations.py @@ -15,7 +15,7 @@ from pathlib import Path from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.pipeline import IngestionPipeline from cgis.resolver.engine import ResolverEngine diff --git a/tests/unit/test_review_fingerprint_closure.py b/tests/unit/test_review_fingerprint_closure.py index 88b100a3..32917711 100644 --- a/tests/unit/test_review_fingerprint_closure.py +++ b/tests/unit/test_review_fingerprint_closure.py @@ -43,7 +43,7 @@ def test_closure_reaches_the_graph_and_drift_sections() -> None: "src/cgis/query/render/mermaid.py", "src/cgis/storage/sqlite_store.py", "src/cgis/query/drift/drift.py", - "src/cgis/extractors/registry.py", + "src/cgis/languages/__init__.py", ): assert expected in closure diff --git a/tests/unit/test_self_types.py b/tests/unit/test_self_types.py index 4ac3bed1..4042f612 100644 --- a/tests/unit/test_self_types.py +++ b/tests/unit/test_self_types.py @@ -1,6 +1,6 @@ """The four sources that populate a class's self_types map (spec D1).""" -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor def _class_self_types(code: str, class_fqn: str) -> dict[str, str]: diff --git a/tests/unit/test_tsconfig_paths.py b/tests/unit/test_tsconfig_paths.py index 143a3945..f22dc1e9 100644 --- a/tests/unit/test_tsconfig_paths.py +++ b/tests/unit/test_tsconfig_paths.py @@ -18,7 +18,7 @@ import pytest from cgis.core.models import EdgeType -from cgis.extractors.typescript_extractor import TypeScriptExtractor +from cgis.languages.typescript.extractor import TypeScriptExtractor from cgis.pipeline import IngestionPipeline from cgis.storage.sqlite_store import SQLiteStore from cgis.tsconfig_paths import TsconfigPaths, load_jsonc diff --git a/tests/unit/test_type_checking_imports.py b/tests/unit/test_type_checking_imports.py index 0a85e916..0cecc40c 100644 --- a/tests/unit/test_type_checking_imports.py +++ b/tests/unit/test_type_checking_imports.py @@ -14,7 +14,7 @@ from pathlib import Path from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.python_extractor import PythonExtractor +from cgis.languages.python.extractor import PythonExtractor from cgis.pipeline import IngestionPipeline from cgis.query.analysis.analyzer import AnalyzerEngine from cgis.storage.sqlite_store import SQLiteStore diff --git a/tests/unit/test_typescript_extractor.py b/tests/unit/test_typescript_extractor.py index d214259a..691368a9 100644 --- a/tests/unit/test_typescript_extractor.py +++ b/tests/unit/test_typescript_extractor.py @@ -3,7 +3,7 @@ import pytest from cgis.core.models import Edge, EdgeType, Node, NodeType -from cgis.extractors.typescript_extractor import TypeScriptExtractor, file_path_to_module_fqn +from cgis.languages.typescript.extractor import TypeScriptExtractor, file_path_to_module_fqn # --------------------------------------------------------------------------- # file_path_to_module_fqn