From 5c8aebc9f2abdbb369730334d6e042e3575e5b25 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Sat, 12 Sep 2026 20:40:10 +0000 Subject: [PATCH 1/3] fix(config): honor declared languages in extraction and imports --- graphify/extract.py | 59 ++++--- graphify/extractors/csharp.py | 13 +- graphify/extractors/ocaml.py | 3 +- graphify/extractors/resolution.py | 82 ++++++++-- graphify/extractors/robot.py | 3 +- graphify/symbol_resolution.py | 5 +- tests/test_language_override_imports.py | 178 ++++++++++++++++++++ tests/test_language_overrides.py | 208 ++++++++++++++++++++++++ 8 files changed, 495 insertions(+), 56 deletions(-) create mode 100644 tests/test_language_override_imports.py diff --git a/graphify/extract.py b/graphify/extract.py index 735520e951..0badb6c7e0 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -96,6 +96,7 @@ _disambiguate_colliding_node_ids, _find_workspace_root, _go_import_path_for_file, + _is_python_package_dir, _is_type_like_definition, _js_call_identifier, _js_default_export_name, @@ -213,7 +214,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: the edge dangles and is silently dropped — the graph loses most ``imports`` edges purely because of where the scan started. Build an alias map from the dotted-module id to the real file-node id by detecting each ``.py`` file's - package root (the contiguous run of ancestor dirs carrying ``__init__.py``) + package root (the contiguous run of ancestor dirs carrying a Python initializer) and rewrite matching ``imports``/``imports_from`` edge targets. Guards: never shadow an existing node id, and drop an alias claimed by more than one file (ambiguous -> leave dangling, as before). Files whose package root IS the @@ -225,7 +226,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} alias_to_files: dict[str, set[str]] = {} for p in paths: - if p.suffix.lower() not in (".py", ".pyi"): + if effective_suffix(p).lower() not in (".py", ".pyi"): continue try: rel = Path(p).resolve().relative_to(root) @@ -238,7 +239,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: levels = 0 # Bounded by the number of dirs between the file and the scan root, so a # pathological `/__init__.py` chain can't loop forever. - while levels < len(parts) - 1 and (d / "__init__.py").is_file(): + while levels < len(parts) - 1 and _is_python_package_dir(d): levels += 1 d = d.parent if levels == 0: @@ -249,7 +250,11 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: file_node = _file_node_id(rel) alias = _make_id(str(Path(*mod_parts).with_suffix(""))) alias_to_files.setdefault(alias, set()).add(file_node) - if p.name in ("__init__.py", "__init__.pyi") and len(mod_parts) > 1: + if ( + p.stem == "__init__" + and effective_suffix(p) in (".py", ".pyi") + and len(mod_parts) > 1 + ): # `import pkg` / `from pkg import x` targets the package-dir id. pkg_alias = _make_id(str(Path(*mod_parts[:-1]))) alias_to_files.setdefault(pkg_alias, set()).add(file_node) @@ -268,7 +273,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: if ( isinstance(e, dict) and e.get("relation") in ("imports", "imports_from") - and str(e.get("source_file", "")).lower().endswith((".py", ".pyi")) + and effective_suffix(str(e.get("source_file", ""))).lower() in (".py", ".pyi") ): tgt = e.get("target") if tgt in alias_map: @@ -277,10 +282,10 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: """Repoint Python sibling-import edges to the real file node in directories - without an __init__.py (#3430). + without a Python package initializer (#3430). When a Python file below the scan root lives in a non-package directory - (no __init__.py in its immediate parent), plain imports of same-directory + (no package initializer in its immediate parent), plain imports of same-directory modules (e.g. `scripts/main.py: import greeter`) target a bare name (`greeter`), while the real file node is scan-root-relative (`scripts_greeter`). Because the directory is not a package, `_repoint_python_package_imports` skips it @@ -288,7 +293,7 @@ def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: (`greeter.greet()`) to be dropped. This pass is strictly importer-directory-local: - - Only applies when the importing file's parent directory has no __init__.py. + - Only applies when the importing file's parent directory has no package initializer. - Resolves only to unambiguous same-directory candidate modules in the scanned corpus. - Never builds a global alias map and never searches outside the importer's directory. - Preserves local aliases (e.g. `import greeter as g`). @@ -304,7 +309,7 @@ def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: # dir_path -> {module_name_id: set_of_file_node_ids} dir_siblings: dict[Path, dict[str, set[str]]] = {} for p in paths: - if p.suffix.lower() not in (".py", ".pyi"): + if effective_suffix(p).lower() not in (".py", ".pyi"): continue try: p_res = Path(p).resolve() @@ -316,11 +321,11 @@ def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: if file_node not in node_ids: continue - if p_res.name in ("__init__.py", "__init__.pyi"): + if p_res.stem == "__init__" and effective_suffix(p_res) in (".py", ".pyi"): # Sibling package directory inside parent_dir (parent_dir / subpkg / __init__.py) pkg_dir = p_res.parent parent_dir = pkg_dir.parent - if not (parent_dir / "__init__.py").is_file() and not (parent_dir / "__init__.pyi").is_file(): + if not _is_python_package_dir(parent_dir, (".py", ".pyi")): mod_key = _make_id(pkg_dir.name) dir_siblings.setdefault(parent_dir, {}).setdefault(mod_key, set()).add(file_node) continue @@ -328,7 +333,7 @@ def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: d = p_res.parent # PEP 328 guard: if the directory is a package, implicit relative imports # are forbidden in Python 3. Do not index packages as loose sibling directories. - if (d / "__init__.py").is_file() or (d / "__init__.pyi").is_file(): + if _is_python_package_dir(d, (".py", ".pyi")): continue mod_key = _make_id(p_res.stem) @@ -351,7 +356,7 @@ def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: if not ( isinstance(e, dict) and e.get("relation") in ("imports", "imports_from") - and str(e.get("source_file", "")).lower().endswith((".py", ".pyi")) + and effective_suffix(str(e.get("source_file", ""))).lower() in (".py", ".pyi") ): continue @@ -1700,7 +1705,7 @@ def extract_python(path: Path) -> dict: def extract_js(path: Path) -> dict: """Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file.""" - suffix = path.suffix.lower() + suffix = effective_suffix(path).lower() is_ts = suffix in (".ts", ".tsx", ".mts", ".cts") if suffix == ".tsx": config = _TSX_CONFIG @@ -6148,7 +6153,7 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: root = Path(root_str) cache_location = Path(cache_location_str) _raise_recursion_limit() - bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES + bypass_cache = effective_suffix(path) in _JS_CACHE_BYPASS_SUFFIXES # Check cache first (avoid re-extraction) if not bypass_cache: @@ -6341,7 +6346,7 @@ def _extract_sequential( if extractor is None: per_file[idx] = {"nodes": [], "edges": []} continue - bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES + bypass_cache = effective_suffix(path) in _JS_CACHE_BYPASS_SUFFIXES # XAML boundary anchors on `root` (the corpus), not the cache location. result = _safe_extract_with_xaml_root(extractor, path, root) # See _extract_single_file: don't cache an anomalous zero-node result (#1666). @@ -6486,7 +6491,7 @@ def extract( if _get_extractor(path) is None: per_file[i] = {"nodes": [], "edges": []} continue - bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES + bypass_cache = effective_suffix(path) in _JS_CACHE_BYPASS_SUFFIXES if not bypass_cache: cached = load_cached(path, root, cache_root=cache_location, salt=cache_salt(path)) if cached is not None: @@ -6578,7 +6583,7 @@ def extract( from graphify.detect import CODE_EXTENSIONS as _CODE_EXTS _no_extractor: dict[str, int] = {} for _p in paths: - _ext = _p.suffix.lower() + _ext = effective_suffix(_p).lower() if _ext in _CODE_EXTS and _get_extractor(_p) is None: _no_extractor[_ext] = _no_extractor.get(_ext, 0) + 1 if _no_extractor: @@ -6605,7 +6610,7 @@ def extract( for i, _p in enumerate(paths): _err = (per_file[i] or {}).get("error") or "" if _DEP_MISSING_MARKER in _err or _DEP_LOAD_FAILED_MARKER in _err: - _ext = _p.suffix.lower() + _ext = effective_suffix(_p).lower() _missing_dep_count[_ext] = _missing_dep_count.get(_ext, 0) + 1 _missing_dep_error.setdefault(_ext, _err) for _ext, _n in sorted(_missing_dep_count.items(), key=lambda kv: (-kv[1], kv[0])): @@ -7144,7 +7149,7 @@ def _learn(e: dict) -> None: # internal class with that simple name, manufacturing a false hub. Parking # such references on an FQN-labeled stub first prevents the merge, and # import-exact resolution of internal references (#1318/#1744) still applies. - _java_sel = [(r, p) for r, p in zip(per_file, paths) if p.suffix == ".java"] + _java_sel = [(r, p) for r, p in zip(per_file, paths) if effective_suffix(p) == ".java"] if _java_sel: try: _resolve_java_type_references( @@ -7155,7 +7160,7 @@ def _learn(e: dict) -> None: logging.getLogger(__name__).warning("Java type-reference resolution failed, skipping: %s", exc) # Resolve internal Go pkg.Type references exactly and park external ones # before the generic bare-label stub rewire can manufacture a collision. - _go_sel = [(r, p) for r, p in zip(per_file, paths) if p.suffix == ".go"] + _go_sel = [(r, p) for r, p in zip(per_file, paths) if effective_suffix(p) == ".go"] if _go_sel: try: _resolve_go_type_references( @@ -7169,9 +7174,9 @@ def _learn(e: dict) -> None: "Go type-reference resolution failed, skipping: %s", exc ) # Cross-file Python import resolution and type-reference repointing (#3252) - py_paths = [p for p in paths if p.suffix == ".py"] + py_paths = [p for p in paths if effective_suffix(p) == ".py"] if py_paths: - py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"] + py_results = [r for r, p in zip(per_file, paths) if effective_suffix(p) == ".py"] try: cross_file_edges = _resolve_cross_file_imports(py_results, py_paths, all_nodes, all_edges) all_edges.extend(cross_file_edges) @@ -7181,9 +7186,9 @@ def _learn(e: dict) -> None: _rewire_unique_stub_nodes(all_nodes, all_edges) # Cross-file Java import resolution - java_paths = [p for p in paths if p.suffix == ".java"] + java_paths = [p for p in paths if effective_suffix(p) == ".java"] if java_paths: - java_results = [r for r, p in zip(per_file, paths) if p.suffix == ".java"] + java_results = [r for r, p in zip(per_file, paths) if effective_suffix(p) == ".java"] try: all_edges.extend(_resolve_cross_file_java_imports(java_results, java_paths)) except Exception as exc: @@ -7194,9 +7199,9 @@ def _learn(e: dict) -> None: # references edges left on shadow stubs, disambiguating same-named types by the # referencing file's `using` directives + enclosing namespace (mirrors Java #1318). _DOTNET_TYPE_EXTS = {".cs", ".razor", ".cshtml"} - cs_paths = [p for p in paths if p.suffix.lower() in _DOTNET_TYPE_EXTS] + cs_paths = [p for p in paths if effective_suffix(p).lower() in _DOTNET_TYPE_EXTS] if cs_paths: - cs_results = [r for r, p in zip(per_file, paths) if p.suffix.lower() in _DOTNET_TYPE_EXTS] + cs_results = [r for r, p in zip(per_file, paths) if effective_suffix(p).lower() in _DOTNET_TYPE_EXTS] try: _resolve_csharp_type_references(cs_results, cs_paths, all_nodes, all_edges) except Exception as exc: diff --git a/graphify/extractors/csharp.py b/graphify/extractors/csharp.py index f823d64c30..dbd6910d3c 100644 --- a/graphify/extractors/csharp.py +++ b/graphify/extractors/csharp.py @@ -14,6 +14,7 @@ from pathlib import Path from graphify.extractors.base import _make_id +from graphify.rcfile import effective_suffix def _build_csharp_type_def_index(all_nodes: list[dict]) -> dict[tuple[str, str], str]: @@ -32,11 +33,7 @@ def _build_csharp_type_def_index(all_nodes: list[dict]) -> dict[tuple[str, str], if not (isinstance(nid, str) and nid and isinstance(label, str) and label): continue source_file = node.get("source_file") - if ( - not isinstance(source_file, str) - or not source_file.endswith(".cs") - or node.get("file_type") != "code" - ): + if not _is_cs_file(source_file) or node.get("file_type") != "code": continue if label.endswith(")") or label.startswith(".") or "." in label: continue @@ -153,11 +150,11 @@ def _resolve_cross_file_csharp_imports( def _is_cs_file(value: object) -> bool: - return isinstance(value, str) and value.endswith(".cs") + return isinstance(value, str) and effective_suffix(value) == ".cs" def _is_dotnet_source_file(value: object) -> bool: - return isinstance(value, str) and value.endswith(_DOTNET_SOURCE_EXTS) + return isinstance(value, str) and effective_suffix(value) in _DOTNET_SOURCE_EXTS def _metadata(value: object) -> dict: @@ -201,7 +198,7 @@ def __init__(self, all_nodes: list[dict], all_edges: list[dict]) -> None: if not ( source_node and isinstance(source_node.get("label"), str) - and source_node.get("label", "").endswith(_DOTNET_SOURCE_EXTS) + and _is_dotnet_source_file(source_node.get("label")) ): continue source_file = source_node.get("source_file") diff --git a/graphify/extractors/ocaml.py b/graphify/extractors/ocaml.py index ce735c13c6..05ad665d04 100644 --- a/graphify/extractors/ocaml.py +++ b/graphify/extractors/ocaml.py @@ -9,6 +9,7 @@ from pathlib import Path from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.rcfile import effective_suffix def extract_ocaml(path: Path) -> dict: @@ -21,7 +22,7 @@ def extract_ocaml(path: Path) -> dict: return {"nodes": [], "edges": [], "error": "tree-sitter-ocaml not installed"} try: - if path.suffix == ".mli": + if effective_suffix(path) == ".mli": language = Language(tsocaml.language_ocaml_interface()) else: language = Language(tsocaml.language_ocaml()) diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 62bb4a6e13..a8ea23d244 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -10,6 +10,7 @@ _make_id, _read_text, ) +from graphify.rcfile import effective_suffix, get_language_overrides import functools import hashlib import json @@ -34,6 +35,18 @@ _JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") +def _declared_language_suffixes(target_exts: tuple[str, ...]) -> tuple[str, ...]: + """Return declared literal suffixes for these languages in deterministic order.""" + # Mapping keys must be single suffixes, never relative path fragments. + suffixes = [ + source_ext + for source_ext, target_ext in get_language_overrides().items() + if target_ext in target_exts + and Path(f"x{source_ext}").suffix == source_ext + ] + return tuple(sorted(suffixes)) + + def _resolve_js_import_path(candidate: Path) -> Path: """Resolve a JS/TS/Svelte import target to a local file when it exists.""" candidate = Path(os.path.normpath(candidate)) @@ -50,19 +63,27 @@ def _resolve_js_import_path(candidate: Path) -> Path: if tsx_candidate.is_file(): return tsx_candidate + declared_exts = _declared_language_suffixes(_JS_RESOLVE_EXTS) + # Append extensions to the full filename, which covers extensionless imports, # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. - for ext in _JS_RESOLVE_EXTS: + # Native suffixes retain precedence over declared ones. + for ext in _JS_RESOLVE_EXTS + declared_exts: with_ext = candidate.parent / f"{candidate.name}{ext}" if with_ext.is_file(): return with_ext - # Only fall back to directory indexes after file candidates lose. + # Only fall back to directory indexes after file candidates lose, native + # index names before declared ones. if candidate.is_dir(): for index_name in _JS_INDEX_FILES: index_candidate = candidate / index_name if index_candidate.is_file(): return index_candidate + for ext in declared_exts: + index_candidate = candidate / f"index{ext}" + if index_candidate.is_file(): + return index_candidate return candidate @@ -1389,17 +1410,17 @@ def _parse_js_tree(path: Path): # .vue embeds the script in non-JS markup; mask it out and parse the #