From c0a0ba0a5b040e245a3d22aa19080576c17acb2d Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:27:32 +0530 Subject: [PATCH 01/11] feat(code): a tree-less Thrift IDL lane lang_thrift.py reads structs, unions, exceptions, enums, typedefs, services and service functions with a token scanner in the shape of lang_proto.py. Field, argument, return and throws types (through list/set/map) and a service's extends are reference sites bound by lane_units.bind_sites; base types bind to nothing. Includes resolve against the including file's directory and its ancestors. .thrift leaves the plain-text list. Closes #624 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Rahul <132360698+rahulsjha@users.noreply.github.com> --- docs/development/adding-language-support.md | 2 +- docs/features/code-index/overview.md | 1 + .../code-intelligence-by-language.md | 2 +- server/code/app/index/ingest_parse.py | 15 +- server/code/domain/extract/lang_detect.py | 1 + server/code/domain/extract/lang_text.py | 1 - server/code/domain/extract/lang_thrift.py | 374 ++++++++++++++++++ server/code/domain/extract/name_scope.py | 1 + server/code/domain/search_scope.py | 1 + tests/code_lanes/corpus.py | 23 +- tests/fixtures/code_lanes/NOTICE.md | 2 +- tests/fixtures/code_lanes/expected.json | 30 ++ .../code_lanes/thrift/acme/machines.thrift | 37 ++ .../code_lanes/thrift/acme/shared.thrift | 17 + tests/unit/lane_walks.py | 6 + tests/unit/test_code_extractor_stamp.py | 4 + tests/unit/test_code_lanes_manifest.py | 2 +- tests/unit/test_code_lang_thrift.py | 149 +++++++ 18 files changed, 658 insertions(+), 10 deletions(-) create mode 100644 server/code/domain/extract/lang_thrift.py create mode 100644 tests/fixtures/code_lanes/thrift/acme/machines.thrift create mode 100644 tests/fixtures/code_lanes/thrift/acme/shared.thrift create mode 100644 tests/unit/test_code_lang_thrift.py diff --git a/docs/development/adding-language-support.md b/docs/development/adding-language-support.md index ee2dee0c86..dc99bbdde5 100644 --- a/docs/development/adding-language-support.md +++ b/docs/development/adding-language-support.md @@ -36,7 +36,7 @@ Every module under `server/code/domain/extract/` is either in a stage of `STAGE_ | Shape | Examples | You provide | | ----------------- | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Grammar lane | `lang_typescript.py`, `lang_rust.py`, `lang_go.py`, `lang_java.py` | A tree-sitter grammar dependency in `pyproject.toml`; symbols and call sites from the tree; chunks and edges come from the shared chunker and edge walker. | -| Tree-less lane | `lang_proto.py`, `lang_gotemplate.py`, `lang_hcl.py`, `lang_k8s.py`, `lang_jsonnet.py` (`LANE_UNIT_LANGS`) | A hand-written scanner; `ParsedFile.tree` stays `None`; chunks from `lane_units.span_chunks`, edges from `reference_sites` + `SITE_RULES` bound by `lane_units.bind_sites`. | +| Tree-less lane | `lang_proto.py`, `lang_thrift.py`, `lang_gotemplate.py`, `lang_hcl.py`, `lang_k8s.py`, `lang_jsonnet.py` (`LANE_UNIT_LANGS`) | A hand-written scanner; `ParsedFile.tree` stays `None`; chunks from `lane_units.span_chunks`, edges from `reference_sites` + `SITE_RULES` bound by `lane_units.bind_sites`. | | Include-only lane | `tla`, `flatbuffers`, `cmake`, `doc` (`_LANE_INCLUDE_LANGS`) | Symbols and includes resolved by the simple lane resolver. | | Embedded language | SQL inside Go and Java (`go_sql_units`, `java_sql_units`), shell islands in Dockerfiles and Makefiles | Units joined into the host file's parse (`_join_island_symbols`), usually behind a kill switch setting. | | Chunk-only | `.css`, text files (`lang_text.py`) | Nothing but detection: the file is chunked and searchable with no symbols. | diff --git a/docs/features/code-index/overview.md b/docs/features/code-index/overview.md index 75082e32d1..2c6271dd1f 100644 --- a/docs/features/code-index/overview.md +++ b/docs/features/code-index/overview.md @@ -134,6 +134,7 @@ still gets file-level impact. | Shell (`.sh`, `.bash`, `.bats`, shebang, rc files) | yes | calls (commands), `source` includes | — | | Prisma (`.prisma`) | yes | model relations | — | | Protocol Buffers (`.proto`), FlatBuffers (`.fbs`) | yes | imports, type uses | — | +| Thrift IDL (`.thrift`) | yes | includes, type uses, service `extends` | — | | Go templates and Helm charts (`.tpl`, `.tmpl`, `.gotmpl`, templated manifests) | yes | template calls | — | | Terraform / HCL (`.tf`, `.tfvars`, `.hcl`), Kubernetes manifests | yes | references (`reads`) | — | | Jsonnet (`.jsonnet`, `.libsonnet`) | yes | calls, imports | — | diff --git a/docs/use-cases/code-intelligence-by-language.md b/docs/use-cases/code-intelligence-by-language.md index 414b69b208..3094915605 100644 --- a/docs/use-cases/code-intelligence-by-language.md +++ b/docs/use-cases/code-intelligence-by-language.md @@ -64,7 +64,7 @@ files ([switches](../features/code-index/overview.md#language-support)). ## Infrastructure as code Terraform and HCL, Kubernetes manifests, Helm charts and Go templates, Jsonnet, CMake, -Protocol Buffers and shell scripts are indexed with their references, includes and calls, +Protocol Buffers, Thrift IDL and shell scripts are indexed with their references, includes and calls, so an agent follows a Helm `include` to its `define`, or a Terraform reference to what it names. Dockerfile, Makefile, Jenkinsfile and CI YAML jobs become symbols with `LOOM_CODE_HOST_ISLANDS` on, after a backfill of those files when the namespace is already diff --git a/server/code/app/index/ingest_parse.py b/server/code/app/index/ingest_parse.py index cfdf8d5c92..d71ac997cb 100644 --- a/server/code/app/index/ingest_parse.py +++ b/server/code/app/index/ingest_parse.py @@ -276,6 +276,12 @@ def _parse_java(path: str, source: bytes) -> ParsedFile: return pf +def _parse_thrift(path: str, source: bytes) -> ParsedFile: + from server.code.domain.extract.lang_thrift import parse_thrift + + return parse_thrift(path, source) + + def _parse_proto(path: str, source: bytes) -> ParsedFile: from server.code.domain.extract.lang_proto import parse_proto @@ -392,6 +398,7 @@ def _parse_doc(path: str, source: bytes) -> ParsedFile: "astro": _parse_astro, "angular_template": _parse_angular_template, "proto": _parse_proto, + "thrift": _parse_thrift, "gotemplate": _parse_gotemplate, "hcl": _parse_hcl, "k8s": _parse_k8s, @@ -408,7 +415,9 @@ def _parse_doc(path: str, source: bytes) -> ParsedFile: # The lanes without a tree-sitter grammar — chunks from their symbol spans, edges from # their reference sites (`lane_units`) -LANE_UNIT_LANGS: frozenset[str] = frozenset({"proto", "gotemplate", "hcl", "k8s", "jsonnet"}) +LANE_UNIT_LANGS: frozenset[str] = frozenset( + {"proto", "thrift", "gotemplate", "hcl", "k8s", "jsonnet"} +) def _lane_module(lang: str) -> Any: @@ -424,6 +433,10 @@ def _lane_module(lang: str) -> Any: from server.code.domain.extract import lang_jsonnet return lang_jsonnet + if lang == "thrift": + from server.code.domain.extract import lang_thrift + + return lang_thrift return lang_proto if lang == "proto" else lang_gotemplate diff --git a/server/code/domain/extract/lang_detect.py b/server/code/domain/extract/lang_detect.py index caf07ccbb0..01a537c1a8 100644 --- a/server/code/domain/extract/lang_detect.py +++ b/server/code/domain/extract/lang_detect.py @@ -272,6 +272,7 @@ def detect_shell(path: str, source: bytes) -> str: ".rego": "config", ".mdc": "doc", ".proto": "proto", + ".thrift": "thrift", ".tpl": "gotemplate", ".tmpl": "gotemplate", ".gotmpl": "gotemplate", diff --git a/server/code/domain/extract/lang_text.py b/server/code/domain/extract/lang_text.py index 105294cea9..d46389647d 100644 --- a/server/code/domain/extract/lang_text.py +++ b/server/code/domain/extract/lang_text.py @@ -74,7 +74,6 @@ ".hx", ".graphql", ".gql", - ".thrift", ".avsc", ".capnp", ".asm", diff --git a/server/code/domain/extract/lang_thrift.py b/server/code/domain/extract/lang_thrift.py new file mode 100644 index 0000000000..9aafe3fc5a --- /dev/null +++ b/server/code/domain/extract/lang_thrift.py @@ -0,0 +1,374 @@ +"""Thrift IDL: structs, unions, exceptions, enums, typedefs, services and their functions. + +A `.thrift` file is the contract between the services a repository builds (the generated +code is excluded as generated); reading it gives the index the API's own names. The +language is small and regular, so a token scanner with brace tracking reads it without a +tree-sitter grammar, in the shape of `lang_proto.py`: + + * `struct X { … }`, `union X { … }` and `exception X { … }` → a `struct` symbol; + `enum X { … }` → `enum`; `typedef T X` → `type`; `service S [extends B] { … }` → + `interface`; each function in a service → a `function` symbol `::S.f` whose + signature is the function as written. Qualified names follow the module path. + * reference sites: a field's, an argument's, a return's or a `throws` entry's named + type (`field_type`), and a service's `extends` (`extends`), bound by + `lane_units.bind_sites`; base types (`i32`, `string`, …) and the element types of + `list<>` / `set<>` / `map<>` are read through, so `list` names `User`. + A type qualified by an include (`shared.User`) binds by its last part. + * `include "shared.thrift"` is an include specifier; Thrift resolves it against the + including file's directory, then the `-I` paths, unknown to the index, so the + candidates are the file's directory and its nearest ancestors, then the root + (`lane_units.ancestor_candidates`), settled against the namespace's live paths. + +`ParsedFile.tree` stays None: chunks come from `lane_units.span_chunks`, edges from the +sites. Outside the extractor stamp with the other lanes. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from server.code.domain.extract.lane_units import SiteRule, ancestor_candidates +from server.code.domain.extract.lang_python import module_scope +from server.code.domain.extract.treesitter import ParsedFile, Symbol + +LANG_THRIFT = "thrift" + +KIND_STRUCT = "struct" +KIND_ENUM = "enum" +KIND_TYPEDEF = "type" +KIND_SERVICE = "interface" +KIND_FUNCTION = "function" + +SITE_FIELD_TYPE = "field_type" +SITE_EXTENDS = "extends" + +SITE_RULES: dict[str, SiteRule] = { + SITE_FIELD_TYPE: SiteRule("uses_type", frozenset({KIND_STRUCT, KIND_ENUM, KIND_TYPEDEF})), + SITE_EXTENDS: SiteRule("inherits", frozenset({KIND_SERVICE})), +} + +# how `lane_units.span_chunks` cuts a Thrift file +CHUNK_OPTIONS: dict[str, Any] = { + "comment_leaders": ("//", "#", "/*", "*"), + "comment_blocks": (("/*", "*/"),), + "header_prefixes": ("namespace", "include", "cpp_include"), +} + +BASE_TYPES: frozenset[str] = frozenset( + {"bool", "byte", "i8", "i16", "i32", "i64", "double", "string", "binary", "uuid", "void"} +) +_CONTAINERS: frozenset[str] = frozenset({"list", "set", "map"}) +_STRUCT_KEYWORDS: frozenset[str] = frozenset({"struct", "union", "exception"}) +_FIELD_REQ: frozenset[str] = frozenset({"required", "optional"}) + +# An unclosed string is one token to the end of its line; an unclosed block comment runs +# to the end of the file, as Thrift's lexer reads them. +_TOKEN_RE = re.compile( + r"//[^\n]*|#[^\n]*|/\*.*?(?:\*/|\Z)|\"(?:[^\"\\\n]|\\.)*+\"?|'(?:[^'\\\n]|\\.)*+'?" + r"|[A-Za-z_][A-Za-z0-9_.]*|-?\d[\w.+-]*|[{}()<>,;:=\[\]]|\S", + re.S, +) +_INCLUDE_RE = re.compile(r"(?m)^[^\S\n]*include\s+[\"']([^\"'\n]+)[\"']") + + +@dataclass +class _Tok: + text: str + line: int + + +@dataclass +class ThriftUnits: + symbols: list[Symbol] = field(default_factory=list) + sites: list[tuple[int, str, str]] = field(default_factory=list) + includes: list[str] = field(default_factory=list) + + +def _tokens(text: str) -> list[_Tok]: + out: list[_Tok] = [] + line = 1 + pos = 0 + for m in _TOKEN_RE.finditer(text): + line += text.count("\n", pos, m.start()) + pos = m.start() + t = m.group(0) + if not t.startswith(("//", "#", "/*")): + out.append(_Tok(t, line)) + return out + + +def _is_name(t: str) -> bool: + return bool(t) and (t[0].isalpha() or t[0] == "_") + + +def _bare_type(name: str) -> str: + """`shared.User` / `User` → `User`; '' for a base type or a non-name.""" + if not _is_name(name) or name in BASE_TYPES or name in _CONTAINERS: + return "" + return name.rsplit(".", 1)[-1] + + +def _join(toks: list[_Tok]) -> str: + """Tokens back to source-like text: `User get(1: UserId id) throws(1: NotFound nf)`.""" + out = "" + for t in toks: + x = t.text + if out and x not in "(),:<>" and out[-1] not in "(<": + out += " " + out += x + if x in ",:": + out += " " + return re.sub(r" +", " ", out).strip() + + +class _Parser: + def __init__(self, path: str, toks: list[_Tok]) -> None: + self.path = path + self.mod = module_scope(path) + self.toks = toks + self.i = 0 + self.units = ThriftUnits() + # each `{` token's matching `}`, found in one pass + self._closes: dict[int, int] = {} + opens: list[int] = [] + for j, t in enumerate(toks): + if t.text == "{": + opens.append(j) + elif t.text == "}" and opens: + self._closes[opens.pop()] = j + + def _peek(self, k: int = 0) -> _Tok | None: + j = self.i + k + return self.toks[j] if j < len(self.toks) else None + + def _add(self, name: str, kind: str, sig: str, start: int, end: int, scope: str = "") -> None: + self.units.symbols.append( + Symbol( + qualified_name=f"{self.mod}::{scope + '.' if scope else ''}{name}", + name=name, + kind=kind, + signature=sig, + path=self.path, + start_line=start, + end_line=max(end, start), + ) + ) + + def _site(self, line: int, type_name: str, kind: str = SITE_FIELD_TYPE) -> None: + bare = _bare_type(type_name) + if bare: + self.units.sites.append((line, bare, kind)) + + def _type(self, j: int, stop: int) -> int: + """Read a type at `j` (a name, or a container with `<…>`), recording its named + types; returns the index after it.""" + if j >= stop: + return j + t = self.toks[j] + if t.text in _CONTAINERS and j + 1 < stop and self.toks[j + 1].text == "<": + depth = 0 + k = j + 1 + while k < stop: + tk = self.toks[k].text + if tk == "<": + depth += 1 + elif tk == ">": + depth -= 1 + if depth == 0: + return k + 1 + elif _is_name(tk): + self._site(self.toks[k].line, tk) + k += 1 + return k + if _is_name(t.text): + self._site(t.line, t.text) + return j + 1 + + def _fields(self, j: int, stop: int) -> None: + """Field lists (`1: required Type name = x,`) between `j` and `stop`: record each + field's type.""" + while j < stop: + t = self.toks[j].text + if t.lstrip("-").isdigit() and j + 1 < stop and self.toks[j + 1].text == ":": + j += 2 + if j < stop and self.toks[j].text in _FIELD_REQ: + j += 1 + j = self._type(j, stop) + continue + j += 1 + + def _block_end(self, open_idx: int) -> int: + j = self._closes.get(open_idx) + return j if j is not None else len(self.toks) - 1 + + def _service(self, start: int, name: str, open_idx: int) -> None: + close = self._block_end(open_idx) + self._add( + name, KIND_SERVICE, f"service {name}", self.toks[start].line, self.toks[close].line + ) + j = open_idx + 1 + while j < close: + # [oneway] ReturnType name ( args ) [throws ( … )] [, ;] + fstart = j + if self.toks[j].text == "oneway": + j += 1 + ret_end = self._type(j, close) + if ( + ret_end < close + and _is_name(self.toks[ret_end].text) + and ret_end + 1 < close + and self.toks[ret_end + 1].text == "(" + ): + fname = self.toks[ret_end].text + args_open = ret_end + 1 + k = args_open + depth = 0 + while k < close: + if self.toks[k].text == "(": + depth += 1 + elif self.toks[k].text == ")": + depth -= 1 + if depth == 0: + break + k += 1 + self._fields(args_open + 1, k) + end = k + if ( + k + 1 < close + and self.toks[k + 1].text == "throws" + and k + 2 < close + and self.toks[k + 2].text == "(" + ): + m = k + 2 + depth = 0 + while m < close: + if self.toks[m].text == "(": + depth += 1 + elif self.toks[m].text == ")": + depth -= 1 + if depth == 0: + break + m += 1 + self._fields(k + 3, m) + end = m + sig = _join(self.toks[fstart : end + 1]) + self._add( + fname, KIND_FUNCTION, sig, self.toks[fstart].line, self.toks[end].line, name + ) + j = end + 1 + else: + j = max(ret_end, j + 1) + self.i = close + 1 + + def parse(self) -> ThriftUnits: + toks = self.toks + n = len(toks) + while self.i < n: + t = toks[self.i].text + nxt = toks[self.i + 1].text if self.i + 1 < n else "" + if t in _STRUCT_KEYWORDS and _is_name(nxt): + start = self.i + j = self.i + 2 + if j < n and toks[j].text == "xsd_all": + j += 1 + if j < n and toks[j].text == "{": + close = self._block_end(j) + self._add(nxt, KIND_STRUCT, f"{t} {nxt}", toks[start].line, toks[close].line) + self._fields(j + 1, close) + self.i = close + 1 + continue + elif t == "enum" and _is_name(nxt): + start = self.i + j = self.i + 2 + if j < n and toks[j].text == "{": + close = self._block_end(j) + self._add(nxt, KIND_ENUM, f"enum {nxt}", toks[start].line, toks[close].line) + self.i = close + 1 + continue + elif t == "typedef": + start = self.i + j = self._type(self.i + 1, n) + if j < n and _is_name(toks[j].text): + self._add( + toks[j].text, + KIND_TYPEDEF, + " ".join(x.text for x in toks[start : j + 1]), + toks[start].line, + toks[j].line, + ) + self.i = j + 1 + continue + self.i = max(self.i + 1, j) + continue + elif t == "service" and _is_name(nxt): + start = self.i + j = self.i + 2 + if j + 1 < n and toks[j].text == "extends": + self._site(toks[j + 1].line, toks[j + 1].text, SITE_EXTENDS) + j += 2 + if j < n and toks[j].text == "{": + self._service(start, nxt, j) + continue + self.i += 1 + return self.units + + +def thrift_units(path: str, text: str) -> ThriftUnits: + units = _Parser(path, _tokens(text)).parse() + units.includes = list(dict.fromkeys(_INCLUDE_RE.findall(text))) + return units + + +def parse_thrift(path: str, source: bytes) -> ParsedFile: + units = thrift_units(path, source.decode("utf-8", "replace")) + pf = ParsedFile(path=path, lang=LANG_THRIFT, source=source) + pf.symbols = units.symbols + pf.includes = units.includes + return pf + + +def reference_sites(pf: ParsedFile) -> list[tuple[int, str, str]]: + """(line, bare type name, `field_type` | `extends`) for every named type a field, + argument, return, `throws` entry or `extends` names, in source order.""" + return thrift_units(pf.path, pf.source.decode("utf-8", "replace")).sites + + +def wanted_paths(from_path: str, specs: list[str]) -> list[str]: + """Every repository path the includes may name, for one membership read.""" + out: dict[str, None] = {} + for spec in specs: + out.update(dict.fromkeys(ancestor_candidates(from_path, spec))) + return list(out) + + +def resolve_imports(from_path: str, specs: list[str], present: set[str]) -> list[str]: + """One indexed file per include: the nearest candidate `present` holds.""" + out: dict[str, None] = {} + for spec in specs: + hit = next((c for c in ancestor_candidates(from_path, spec) if c in present), None) + if hit and hit != from_path: + out[hit] = None + return list(out) + + +__all__ = [ + "BASE_TYPES", + "CHUNK_OPTIONS", + "KIND_ENUM", + "KIND_FUNCTION", + "KIND_SERVICE", + "KIND_STRUCT", + "KIND_TYPEDEF", + "LANG_THRIFT", + "SITE_EXTENDS", + "SITE_FIELD_TYPE", + "SITE_RULES", + "ThriftUnits", + "parse_thrift", + "reference_sites", + "resolve_imports", + "thrift_units", + "wanted_paths", +] diff --git a/server/code/domain/extract/name_scope.py b/server/code/domain/extract/name_scope.py index 924f950432..29d20cea6f 100644 --- a/server/code/domain/extract/name_scope.py +++ b/server/code/domain/extract/name_scope.py @@ -67,6 +67,7 @@ "rust": NATIVE, # Protobuf and Go-template names are module-scoped `::Name` like Python's "proto": SCRIPT, + "thrift": SCRIPT, "gotemplate": SCRIPT, "hcl": SCRIPT, # Dotted directory scopes "k8s": SCRIPT, # Dotted file scopes diff --git a/server/code/domain/search_scope.py b/server/code/domain/search_scope.py index 548da6c71e..d2d53cba84 100644 --- a/server/code/domain/search_scope.py +++ b/server/code/domain/search_scope.py @@ -56,6 +56,7 @@ "json": (".json",), "toml": (".toml",), "proto": (".proto",), + "thrift": (".thrift",), "cmake": (".cmake", "CMakeLists.txt"), "xml": (".xml",), "html": (".html", ".htm"), diff --git a/tests/code_lanes/corpus.py b/tests/code_lanes/corpus.py index 406c9af31e..273e016c84 100644 --- a/tests/code_lanes/corpus.py +++ b/tests/code_lanes/corpus.py @@ -142,6 +142,12 @@ def langs_for(self, path: str) -> frozenset[str]: ), Lane("prisma/", "prisma", "models, relations, an enum and attributes of the vendored schema"), Lane("proto/", "proto", "a package, an import, messages, an enum, a service and its RPCs"), + Lane( + "thrift/", + "thrift", + "an include, a typedef, an enum, a struct, a union, an exception, a service " + "extending an included one, and its functions", + ), Lane( "python/", "python", @@ -420,7 +426,7 @@ def _summarize() -> dict[str, Any]: pf.includes.append(hit) elif pf.lang == "rust": pf.includes = resolve_uses(rel, pf.includes, present) - elif pf.lang in ("proto", "gotemplate"): + elif pf.lang in ("proto", "thrift", "gotemplate"): # Import candidates settle against the corpus paths lane = _lane_module(pf.lang) pf.includes = lane.resolve_imports(rel, list(pf.includes), present) @@ -547,7 +553,7 @@ def _summarize() -> dict[str, Any]: ] if pf.lang == "prisma": es = [*es, *prisma_edges(pf)] - if pf.lang in ("proto", "gotemplate"): + if pf.lang in ("proto", "thrift", "gotemplate"): # The reference sites over the lane's table lane = _lane_module(pf.lang) es = [*es, *bind_sites(pf, lane.reference_sites(pf), gated, lane.SITE_RULES)] @@ -571,7 +577,14 @@ def _summarize() -> dict[str, Any]: sql_gated = gate_name_table(name_table, SQL_LANG) es = [*es, *sql_edges(rel, module_scope(rel), statements, pf.symbols, sql_gated)] es.extend(_column_edges(rel, statements, pf, sql_gated)) - if pf.lang in ts_langs or pf.lang in ("rust", "proto", "gotemplate", "config", "python"): + if pf.lang in ts_langs or pf.lang in ( + "rust", + "proto", + "thrift", + "gotemplate", + "config", + "python", + ): files[rel]["includes"] = list(pf.includes) for e in es: target = kind_of.get(e.to_symbol_id, "file" if e.edge_kind == "includes" else "?") @@ -583,8 +596,10 @@ def _summarize() -> dict[str, Any]: def _lane_module(lang: str) -> Any: """The tree-less lane modules, as `ingest._lane_module`.""" - from server.code.domain.extract import lang_gotemplate, lang_proto + from server.code.domain.extract import lang_gotemplate, lang_proto, lang_thrift + if lang == "thrift": + return lang_thrift return lang_proto if lang == "proto" else lang_gotemplate diff --git a/tests/fixtures/code_lanes/NOTICE.md b/tests/fixtures/code_lanes/NOTICE.md index d2d63a90e4..54ac5c2dba 100644 --- a/tests/fixtures/code_lanes/NOTICE.md +++ b/tests/fixtures/code_lanes/NOTICE.md @@ -28,7 +28,7 @@ bats test, a Jinja template, a platform-suffixed Go file), so don't reformat or | java/ | synthetic: a Spring / JDBC repository, a generic interface and a generic class implementing it | — | | js/, ts/constructs/, ts/synthetic/ | synthetic: CommonJS, IIFE, class expressions, overloads, React components, SQL in template literals | — | | vue/ | synthetic single-file components and a composable | — | -| proto/, helm/ | synthetic | — | +| proto/, thrift/, helm/ | synthetic | — | | rust/synthetic/ | synthetic: a trait and its impl, a declarative macro, an async function | — | | sql/views/ | synthetic: a ClickHouse table, a materialized view into its rollup, a view and a CTE | — | | tla/, fbs/, cmake/, md/ | synthetic: one toy ledger described in TLA+, FlatBuffers, CMake and Markdown (with fenced code) | — | diff --git a/tests/fixtures/code_lanes/expected.json b/tests/fixtures/code_lanes/expected.json index dd752a6790..fb561bca70 100644 --- a/tests/fixtures/code_lanes/expected.json +++ b/tests/fixtures/code_lanes/expected.json @@ -113,6 +113,11 @@ "sql/views/rollup.sql | calls | column": 3, "sql/views/rollup.sql | calls | table": 4, "sql/views/rollup.sql | calls | view": 1, + "thrift/acme/machines.thrift | includes | file": 1, + "thrift/acme/machines.thrift | inherits | interface": 1, + "thrift/acme/machines.thrift | uses_type | enum": 2, + "thrift/acme/machines.thrift | uses_type | struct": 7, + "thrift/acme/machines.thrift | uses_type | type": 6, "tla/Ledger.cfg | calls | operator": 3, "tla/Ledger.cfg | includes | file": 1, "tla/Ledger.tla | calls | constant": 3, @@ -1665,6 +1670,31 @@ "view": 2 } }, + "thrift/acme/machines.thrift": { + "facets": {}, + "includes": [ + "thrift/acme/shared.thrift" + ], + "lang": "thrift", + "symbols": { + "enum": 1, + "function": 4, + "interface": 1, + "struct": 3, + "type": 1 + } + }, + "thrift/acme/shared.thrift": { + "facets": {}, + "includes": [], + "lang": "thrift", + "symbols": { + "function": 1, + "interface": 1, + "struct": 2, + "type": 1 + } + }, "tla/Ledger.cfg": { "facets": { "checks": 3 diff --git a/tests/fixtures/code_lanes/thrift/acme/machines.thrift b/tests/fixtures/code_lanes/thrift/acme/machines.thrift new file mode 100644 index 0000000000..da359c698f --- /dev/null +++ b/tests/fixtures/code_lanes/thrift/acme/machines.thrift @@ -0,0 +1,37 @@ +namespace py acme.machines +namespace java com.acme.machines + +include "shared.thrift" + +typedef string MachineId + +enum State { + PENDING = 1, + RUNNING = 2, + STOPPED = 3 +} + +/* A machine as the API returns it */ +struct Machine { + 1: required MachineId id, + 2: optional State state = State.PENDING, + 3: shared.Timestamp created_at, + 4: map labels +} + +union Selector { + 1: MachineId id, + 2: list label_keys +} + +exception InvalidState { + 1: State current, + 2: string reason +} + +service Machines extends shared.BaseService { + Machine get(1: MachineId id) throws (1: shared.NotFound not_found), + list list_machines(1: shared.Page page, 2: Selector selector), + void stop(1: MachineId id) throws (1: shared.NotFound not_found, 2: InvalidState invalid), + oneway void touch(1: MachineId id) +} diff --git a/tests/fixtures/code_lanes/thrift/acme/shared.thrift b/tests/fixtures/code_lanes/thrift/acme/shared.thrift new file mode 100644 index 0000000000..1f8d29e533 --- /dev/null +++ b/tests/fixtures/code_lanes/thrift/acme/shared.thrift @@ -0,0 +1,17 @@ +namespace py acme.shared + +// Types every acme service shares +typedef i64 Timestamp + +struct Page { + 1: required i32 offset, + 2: required i32 limit +} + +exception NotFound { + 1: string message +} + +service BaseService { + string version() +} diff --git a/tests/unit/lane_walks.py b/tests/unit/lane_walks.py index a1a0702834..2a80390198 100644 --- a/tests/unit/lane_walks.py +++ b/tests/unit/lane_walks.py @@ -323,6 +323,10 @@ def walk(text: str) -> object: return lang_rust.wanted_paths("src/a/b.rs", specs) if lane == "go": return lang_go.go_wanted_paths(specs, [("m", "")]) + if lane == "thrift": + from server.code.domain.extract import lang_thrift + + return lang_thrift.wanted_paths("a/b/c.thrift", specs) return lang_proto.wanted_paths("a/b/c.proto", specs) return walk @@ -430,6 +434,7 @@ def _inline_templates(n: int) -> str: ("rust-long-use", _wanted("rust"), _run("", "a::")), ("go-imports", _wanted("go"), _numbered("m/p{}\n")), ("proto-imports", _wanted("proto"), _numbered("p{}.proto\n")), + ("thrift-includes", _wanted("thrift"), _numbered("t{}.thrift\n")), ("python-route-rows", _py_routes, _run("", "app.add_url_rule('a')\n")), ("python-unclosed-include-router", _py_routes, _run("", "a.include_router(a,")), ( @@ -538,6 +543,7 @@ def cost(n: int) -> float: "astro": ("a.astro", "", "typescript"), "angular_template": ("a.component.html", "", "typescript"), "proto": ("a.proto", "", "proto"), + "thrift": ("a.thrift", "", "thrift"), "gotemplate": ("chart/templates/a.yaml", "{{ .Values.a }}\n", "gotemplate"), "hcl": ("a.tf", "", "hcl"), "k8s": ("a.yaml", "apiVersion: v1\nkind: ConfigMap\n", "k8s"), diff --git a/tests/unit/test_code_extractor_stamp.py b/tests/unit/test_code_extractor_stamp.py index 0bb637818f..6839629610 100644 --- a/tests/unit/test_code_extractor_stamp.py +++ b/tests/unit/test_code_extractor_stamp.py @@ -85,6 +85,10 @@ "protobuf scanner: messages, enums, services, rpcs, type sites and " "import candidates; applied by a targeted backfill over a namespace's .proto files" ), + "lang_thrift.py": ( + "Thrift IDL scanner: structs, enums, typedefs, services, functions, type sites " + "and include candidates — a lane module outside the stamp like lang_proto.py" + ), "lang_jsonnet.py": ( "jsonnet bindings, members and imports — a lane module outside the " "stamp like lang_proto.py; rows arrive by a targeted backfill" diff --git a/tests/unit/test_code_lanes_manifest.py b/tests/unit/test_code_lanes_manifest.py index 0bae78c2a8..a17ec1546f 100644 --- a/tests/unit/test_code_lanes_manifest.py +++ b/tests/unit/test_code_lanes_manifest.py @@ -23,7 +23,7 @@ # The languages whose summary row records includes resolved against the # corpus's own paths; the others record the specifier as written. RESOLVED_INCLUDE_LANGS = frozenset( - {"typescript", "javascript", "rust", "proto", "gotemplate", "python"} + {"typescript", "javascript", "rust", "proto", "thrift", "gotemplate", "python"} ) _EDGE_KEY = re.compile(r"^(?P.+) \| (?P[a-z_]+) \| (?P[a-z_?]+)$") _SECRET = re.compile( diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py new file mode 100644 index 0000000000..28c52592ca --- /dev/null +++ b/tests/unit/test_code_lang_thrift.py @@ -0,0 +1,149 @@ +"""The Thrift IDL lane: symbols, reference sites, includes and registration.""" + +from __future__ import annotations + +import time + +from server.code.app.index.ingest_parse import LANE_UNIT_LANGS, _lane_module +from server.code.domain.extract import lang_thrift as th +from server.code.domain.extract.lane_units import bind_sites +from server.code.domain.extract.lang_detect import detect_by_extension +from server.code.domain.extract.name_scope import gate_name_table +from tests.unit.code_fixtures import assert_linear + +SRC = b"""namespace py acme +include "shared.thrift" +include "shared.thrift" + +typedef i64 UserId // trailing comment +enum Color { RED = 1, BLUE = 2 } + +/* struct Ghost { } inside a comment is not a symbol */ +struct User { + 1: required UserId id, + 2: optional list colors, + 3: map addresses, +} + +union Key { 1: UserId id, 2: string email } +exception NotFound { 1: string msg } + +service Users extends shared.Base { + User get(1: UserId id) throws (1: NotFound nf), + oneway void ping(); + set all() +} +""" + + +def _units() -> th.ThriftUnits: + return th.thrift_units("idl/users.thrift", SRC.decode()) + + +def test_symbols_kinds_and_names() -> None: + got = {(s.kind, s.qualified_name) for s in _units().symbols} + assert got == { + ("type", "idl.users::UserId"), + ("enum", "idl.users::Color"), + ("struct", "idl.users::User"), + ("struct", "idl.users::Key"), + ("struct", "idl.users::NotFound"), + ("interface", "idl.users::Users"), + ("function", "idl.users::Users.get"), + ("function", "idl.users::Users.ping"), + ("function", "idl.users::Users.all"), + } + + +def test_spans_and_signatures() -> None: + by = {s.name: s for s in _units().symbols} + assert (by["User"].start_line, by["User"].end_line) == (9, 13) + assert (by["Users"].start_line, by["Users"].end_line) == (18, 22) + assert by["get"].signature == "User get(1: UserId id) throws(1: NotFound nf)" + assert by["ping"].signature == "oneway void ping()" + assert by["all"].signature == "set all()" + assert by["NotFound"].signature == "exception NotFound" + assert by["UserId"].signature == "typedef i64 UserId" + + +def test_reference_sites_read_through_containers_and_includes() -> None: + sites = _units().sites + assert (10, "UserId", "field_type") in sites + assert (11, "Color", "field_type") in sites + assert (12, "Address", "field_type") in sites # `shared.Address` binds by its last part + assert (18, "Base", "extends") in sites + assert {(19, "User", "field_type"), (19, "NotFound", "field_type")} <= set(sites) + assert (21, "User", "field_type") in sites + names = {n for _, n, _ in sites} + assert not names & th.BASE_TYPES + assert "Ghost" not in {s.name for s in _units().symbols} + + +def test_includes_are_deduplicated_and_resolved_nearest_first() -> None: + assert _units().includes == ["shared.thrift"] + present = {"idl/shared.thrift", "shared.thrift"} + assert th.resolve_imports("idl/users.thrift", ["shared.thrift"], present) == [ + "idl/shared.thrift" + ] + assert "idl/shared.thrift" in th.wanted_paths("idl/users.thrift", ["shared.thrift"]) + + +def test_unterminated_input_does_not_raise() -> None: + for text in ("struct A {", "service S { void f(", "typedef list<", 'include "x', "/* open"): + th.thrift_units("a.thrift", text) + + +def test_registered_as_a_lane() -> None: + assert detect_by_extension("api/users.thrift") == "thrift" + assert "thrift" in LANE_UNIT_LANGS + assert _lane_module("thrift") is th + pf = th.parse_thrift("a.thrift", SRC) + assert pf.lang == "thrift" and pf.tree is None and pf.includes == ["shared.thrift"] + + +def test_edges_bind_same_file_then_repo_wide_and_builtins_to_nothing() -> None: + pf = th.parse_thrift("idl/users.thrift", SRC) + table = { + "Address": [("sid1", "idl.shared::Address", "struct")], + "Base": [("sid2", "idl.shared::Base", "interface")], + "Ghost": [("sid3", "a::Ghost", "struct"), ("sid4", "b::Ghost", "struct")], + } + edges = bind_sites(pf, th.reference_sites(pf), gate_name_table(table, "thrift"), th.SITE_RULES) + got = { + (e.from_qualified_name.split("::")[1], e.to_qualified_name.split("::")[1], e.edge_kind) + for e in edges + } + assert ("User", "UserId", "uses_type") in got # same file + assert ("User", "Color", "uses_type") in got # inside list<> + assert ("User", "Address", "uses_type") in got # included, repository-unique + assert ("Users", "Base", "inherits") in got + assert ("Users.get", "NotFound", "uses_type") in got # throws + assert ("Users.all", "User", "uses_type") in got # return inside set<> + assert not any(t in th.BASE_TYPES for _f, t, _k in got) + + +def _cost(n: int) -> float: + text = "".join( + f"struct S{i} {{ 1: list a }}\nservice V{i} {{ S{i} f(1: S{i} x) }}\n" + for i in range(n) + ) + best = float("inf") + for _ in range(3): + t = time.perf_counter() + th.thrift_units("a.thrift", text) + best = min(best, time.perf_counter() - t) + return best + + +def test_a_file_of_many_definitions_parses_in_linear_time() -> None: + assert_linear(_cost, 1_000, 4_000) + + +def test_unclosed_headers_repeated_cost_linear_time() -> None: + def cost(n: int) -> float: + text = "typedef list< struct A service S extends " * n + t = time.perf_counter() + th.thrift_units("a.thrift", text) + return time.perf_counter() - t + + assert_linear(cost, 2_000, 8_000) From 3665dc77a0de96d1a6b2267e80a8d5d3b5fed882 Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:27:11 +0530 Subject: [PATCH 02/11] test: give the Thrift fixtures a test-inventory workflow Co-Authored-By: Claude Opus 5.5 Signed-off-by: Rahul <132360698+rahulsjha@users.noreply.github.com> --- tests/inventory/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/inventory/build.py b/tests/inventory/build.py index 0bd1519d48..b6cd431633 100644 --- a/tests/inventory/build.py +++ b/tests/inventory/build.py @@ -85,6 +85,7 @@ "rust", "shell", "sql", + "thrift", "tla", "ts", "vue", From 0d527c95627c22bf4335dc3e11b74e17549da9cc Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:28:41 +0530 Subject: [PATCH 03/11] docs: format the language-support table with prettier --- docs/development/adding-language-support.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/development/adding-language-support.md b/docs/development/adding-language-support.md index dc99bbdde5..487a7321d5 100644 --- a/docs/development/adding-language-support.md +++ b/docs/development/adding-language-support.md @@ -33,13 +33,13 @@ Every module under `server/code/domain/extract/` is either in a stage of `STAGE_ ## 1. Choose the lane's shape -| Shape | Examples | You provide | -| ----------------- | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Grammar lane | `lang_typescript.py`, `lang_rust.py`, `lang_go.py`, `lang_java.py` | A tree-sitter grammar dependency in `pyproject.toml`; symbols and call sites from the tree; chunks and edges come from the shared chunker and edge walker. | +| Shape | Examples | You provide | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Grammar lane | `lang_typescript.py`, `lang_rust.py`, `lang_go.py`, `lang_java.py` | A tree-sitter grammar dependency in `pyproject.toml`; symbols and call sites from the tree; chunks and edges come from the shared chunker and edge walker. | | Tree-less lane | `lang_proto.py`, `lang_thrift.py`, `lang_gotemplate.py`, `lang_hcl.py`, `lang_k8s.py`, `lang_jsonnet.py` (`LANE_UNIT_LANGS`) | A hand-written scanner; `ParsedFile.tree` stays `None`; chunks from `lane_units.span_chunks`, edges from `reference_sites` + `SITE_RULES` bound by `lane_units.bind_sites`. | -| Include-only lane | `tla`, `flatbuffers`, `cmake`, `doc` (`_LANE_INCLUDE_LANGS`) | Symbols and includes resolved by the simple lane resolver. | -| Embedded language | SQL inside Go and Java (`go_sql_units`, `java_sql_units`), shell islands in Dockerfiles and Makefiles | Units joined into the host file's parse (`_join_island_symbols`), usually behind a kill switch setting. | -| Chunk-only | `.css`, text files (`lang_text.py`) | Nothing but detection: the file is chunked and searchable with no symbols. | +| Include-only lane | `tla`, `flatbuffers`, `cmake`, `doc` (`_LANE_INCLUDE_LANGS`) | Symbols and includes resolved by the simple lane resolver. | +| Embedded language | SQL inside Go and Java (`go_sql_units`, `java_sql_units`), shell islands in Dockerfiles and Makefiles | Units joined into the host file's parse (`_join_island_symbols`), usually behind a kill switch setting. | +| Chunk-only | `.css`, text files (`lang_text.py`) | Nothing but detection: the file is chunked and searchable with no symbols. | A grammar lane's grammar is **not** in `STAMPED_GRAMMARS`, so upgrading it moves no stamp. The lane owns re-deriving its rows. From e78d11748fdd91685288e8eb4c8d1d11b73f952e Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:49:29 +0530 Subject: [PATCH 04/11] fix(code): read Thrift includes from the comment-aware tokens An include inside a block comment was read by the raw-source regex and became a dependency edge. Includes now come from the token stream, which drops comments. --- server/code/domain/extract/lang_thrift.py | 13 +++++++++++-- tests/unit/test_code_lang_thrift.py | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/server/code/domain/extract/lang_thrift.py b/server/code/domain/extract/lang_thrift.py index 9aafe3fc5a..c221803de4 100644 --- a/server/code/domain/extract/lang_thrift.py +++ b/server/code/domain/extract/lang_thrift.py @@ -70,7 +70,6 @@ r"|[A-Za-z_][A-Za-z0-9_.]*|-?\d[\w.+-]*|[{}()<>,;:=\[\]]|\S", re.S, ) -_INCLUDE_RE = re.compile(r"(?m)^[^\S\n]*include\s+[\"']([^\"'\n]+)[\"']") @dataclass @@ -103,6 +102,11 @@ def _is_name(t: str) -> bool: return bool(t) and (t[0].isalpha() or t[0] == "_") +def _is_literal(t: str) -> bool: + """A closed, non-empty string literal: `"shared.thrift"` or `'shared.thrift'`.""" + return len(t) > 2 and t[0] in "\"'" and t[-1] == t[0] + + def _bare_type(name: str) -> str: """`shared.User` / `User` → `User`; '' for a base type or a non-name.""" if not _is_name(name) or name in BASE_TYPES or name in _CONTAINERS: @@ -279,6 +283,11 @@ def parse(self) -> ThriftUnits: self._fields(j + 1, close) self.i = close + 1 continue + elif t == "include" and _is_literal(nxt): + # read from the tokens, so an include inside a comment is not one + self.units.includes.append(nxt[1:-1]) + self.i += 2 + continue elif t == "enum" and _is_name(nxt): start = self.i j = self.i + 2 @@ -317,7 +326,7 @@ def parse(self) -> ThriftUnits: def thrift_units(path: str, text: str) -> ThriftUnits: units = _Parser(path, _tokens(text)).parse() - units.includes = list(dict.fromkeys(_INCLUDE_RE.findall(text))) + units.includes = list(dict.fromkeys(units.includes)) return units diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py index 28c52592ca..28dc04511e 100644 --- a/tests/unit/test_code_lang_thrift.py +++ b/tests/unit/test_code_lang_thrift.py @@ -88,6 +88,15 @@ def test_includes_are_deduplicated_and_resolved_nearest_first() -> None: assert "idl/shared.thrift" in th.wanted_paths("idl/users.thrift", ["shared.thrift"]) +def test_includes_inside_comments_are_not_includes() -> None: + src = b'/*\ninclude "unused.thrift"\n*/\nstruct Live { 1: i32 x }\n' + assert th.parse_thrift("idl/main.thrift", src).includes == [] + text = '// include "a.thrift"\n# include "b.thrift"\ninclude "c.thrift" /* include "d.thrift" */\n' + assert th.thrift_units("m.thrift", text).includes == ["c.thrift"] + # an unclosed string is not a file name + assert th.thrift_units("m.thrift", 'include "x').includes == [] + + def test_unterminated_input_does_not_raise() -> None: for text in ("struct A {", "service S { void f(", "typedef list<", 'include "x', "/* open"): th.thrift_units("a.thrift", text) From 3ee5f74c5583292e91d8a41bbd022a7850421698 Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:35:54 +0530 Subject: [PATCH 05/11] test: format the Thrift include test with ruff --- tests/unit/test_code_lang_thrift.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py index 28dc04511e..65ea92acd2 100644 --- a/tests/unit/test_code_lang_thrift.py +++ b/tests/unit/test_code_lang_thrift.py @@ -91,7 +91,9 @@ def test_includes_are_deduplicated_and_resolved_nearest_first() -> None: def test_includes_inside_comments_are_not_includes() -> None: src = b'/*\ninclude "unused.thrift"\n*/\nstruct Live { 1: i32 x }\n' assert th.parse_thrift("idl/main.thrift", src).includes == [] - text = '// include "a.thrift"\n# include "b.thrift"\ninclude "c.thrift" /* include "d.thrift" */\n' + text = ( + '// include "a.thrift"\n# include "b.thrift"\ninclude "c.thrift" /* include "d.thrift" */\n' + ) assert th.thrift_units("m.thrift", text).includes == ["c.thrift"] # an unclosed string is not a file name assert th.thrift_units("m.thrift", 'include "x').includes == [] From 56d86c576a9b477b5cc0515e01d4d6d0871b41c7 Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:28:26 +0530 Subject: [PATCH 06/11] fix(thrift): read field types when the numeric field ID is absent The IDL grammar makes FieldID optional in struct fields, argument lists and throws lists. _fields now reads each field as [ID:] [req] Type [&] name [= value] [xsd options] [(annotations)], so an ID-less field's type is a reference site, and default values and annotations are skipped. Signed-off-by: Rahul <132360698+rahulsjha@users.noreply.github.com> --- server/code/domain/extract/lang_thrift.py | 61 +++++++++++++++++++---- tests/unit/test_code_lang_thrift.py | 52 +++++++++++++++++++ 2 files changed, 104 insertions(+), 9 deletions(-) diff --git a/server/code/domain/extract/lang_thrift.py b/server/code/domain/extract/lang_thrift.py index c221803de4..81f6e718c1 100644 --- a/server/code/domain/extract/lang_thrift.py +++ b/server/code/domain/extract/lang_thrift.py @@ -62,6 +62,8 @@ _CONTAINERS: frozenset[str] = frozenset({"list", "set", "map"}) _STRUCT_KEYWORDS: frozenset[str] = frozenset({"struct", "union", "exception"}) _FIELD_REQ: frozenset[str] = frozenset({"required", "optional"}) +_XSD_FIELD_FLAGS: frozenset[str] = frozenset({"xsd_optional", "xsd_nillable"}) +_PAIRS: dict[str, str] = {"(": ")", "[": "]", "{": "}", "<": ">"} # An unclosed string is one token to the end of its line; an unclosed block comment runs # to the end of the file, as Thrift's lexer reads them. @@ -190,18 +192,59 @@ def _type(self, j: int, stop: int) -> int: self._site(t.line, t.text) return j + 1 - def _fields(self, j: int, stop: int) -> None: - """Field lists (`1: required Type name = x,`) between `j` and `stop`: record each - field's type.""" + def _group_end(self, j: int, stop: int) -> int: + """At an opening bracket, the index after its match (or `stop`); else `j + 1`.""" + close = _PAIRS.get(self.toks[j].text) + if close is None: + return j + 1 + opener = self.toks[j].text + depth = 0 while j < stop: t = self.toks[j].text - if t.lstrip("-").isdigit() and j + 1 < stop and self.toks[j + 1].text == ":": - j += 2 - if j < stop and self.toks[j].text in _FIELD_REQ: - j += 1 - j = self._type(j, stop) - continue + if t == opener: + depth += 1 + elif t == close: + depth -= 1 + if depth == 0: + return j + 1 + j += 1 + return stop + + def _field(self, j: int, stop: int) -> int: + """One field, `[1:] [required|optional] Type [&] name [= value] [xsd options] + [(annotations)]`, as in the IDL grammar, where the numeric ID is optional: record + its type and return the index after it.""" + t = self.toks[j].text + if t.lstrip("-").isdigit() and j + 1 < stop and self.toks[j + 1].text == ":": + j += 2 + if j < stop and self.toks[j].text in _FIELD_REQ: + j += 1 + j = self._type(j, stop) + for skip in ("(", "&"): # type annotations, then a C++ reference marker + if j < stop and self.toks[j].text == skip: + j = self._group_end(j, stop) + if j < stop and _is_name(self.toks[j].text): + j += 1 + if j < stop and self.toks[j].text == "=" and j + 1 < stop: + j = self._group_end(j + 1, stop) + while j < stop and self.toks[j].text in _XSD_FIELD_FLAGS: j += 1 + if j + 1 < stop and self.toks[j].text == "xsd_attrs" and self.toks[j + 1].text == "{": + attrs_end = self._group_end(j + 1, stop) + self._fields(j + 2, attrs_end - 1) + j = attrs_end + if j < stop and self.toks[j].text == "(": + j = self._group_end(j, stop) + return j + + def _fields(self, j: int, stop: int) -> None: + """Field lists between `j` and `stop`, with or without numeric IDs and separators: + record each field's type.""" + while j < stop: + if self.toks[j].text in ",;": + j += 1 + continue + j = max(self._field(j, stop), j + 1) def _block_end(self, open_idx: int) -> int: j = self._closes.get(open_idx) diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py index 65ea92acd2..ab1018b784 100644 --- a/tests/unit/test_code_lang_thrift.py +++ b/tests/unit/test_code_lang_thrift.py @@ -99,6 +99,58 @@ def test_includes_inside_comments_are_not_includes() -> None: assert th.thrift_units("m.thrift", 'include "x').includes == [] +def _uses_type(path: str, src: bytes) -> set[tuple[str, str]]: + pf = th.parse_thrift(path, src) + edges = bind_sites(pf, th.reference_sites(pf), gate_name_table({}, "thrift"), th.SITE_RULES) + return { + (e.from_qualified_name.split("::")[1], e.to_qualified_name.split("::")[1]) + for e in edges + if e.edge_kind == "uses_type" + } + + +def test_fields_without_a_numeric_id_record_their_types() -> None: + # the IDL grammar makes `FieldID ':'` optional + src = b"struct User { 1: string id } +struct Envelope { optional User user } +" + assert (2, "User", "field_type") in th.parse_thrift("idl/e.thrift", src).sites + assert ("Envelope", "User") in _uses_type("idl/e.thrift", src) + with_id = b"struct User { 1: string id } +struct Envelope { 1: optional User user } +" + assert ("Envelope", "User") in _uses_type("idl/e.thrift", with_id) + + +def test_id_less_fields_in_arguments_throws_and_mixed_lists() -> None: + src = b"""struct Id { 1: i64 v } +struct Tag { 1: string t } +exception Missing { 1: string why } +struct Mixed { + required Id id = {"v": 1} + list (python.immutable = "") tags; + 3: Tag& primary xsd_optional (doc = "x"), + map byName +} +service Lookup { + Mixed find(Id id, 2: Tag tag) throws (Missing missing) +} +""" + sites = th.parse_thrift("idl/l.thrift", src).sites + assert { + (5, "Id", "field_type"), + (6, "Tag", "field_type"), + (7, "Tag", "field_type"), + (8, "Id", "field_type"), + } <= set(sites) + got = _uses_type("idl/l.thrift", src) + assert {("Mixed", "Id"), ("Mixed", "Tag")} <= got + assert {("Lookup.find", "Id"), ("Lookup.find", "Tag"), ("Lookup.find", "Missing")} <= got + # field names, default values and annotations are not types + names = {n for _, n, _ in sites} + assert not names & {"id", "tags", "primary", "byName", "v", "python", "doc", "x"} + + def test_unterminated_input_does_not_raise() -> None: for text in ("struct A {", "service S { void f(", "typedef list<", 'include "x', "/* open"): th.thrift_units("a.thrift", text) From 2402850d364fe4d8a7c58a1f62a33d7052def14d Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:59:09 +0530 Subject: [PATCH 07/11] test(thrift): cover struct, argument and throws fields without a numeric ID Signed-off-by: Rahul <132360698+rahulsjha@users.noreply.github.com> --- tests/unit/test_code_lang_thrift.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py index ab1018b784..3f075a0f8a 100644 --- a/tests/unit/test_code_lang_thrift.py +++ b/tests/unit/test_code_lang_thrift.py @@ -111,14 +111,10 @@ def _uses_type(path: str, src: bytes) -> set[tuple[str, str]]: def test_fields_without_a_numeric_id_record_their_types() -> None: # the IDL grammar makes `FieldID ':'` optional - src = b"struct User { 1: string id } -struct Envelope { optional User user } -" - assert (2, "User", "field_type") in th.parse_thrift("idl/e.thrift", src).sites + src = b"struct User { 1: string id }\nstruct Envelope { optional User user }\n" + assert (2, "User", "field_type") in th.thrift_units("idl/e.thrift", src.decode()).sites assert ("Envelope", "User") in _uses_type("idl/e.thrift", src) - with_id = b"struct User { 1: string id } -struct Envelope { 1: optional User user } -" + with_id = b"struct User { 1: string id }\nstruct Envelope { 1: optional User user }\n" assert ("Envelope", "User") in _uses_type("idl/e.thrift", with_id) @@ -136,7 +132,7 @@ def test_id_less_fields_in_arguments_throws_and_mixed_lists() -> None: Mixed find(Id id, 2: Tag tag) throws (Missing missing) } """ - sites = th.parse_thrift("idl/l.thrift", src).sites + sites = th.thrift_units("idl/l.thrift", src.decode()).sites assert { (5, "Id", "field_type"), (6, "Tag", "field_type"), From 920ce3d13e5450cbcf8f6acbb0f27f07de2dca5d Mon Sep 17 00:00:00 2001 From: Rahul <132360698+rahulsjha@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:32:08 +0530 Subject: [PATCH 08/11] Move the Thrift fixture out of the plain-text cases A .thrift file now goes to the Thrift lane, so assert that the ingest dispatcher picks the lane and its service symbols instead. --- tests/unit/test_lang_text.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_lang_text.py b/tests/unit/test_lang_text.py index 5dd4051703..69677ac9a0 100644 --- a/tests/unit/test_lang_text.py +++ b/tests/unit/test_lang_text.py @@ -28,7 +28,6 @@ def test_code_without_a_parser_and_plain_text_are_kept_as_text() -> None: "notes.txt": b"hello", "setup.cfg": b"[flake8]\nmax-line-length = 100\n", "gradle/build.gradle": b"plugins {}", - "api/service.thrift": b"service Q { i32 a() }", "LICENSE": b"MIT", "Justfile": b"build:\n cargo build", }.items(): @@ -37,6 +36,14 @@ def test_code_without_a_parser_and_plain_text_are_kept_as_text() -> None: assert pf is not None and pf.lang == TEXT_LANG and pf.symbols == [] and err == "", path +def test_thrift_goes_to_its_lane_not_to_text() -> None: + src = b"service Q { i32 a() }" + assert text_lang_for_path("api/service.thrift", src) == "" + pf, err = parse_for_ingest("api/service.thrift", src) + assert pf is not None and pf.lang == "thrift" and err == "" + assert {(s.kind, s.name) for s in pf.symbols} >= {("interface", "Q"), ("function", "a")} + + def test_binaries_oversized_and_empty_stay_out_and_claimed_types_are_untouched() -> None: assert text_lang_for_path("img/a.png", b"\x89PNG") == "" assert text_lang_for_path("dist/app.min.js", b"x") == "" From 96be03c3d5fa4062c8ca44226a5e138642c86504 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:40:07 +0000 Subject: [PATCH 09/11] Replicated erasure fixtures provide isolated write leases Fenced memory inserts require a lease table in the selected schema. Give each fixture database its own Keeper root while preserving the replicated content checks. Signed-off-by: zlareb1 --- .../integration/test_erasure_replicated_ch.py | 5 + tests/unit/test_erasure_replicated_fixture.py | 96 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/unit/test_erasure_replicated_fixture.py diff --git a/tests/integration/test_erasure_replicated_ch.py b/tests/integration/test_erasure_replicated_ch.py index a1fa3f60c9..611f07969d 100644 --- a/tests/integration/test_erasure_replicated_ch.py +++ b/tests/integration/test_erasure_replicated_ch.py @@ -74,6 +74,11 @@ def db(raw: Any, monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: parameters={"db": name}, ).result_rows assert engines == [(True,)] + # Fenced memory inserts need leases isolated from the source schema's keys. + raw.command( + f"CREATE TABLE {name}._migration_lock AS {source}._migration_lock " + f"ENGINE = KeeperMap('/{name}/_migration_lock') PRIMARY KEY name" + ) monkeypatch.setattr(settings, "ch_database", name) yield name finally: diff --git a/tests/unit/test_erasure_replicated_fixture.py b/tests/unit/test_erasure_replicated_fixture.py new file mode 100644 index 0000000000..3147871591 --- /dev/null +++ b/tests/unit/test_erasure_replicated_fixture.py @@ -0,0 +1,96 @@ +"""The replicated erasure fixture supplies an isolated write-lease store.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from server.core import leases +from server.settings import settings +from tests.integration import test_erasure_replicated_ch as replicated +from tests.unit.claims_fake import FakeLockTable + + +class Schema: + def __init__(self): + self.tables = {} + self.commands = [] + + def command(self, sql): + self.commands.append(sql) + if sql.startswith("CREATE TABLE "): + self.tables[sql.split()[2]] = sql + + def query(self, sql, parameters=None): + if "engine_full" in sql: + assert parameters["db"] == "source_schema" + return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)]) + assert "SELECT DISTINCT engine" in sql + flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()} + return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)]) + + +class FixtureLocks(FakeLockTable): + def __init__(self, schema): + super().__init__() + self.schema = schema + + async def query_one(self, sql, params=None): + if "loom._migration_lock" in sql: + if f"{settings.ch_database}._migration_lock" not in self.schema.tables: + raise RuntimeError("Unknown table _migration_lock") + return await super().query_one(sql, params) + + +@pytest.fixture +def isolated_schema(monkeypatch): + monkeypatch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, monkeypatch) + name = next(stream) + try: + yield schema, name + finally: + stream.close() + assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC" + + +async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema): + schema, name = isolated_schema + keeper = FixtureLocks(schema) + lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper) + assert lease is not None and lease.held + assert settings.ch_database == name + await lease.release() + assert keeper.keys == {} + + +async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema): + schema, name = isolated_schema + schema.tables.pop(f"{name}._migration_lock", None) + with pytest.raises(leases.LeaseUnavailable): + await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema)) + + +def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch): + paths = [] + for _ in range(2): + with monkeypatch.context() as patch: + patch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, patch) + name = next(stream) + try: + ddl = schema.tables[f"{name}._migration_lock"] + assert "AS source_schema._migration_lock" in ddl + assert f"KeeperMap('/{name}/_migration_lock')" in ddl + assert "PRIMARY KEY name" in ddl + assert all( + "ENGINE = Replicated" in schema.tables[f"{name}.{table}"] + for table in replicated._TABLES + ) + paths.append(name) + finally: + stream.close() + assert paths[0] != paths[1] From abf0ebc86bfa602b913edc7fbe66e0dd229182b1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:13:13 +0000 Subject: [PATCH 10/11] Thrift references survive root files and nested field syntax Signed-off-by: zlareb1 --- server/code/domain/extract/lang_thrift.py | 61 ++++++++--------- tests/unit/test_code_lang_thrift.py | 83 +++++++++++++++++++++++ 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/server/code/domain/extract/lang_thrift.py b/server/code/domain/extract/lang_thrift.py index 81f6e718c1..1ac02f7391 100644 --- a/server/code/domain/extract/lang_thrift.py +++ b/server/code/domain/extract/lang_thrift.py @@ -69,7 +69,7 @@ # to the end of the file, as Thrift's lexer reads them. _TOKEN_RE = re.compile( r"//[^\n]*|#[^\n]*|/\*.*?(?:\*/|\Z)|\"(?:[^\"\\\n]|\\.)*+\"?|'(?:[^'\\\n]|\\.)*+'?" - r"|[A-Za-z_][A-Za-z0-9_.]*|-?\d[\w.+-]*|[{}()<>,;:=\[\]]|\S", + r"|[A-Za-z_][A-Za-z0-9_.]*|[+-]?\d[\w.+-]*|[{}()<>,;:=\[\]]|\S", re.S, ) @@ -132,7 +132,9 @@ def _join(toks: list[_Tok]) -> str: class _Parser: def __init__(self, path: str, toks: list[_Tok]) -> None: self.path = path - self.mod = module_scope(path) + scope = module_scope(path) + # A root file needs a dotted scope to pass the script-name gate. + self.mod = scope if "." in scope else f"{scope}.thrift" self.toks = toks self.i = 0 self.units = ThriftUnits() @@ -198,6 +200,8 @@ def _group_end(self, j: int, stop: int) -> int: if close is None: return j + 1 opener = self.toks[j].text + if opener == "{": + return min(self._closes.get(j, stop) + 1, stop) depth = 0 while j < stop: t = self.toks[j].text @@ -210,12 +214,12 @@ def _group_end(self, j: int, stop: int) -> int: j += 1 return stop - def _field(self, j: int, stop: int) -> int: + def _field(self, j: int, stop: int) -> tuple[int, tuple[int, int] | None]: """One field, `[1:] [required|optional] Type [&] name [= value] [xsd options] [(annotations)]`, as in the IDL grammar, where the numeric ID is optional: record - its type and return the index after it.""" + its type and return the index after it plus an optional attributes range.""" t = self.toks[j].text - if t.lstrip("-").isdigit() and j + 1 < stop and self.toks[j + 1].text == ":": + if t.lstrip("+-").isdigit() and j + 1 < stop and self.toks[j + 1].text == ":": j += 2 if j < stop and self.toks[j].text in _FIELD_REQ: j += 1 @@ -229,22 +233,31 @@ def _field(self, j: int, stop: int) -> int: j = self._group_end(j + 1, stop) while j < stop and self.toks[j].text in _XSD_FIELD_FLAGS: j += 1 + attrs = None if j + 1 < stop and self.toks[j].text == "xsd_attrs" and self.toks[j + 1].text == "{": attrs_end = self._group_end(j + 1, stop) - self._fields(j + 2, attrs_end - 1) + attrs = (j + 2, attrs_end - 1) j = attrs_end if j < stop and self.toks[j].text == "(": j = self._group_end(j, stop) - return j + return j, attrs def _fields(self, j: int, stop: int) -> None: """Field lists between `j` and `stop`, with or without numeric IDs and separators: record each field's type.""" - while j < stop: - if self.toks[j].text in ",;": - j += 1 - continue - j = max(self._field(j, stop), j + 1) + pending = [(j, stop)] + while pending: + j, stop = pending.pop() + while j < stop: + if self.toks[j].text in ",;": + j += 1 + continue + after, attrs = self._field(j, stop) + if attrs is not None: + pending.append((after, stop)) + j, stop = attrs + else: + j = max(after, j + 1) def _block_end(self, open_idx: int) -> int: j = self._closes.get(open_idx) @@ -270,16 +283,7 @@ def _service(self, start: int, name: str, open_idx: int) -> None: ): fname = self.toks[ret_end].text args_open = ret_end + 1 - k = args_open - depth = 0 - while k < close: - if self.toks[k].text == "(": - depth += 1 - elif self.toks[k].text == ")": - depth -= 1 - if depth == 0: - break - k += 1 + k = self._group_end(args_open, close) - 1 self._fields(args_open + 1, k) end = k if ( @@ -288,18 +292,11 @@ def _service(self, start: int, name: str, open_idx: int) -> None: and k + 2 < close and self.toks[k + 2].text == "(" ): - m = k + 2 - depth = 0 - while m < close: - if self.toks[m].text == "(": - depth += 1 - elif self.toks[m].text == ")": - depth -= 1 - if depth == 0: - break - m += 1 + m = self._group_end(k + 2, close) - 1 self._fields(k + 3, m) end = m + if end + 1 < close and self.toks[end + 1].text == "(": + end = self._group_end(end + 1, close) - 1 sig = _join(self.toks[fstart : end + 1]) self._add( fname, KIND_FUNCTION, sig, self.toks[fstart].line, self.toks[end].line, name diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py index 3f075a0f8a..eac6c750d6 100644 --- a/tests/unit/test_code_lang_thrift.py +++ b/tests/unit/test_code_lang_thrift.py @@ -3,6 +3,9 @@ from __future__ import annotations import time +from unittest.mock import AsyncMock, Mock + +import pytest from server.code.app.index.ingest_parse import LANE_UNIT_LANGS, _lane_module from server.code.domain.extract import lang_thrift as th @@ -206,3 +209,83 @@ def cost(n: int) -> float: return time.perf_counter() - t assert_linear(cost, 2_000, 8_000) + + +async def test_root_file_types_bind_through_the_production_name_gate() -> None: + from server.code.app.index.ingest_edges import _lane_site_edges + + shared = th.parse_thrift("shared.thrift", b"struct Payload {}\nservice Base {}\n") + caller = th.parse_thrift( + "main.thrift", + b'include "shared.thrift"\nservice Child extends shared.Base {\n' + b" shared.Payload get()\n}\n", + ) + table = { + symbol.name: [(symbol.qualified_name, symbol.qualified_name, symbol.kind)] + for symbol in shared.symbols + } + table["Base"].append(("native-base", "Native::Base", "interface")) + table["Payload"].append(("native-payload", "Native::Payload", "struct")) + repo = Mock(name_candidates=AsyncMock(return_value=table)) + edges = await _lane_site_edges(repo, "o", "n", caller) + assert { + (edge.from_qualified_name, edge.to_qualified_name, edge.edge_kind, edge.confidence) + for edge in edges + } == { + ("main.thrift::Child", "shared.thrift::Base", "inherits", 0.6), + ("main.thrift::Child.get", "shared.thrift::Payload", "uses_type", 0.6), + } + repo.name_candidates.assert_awaited_once_with("o", "n", ["Base", "Payload"]) + + +def test_function_annotations_do_not_become_type_sites() -> None: + src = b"""struct Audit { 1: string value } +service S { + void ping() + (Audit = "metadata") + Audit fetch() +} +""" + assert th.thrift_units("idl/main.thrift", src.decode()).sites == [(5, "Audit", "field_type")] + assert _uses_type("idl/main.thrift", src) == {("S.fetch", "Audit")} + + +@pytest.mark.parametrize("default", ["+1", "-1", "+1.5e+2", "-1.5e-2"]) +@pytest.mark.parametrize("field_id", ["", "+1: "]) +def test_signed_defaults_preserve_the_following_id_less_field(default: str, field_id: str) -> None: + src = ( + "struct Item { 1: string value }\n" + f"struct Box {{\n {field_id}double scale = {default}\n Item item\n}}\n" + ).encode() + assert _uses_type("idl/main.thrift", src) == {("Box", "Item")} + + +def _nested_attrs(n: int) -> str: + return ( + "struct Leaf {}\nstruct A {\n" + + "1: A a xsd_attrs {\n" * n + + "1: Leaf leaf\n" + + "}\n" * n + + "2: Leaf tail\n}\n" + ) + + +def test_nested_xsd_attrs_preserve_source_order_without_recursion() -> None: + depth = 1_500 + units = th.thrift_units("idl/main.thrift", _nested_attrs(depth)) + assert [symbol.name for symbol in units.symbols] == ["Leaf", "A"] + assert units.sites == [ + *((line, "A", "field_type") for line in range(3, depth + 3)), + (depth + 3, "Leaf", "field_type"), + (depth * 2 + 4, "Leaf", "field_type"), + ] + + +def test_nested_xsd_attrs_cost_linear_time() -> None: + def cost(n: int) -> float: + text = _nested_attrs(n) + started = time.perf_counter() + th.thrift_units("idl/main.thrift", text) + return time.perf_counter() - started + + assert_linear(cost, 500, 2_000) From 74f72ee90c757b840f33782d64069fefbec2ba8a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:30:18 +0000 Subject: [PATCH 11/11] Thrift type metadata stays out of dependency sites Signed-off-by: zlareb1 --- server/code/domain/extract/lang_thrift.py | 62 +++++++++-------- tests/unit/test_code_lang_thrift.py | 81 ++++++++++++++++++++++- 2 files changed, 116 insertions(+), 27 deletions(-) diff --git a/server/code/domain/extract/lang_thrift.py b/server/code/domain/extract/lang_thrift.py index 1ac02f7391..5c7cab5349 100644 --- a/server/code/domain/extract/lang_thrift.py +++ b/server/code/domain/extract/lang_thrift.py @@ -69,7 +69,7 @@ # to the end of the file, as Thrift's lexer reads them. _TOKEN_RE = re.compile( r"//[^\n]*|#[^\n]*|/\*.*?(?:\*/|\Z)|\"(?:[^\"\\\n]|\\.)*+\"?|'(?:[^'\\\n]|\\.)*+'?" - r"|[A-Za-z_][A-Za-z0-9_.]*|[+-]?\d[\w.+-]*|[{}()<>,;:=\[\]]|\S", + r"|[A-Za-z_][A-Za-z0-9_.]*|[+-]?(?:\d|\.\d)[\w.+-]*|[{}()<>,;:=\[\]]|\S", re.S, ) @@ -147,10 +147,6 @@ def __init__(self, path: str, toks: list[_Tok]) -> None: elif t.text == "}" and opens: self._closes[opens.pop()] = j - def _peek(self, k: int = 0) -> _Tok | None: - j = self.i + k - return self.toks[j] if j < len(self.toks) else None - def _add(self, name: str, kind: str, sig: str, start: int, end: int, scope: str = "") -> None: self.units.symbols.append( Symbol( @@ -169,30 +165,45 @@ def _site(self, line: int, type_name: str, kind: str = SITE_FIELD_TYPE) -> None: if bare: self.units.sites.append((line, bare, kind)) + def _type_metadata_end(self, j: int, stop: int) -> int: + if j < stop and self.toks[j].text == "(": + return self._group_end(j, stop) + if ( + j + 1 < stop + and self.toks[j].text == "cpp_type" + and self.toks[j + 1].text.startswith(('"', "'")) + ): + return j + 2 + return j + def _type(self, j: int, stop: int) -> int: - """Read a type at `j` (a name, or a container with `<…>`), recording its named - types; returns the index after it.""" + """Read a type and its metadata, recording only named dependencies.""" if j >= stop: return j t = self.toks[j] - if t.text in _CONTAINERS and j + 1 < stop and self.toks[j + 1].text == "<": - depth = 0 - k = j + 1 - while k < stop: - tk = self.toks[k].text - if tk == "<": - depth += 1 - elif tk == ">": - depth -= 1 - if depth == 0: - return k + 1 - elif _is_name(tk): - self._site(self.toks[k].line, tk) - k += 1 - return k if _is_name(t.text): self._site(t.line, t.text) - return j + 1 + j += 1 + if t.text in _CONTAINERS: + j = self._type_metadata_end(j, stop) + if j < stop and self.toks[j].text == "<": + depth = 1 + j += 1 + while j < stop and depth: + after = self._type_metadata_end(j, stop) + if after != j: + j = after + continue + tk = self.toks[j].text + if tk == "<": + depth += 1 + elif tk == ">": + depth -= 1 + elif _is_name(tk): + self._site(self.toks[j].line, tk) + j += 1 + j = self._type_metadata_end(j, stop) + return self._type_metadata_end(j, stop) def _group_end(self, j: int, stop: int) -> int: """At an opening bracket, the index after its match (or `stop`); else `j + 1`.""" @@ -224,9 +235,8 @@ def _field(self, j: int, stop: int) -> tuple[int, tuple[int, int] | None]: if j < stop and self.toks[j].text in _FIELD_REQ: j += 1 j = self._type(j, stop) - for skip in ("(", "&"): # type annotations, then a C++ reference marker - if j < stop and self.toks[j].text == skip: - j = self._group_end(j, stop) + if j < stop and self.toks[j].text == "&": + j += 1 if j < stop and _is_name(self.toks[j].text): j += 1 if j < stop and self.toks[j].text == "=" and j + 1 < stop: diff --git a/tests/unit/test_code_lang_thrift.py b/tests/unit/test_code_lang_thrift.py index eac6c750d6..46ec7c6d41 100644 --- a/tests/unit/test_code_lang_thrift.py +++ b/tests/unit/test_code_lang_thrift.py @@ -250,7 +250,9 @@ def test_function_annotations_do_not_become_type_sites() -> None: assert _uses_type("idl/main.thrift", src) == {("S.fetch", "Audit")} -@pytest.mark.parametrize("default", ["+1", "-1", "+1.5e+2", "-1.5e-2"]) +@pytest.mark.parametrize( + "default", ["+1", "-1", "+1.5e+2", "-1.5e-2", ".5", "+.5", "-.5", "+.5e-2"] +) @pytest.mark.parametrize("field_id", ["", "+1: "]) def test_signed_defaults_preserve_the_following_id_less_field(default: str, field_id: str) -> None: src = ( @@ -289,3 +291,80 @@ def cost(n: int) -> float: return time.perf_counter() - started assert_linear(cost, 500, 2_000) + + +def test_nested_type_annotations_are_not_type_sites() -> None: + src = b"""struct Audit {} +struct Item {} +struct Box { + 1: list names + 2: map (Audit = "metadata")> values +} +""" + assert th.thrift_units("idl/main.thrift", src.decode()).sites == [(5, "Item", "field_type")] + assert _uses_type("idl/main.thrift", src) == {("Box", "Item")} + + +def test_an_annotated_alias_keeps_its_symbol_and_type_edge() -> None: + src = b"""struct Audit {} +struct Meta {} +typedef list (Meta = "metadata") AuditList +""" + units = th.thrift_units("idl/main.thrift", src.decode()) + alias = next(symbol for symbol in units.symbols if symbol.name == "AuditList") + assert (alias.kind, alias.start_line, alias.end_line) == ("type", 3, 3) + assert units.sites == [(3, "Audit", "field_type")] + assert _uses_type("idl/main.thrift", src) == {("AuditList", "Audit")} + + +def test_an_annotated_return_keeps_its_signature_span_and_source_edge() -> None: + src = b"""struct Audit {} +struct Meta {} +service S { + list + (Meta = "metadata") + fetch() + string (Meta = "metadata") ping() + Audit following() +} +""" + units = th.thrift_units("idl/main.thrift", src.decode()) + by_name = {symbol.name: symbol for symbol in units.symbols} + fetch = by_name["fetch"] + assert (fetch.start_line, fetch.end_line) == (4, 6) + assert fetch.signature == 'list(Meta = "metadata") fetch()' + assert by_name["ping"].signature == 'string(Meta = "metadata") ping()' + assert units.sites == [(4, "Audit", "field_type"), (8, "Audit", "field_type")] + assert _uses_type("idl/main.thrift", src) == {("S.fetch", "Audit"), ("S.following", "Audit")} + + +def test_native_type_hints_do_not_hide_nested_types_or_create_sites() -> None: + src = b"""struct Audit {} +struct Item {} +typedef list cpp_type "vector" Items +struct Box { + 1: map cpp_type "Map" > entries +} +service S { + list cpp_type "vector" fetch() +} +""" + units = th.thrift_units("idl/main.thrift", src.decode()) + assert [symbol.name for symbol in units.symbols] == [ + "Audit", + "Item", + "Items", + "Box", + "S", + "fetch", + ] + assert units.sites == [ + (3, "Item", "field_type"), + (5, "Item", "field_type"), + (8, "Item", "field_type"), + ] + assert _uses_type("idl/main.thrift", src) == { + ("Items", "Item"), + ("Box", "Item"), + ("S.fetch", "Item"), + }