From 6cd5762f0c5a832195cd0d5d794d235ecca4d8fe Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Wed, 16 Sep 2026 15:20:08 -0500 Subject: [PATCH 1/3] feat(serve): load a taxonomy package with no report in it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `load_filing` refused a taxonomy published on its own — a zip or directory of schemas and linkbases with no report — with "No inline document or XBRL instance found", even though every tool works on its DTS once Arelle is pointed at the right schema. - A package with no report now loads from an entry point: the first one `META-INF/taxonomyPackage.xml` lists, resolved into the package through its catalog (FASB's US GAAP and SRT packages open with `entire/…-entryPoint-all`), or, with no manifest (GASB's exposure drafts), the one schema nothing else in the package imports. Several root schemas and no manifest asks for one. - `load_filing` takes `entry_point` to load another, by its path in the package, file name or name. - The receipt and `describe_filing` carry a `taxonomy` block naming the entry point loaded and the others on offer, and the next steps point at resolve_element, disclosures, statement and information_block. - An elements-only entry point has no networks for the tools to read; it is refused as such rather than as "not XBRL". - A `.zip` URL is downloaded and loaded as a local package, so a published taxonomy loads by its URL (`https://xbrl.fasb.org/us-gaap/2026/us-gaap-2026.zip`). --- tests/test_serve.py | 212 +++++++++++++++++++++++++++++ xbrlkit/serve/README.md | 12 ++ xbrlkit/serve/server.py | 28 +++- xbrlkit/serve/session.py | 285 +++++++++++++++++++++++++++++++++++---- xbrlkit/serve/tools.py | 38 +++++- 5 files changed, 544 insertions(+), 31 deletions(-) diff --git a/tests/test_serve.py b/tests/test_serve.py index aa544ab..95593a6 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -13,6 +13,7 @@ import json import os +import shutil import textwrap from datetime import date from pathlib import Path @@ -827,6 +828,217 @@ def test_find_load_target_prefers_inline_document(tmp_path: Path) -> None: assert _find_load_target(tmp_path).name == "acme-20241231.xml" +# -- a taxonomy published on its own ------------------------------------------------ + + +XS = "xmlns:xs='http://www.w3.org/2001/XMLSchema'" +GAAP_URL = "https://taxonomies.example/gaap/2025/" + + +def _bare_taxonomy(root: Path) -> Path: + """A taxonomy shipped the way GASB's exposure draft is: no manifest, one + schema importing its roles and types, its linkbases beside it pointing back + at it.""" + root.mkdir(parents=True, exist_ok=True) + (root / "gov-2026.xsd").write_text( + f"" + "" + "" + "" + "" + ) + (root / "gov-roles.xsd").write_text(f"") + (root / "gov-types.xsd").write_text(f"") + (root / "gov-2026-pre.xml").write_text( + "" + "" + "" + ) + return root + + +def _published_package(root: Path) -> Path: + """A taxonomy package shipped the way FASB ships US GAAP: a manifest listing + its entry points by their published URLs, a catalog mapping those URLs into + the package, and no report.""" + pkg = root / "gaap-2025" + (pkg / "META-INF").mkdir(parents=True) + + def entry(name: str, href: str) -> str: + return ( + f"{name}" + f"" + ) + + (pkg / "META-INF" / "taxonomyPackage.xml").write_text( + "" + "" + + entry("Everything", f"{GAAP_URL}entire/gaap-entryPoint-all-2025.xsd") + + entry("Published elsewhere", "https://elsewhere.example/other.xsd") + + entry("Elements only", f"{GAAP_URL}elts/gaap-2025.xsd") + + entry("Meta model", "../meta/gaap-meta-2025.xsd") + + "" + ) + (pkg / "META-INF" / "catalog.xml").write_text( + "" + f"" + ) + for rel in ( + "entire/gaap-entryPoint-all-2025.xsd", + "elts/gaap-2025.xsd", + "meta/gaap-meta-2025.xsd", + ): + (pkg / rel).parent.mkdir(parents=True, exist_ok=True) + (pkg / rel).write_text(f"") + return root + + +def _zip(tree: Path, archive: Path) -> Path: + import zipfile + + with zipfile.ZipFile(archive, "w") as zf: + for path in sorted(tree.rglob("*")): + if path.is_file(): + zf.write(path, path.relative_to(tree)) + return archive + + +def test_a_taxonomy_holds_no_report(tmp_path: Path) -> None: + assert _find_load_target(_bare_taxonomy(tmp_path)) is None + assert _find_load_target(_published_package(tmp_path / "pkg")) is None + + +def test_a_bare_taxonomy_loads_from_the_schema_nothing_imports(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + # The roles and types are imported, so they are not where the DTS starts; + # the linkbase pointing back at the main schema does not make it an import. + chosen = _choose_entry_point(_bare_taxonomy(tmp_path), None) + assert chosen.entry_point.document == "gov-2026.xsd" + assert chosen.others == [] + + +def test_a_bare_taxonomy_with_several_roots_asks_for_one(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + root = _bare_taxonomy(tmp_path) + (root / "gov-2026-alt.xsd").write_text(f"") + with pytest.raises(SourceError, match="entry_point"): + _choose_entry_point(root, None) + chosen = _choose_entry_point(root, "gov-2026-alt") + assert chosen.entry_point.document == "gov-2026-alt.xsd" + assert [e.document for e in chosen.others] == ["gov-2026.xsd"] + + +def test_a_manifest_loads_the_first_entry_point_it_lists(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + chosen = _choose_entry_point(_published_package(tmp_path), None) + assert chosen.entry_point.name == "Everything" + assert chosen.entry_point.document == "gaap-2025/entire/gaap-entryPoint-all-2025.xsd" + # The one published outside the package is not offered; a relative href + # resolves against the manifest. + assert [e.document for e in chosen.others] == [ + "gaap-2025/elts/gaap-2025.xsd", + "gaap-2025/meta/gaap-meta-2025.xsd", + ] + + +def test_an_entry_point_is_named_by_path_file_or_name(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + root = _published_package(tmp_path) + + def chosen(wanted: str) -> str: + return _choose_entry_point(root, wanted).entry_point.document + + elements = "gaap-2025/elts/gaap-2025.xsd" + assert chosen("gaap-2025/elts/gaap-2025.xsd") == elements + assert chosen("gaap-2025.xsd") == elements + assert chosen("Elements only") == elements + assert chosen("gaap-entryPoint-all-2025").endswith("entryPoint-all-2025.xsd") + assert chosen("meta") == "gaap-2025/meta/gaap-meta-2025.xsd" + with pytest.raises(SourceError, match="matches 3"): + chosen("gaap") + with pytest.raises(SourceError, match="No entry point matches 'nope'"): + chosen("nope") + + +def test_a_taxonomy_zip_loads_with_its_entry_point_stated( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from xbrlkit.serve.session import NoXbrlFound + + parsed: list[Path] = [] + + def fake_parse(self, target, accession, filing, entity, packages=None): + parsed.append(Path(target)) + if Path(target).name == "gaap-2025.xsd": + raise NoXbrlFound(f"{target} holds no XBRL facts or concepts") + return _model() + + monkeypatch.setattr(FilingSession, "_parse", fake_parse) + archive = _zip(_published_package(tmp_path / "tree"), tmp_path / "gaap-2025.zip") + session = FilingSession() + try: + lf = session.load(str(archive)) + assert ( + parsed[-1].as_posix().endswith("gaap-2025/entire/gaap-entryPoint-all-2025.xsd") + ) + receipt = tools.load_receipt(lf) + taxonomy = receipt["taxonomy"] + assert taxonomy["entry_point"]["name"] == "Everything" + assert [e["document"] for e in taxonomy["other_entry_points"]] == [ + "gaap-2025/elts/gaap-2025.xsd", + "gaap-2025/meta/gaap-meta-2025.xsd", + ] + assert tools.describe_filing(lf)["next"][0].startswith("resolve_element") + + other = session.load(str(archive), entry_point="meta") + assert other.taxonomy is not None + assert other.taxonomy.entry_point.document == "gaap-2025/meta/gaap-meta-2025.xsd" + + # An elements-only schema has no networks to read; the error says so and + # names the entry points that do, rather than that this is not XBRL. + with pytest.raises(SourceError, match="declares no networks"): + session.load(str(archive), entry_point="Elements only") + finally: + session.close() + + +def test_a_taxonomy_zip_loads_by_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(FilingSession, "_parse", lambda self, *a, **kw: _model()) + archive = _zip(_bare_taxonomy(tmp_path / "tree"), tmp_path / "gov-2026.zip") + session = FilingSession() + + def fake_fetch(url: str, into: Path | None = None) -> Path: + assert into is not None + return Path(shutil.copy(archive, into / Path(url).name)) + + monkeypatch.setattr(session, "_fetch", fake_fetch) + try: + lf = session.load("https://taxonomies.example/gov/gov-2026.zip") + assert lf.taxonomy is not None + assert lf.taxonomy.entry_point.document == "gov-2026.xsd" + finally: + session.close() + + +def test_entry_point_is_refused_where_there_is_no_package(tmp_path: Path) -> None: + schema = _bare_taxonomy(tmp_path) / "gov-2026.xsd" + session = FilingSession() + try: + for source in (str(schema), "ACME", "https://example.test/report.htm"): + with pytest.raises(SourceError, match="taxonomy package"): + session.load(source, entry_point="gov-2026") + finally: + session.close() + + # -- the pure profile and the document toggle ------------------------------------- diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md index e7d0231..ae8ae88 100644 --- a/xbrlkit/serve/README.md +++ b/xbrlkit/serve/README.md @@ -25,6 +25,7 @@ chat: *"load NVIDIA's latest 10-K"*, *"load `1045810:0001045810-26-000021`"*, | an EDGAR `cik:accession` | `1045810:0001045810-26-000021` | | a **LEI**, or a filings.xbrl.org filing id | `lei:213800H2PQMIF3OVZY47` | | a local filing | an inline `.htm`, an instance `.xml`, a directory, a `.zip` package | +| a **taxonomy** published on its own, by path or URL | `us-gaap-2025.zip`, `https://xbrl.fasb.org/us-gaap/2026/us-gaap-2026.zip`, an unpacked taxonomy directory | | a **JSON report**, by path or URL | `.clawdog.jsonld`, `.tavi.json`, `.holon.jsonld`, or a `model.json` from `export_filing` | | a URL Arelle can load | any of the above on the web | @@ -34,6 +35,17 @@ The two indexes behind the first three are [`edgar/`](../edgar/README.md) and fetch. Several filings load at once, each under an id; `unload_filing` drops one. +A zip or directory with no report in it is a taxonomy, and loads from an +entry point: the first one its `META-INF/taxonomyPackage.xml` lists +(FASB's US GAAP package lists `entire/us-gaap-entryPoint-all` first), or — +with no manifest, as GASB's exposure drafts ship — the one schema nothing else +in the package imports. The receipt's `taxonomy` names the entry point loaded +and the others on offer; `entry_point` on `load_filing` picks another, by its +path, file name or name. A taxonomy answers with its concepts and networks — +`resolve_element`, `disclosures`, `statement`, `information_block` — and no +facts. An elements-only entry point (a schema with no linkbases) has no +networks to read, and is refused as such. + ## The tools The tools are the shapes a reader needs, not a query language. diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index c8e3127..9224469 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -59,7 +59,8 @@ START - list_filings says what is loaded. Nothing? load_filing takes a local path \ -(an inline .htm, an instance .xml, a filing directory or zip), a URL, an \ +(an inline .htm, an instance .xml, a filing directory or zip, or a taxonomy \ +package with no report in it), a URL, an \ EDGAR `cik:accession`, or a ticker (`NVDA`, `NVDA 10-Q`). Outside the SEC it \ takes `lei:` for a filer's latest filing on filings.xbrl.org, or that \ index's own filing id — ESEF and the national regimes, identified by LEI \ @@ -311,9 +312,14 @@ async def search_filings( "the SEC — `lei:` for that filer's latest filing on " "filings.xbrl.org, or one of that index's filing ids (e.g. " "`213800H2PQMIF3OVZY47-2022-03-31-ESEF-GB-0`). Any XBRL " - "taxonomy loads: US GAAP, IFRS, ESEF, ACFR. Takes seconds to a minute; " - "the taxonomy cache makes repeat loads fast, and a JSON report loads at " - "once, with no Arelle and no taxonomy fetch." + "taxonomy loads: US GAAP, IFRS, ESEF, ACFR. A taxonomy published on its " + "own loads too — a .zip or directory with schemas and linkbases and no " + "report, local or by URL (`https://xbrl.fasb.org/us-gaap/2026/" + "us-gaap-2026.zip`): its concepts and networks, no facts. The receipt's " + "`taxonomy` names the entry point loaded — the first the package's " + "manifest lists, or its one root schema — and the others it offers. " + "Takes seconds to a minute; the taxonomy cache makes repeat loads fast, " + "and a JSON report loads at once, with no Arelle and no taxonomy fetch." ), structured_output=False, ) @@ -327,9 +333,21 @@ async def load_filing( ) ), ] = None, + entry_point: Annotated[ + str | None, + Field( + description=( + "For a taxonomy package: the entry point to load, by its document " + "path, file name or name from the receipt's `taxonomy` " + "(e.g. `us-gaap-entryPoint-all-2025`). Omit for the default." + ) + ), + ] = None, ) -> str: try: - loaded = await anyio.to_thread.run_sync(session.load, source, filing_id) + loaded = await anyio.to_thread.run_sync( + session.load, source, filing_id, entry_point + ) except (SourceError, FileNotFoundError, ValueError) as exc: return _error(str(exc)) return run(receipt, loaded) diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index ee761fd..0f43816 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -99,6 +99,28 @@ class TextSection: heading_chars: int = 0 +@dataclass +class EntryPoint: + """One way into a taxonomy: the schema its DTS is discovered from. + + ``document`` is the schema's path inside the package, as a caller names it + back through ``entry_point``; ``path`` is where it sits on disk. + """ + + name: str + document: str + path: Path + + +@dataclass +class TaxonomyEntry: + """The entry point a taxonomy package with no report was loaded from, and + the ones it offers that were not.""" + + entry_point: EntryPoint + others: list[EntryPoint] = field(default_factory=list) + + @dataclass class LoadedFiling: """One filing the server holds: the model plus its readable text. @@ -134,6 +156,10 @@ class LoadedFiling: # for, and the reader who asked to see the report asked to see that one. source_document: str | None = None source_kind: str | None = None + # Set when the source was a taxonomy published on its own — a package with + # schemas and linkbases and no report — and so holds concepts and networks + # but no facts. + taxonomy: TaxonomyEntry | None = None @property def has_xbrl(self) -> bool: @@ -319,13 +345,20 @@ def get(self, filing: str | None = None) -> LoadedFiling: # -- loading --------------------------------------------------------------- - def load(self, source: str, filing_id: str | None = None) -> LoadedFiling: - """Resolve ``source`` and load it; returns the new :class:`LoadedFiling`.""" + def load( + self, source: str, filing_id: str | None = None, entry_point: str | None = None + ) -> LoadedFiling: + """Resolve ``source`` and load it; returns the new :class:`LoadedFiling`. + + ``entry_point`` names the schema to load from a taxonomy package — a zip + or directory, local or by URL — in place of the one chosen by default. + """ source = source.strip() if not source: raise SourceError("An empty source.") + entry_point = (entry_point or "").strip() or None with self._lock: - loaded = self._load(source) + loaded = self._load(source, entry_point) wanted = filing_id or loaded.id loaded.id = self._unique_id(wanted) self._filings[loaded.id] = loaded @@ -350,12 +383,14 @@ def _unique_id(self, wanted: str) -> str: n += 1 return f"{wanted}-{n}" - def _load(self, source: str) -> LoadedFiling: + def _load(self, source: str, entry_point: str | None = None) -> LoadedFiling: path = Path(source).expanduser() if path.exists(): - return self._load_local(path, source) + return self._load_local(path, source, entry_point) if source.startswith(("http://", "https://")): - return self._load_url(source) + return self._load_url(source, entry_point) + if entry_point: + raise SourceError(_ENTRY_POINT_NEEDS_A_PACKAGE.format(source=source)) m = _CIK_ACCESSION_RE.match(source) if m: return self._load_edgar(m.group(1), m.group(2), source) @@ -385,18 +420,31 @@ def _load(self, source: str) -> LoadedFiling: "filing outside EDGAR — `lei:` or a filings.xbrl.org filing id." ) - def _load_local(self, path: Path, source: str) -> LoadedFiling: + def _load_local( + self, path: Path, source: str, entry_point: str | None = None + ) -> LoadedFiling: package_dir: Path | None = None + taxonomy: TaxonomyEntry | None = None + is_package = path.is_dir() or path.suffix.lower() == ".zip" + if entry_point and not is_package: + raise SourceError(_ENTRY_POINT_NEEDS_A_PACKAGE.format(source=source)) if path.is_file() and path.suffix.lower() in _JSON_SUFFIXES: return self._load_json(path, source) - if path.is_dir(): - package_dir = path - target = _find_load_target(path) - elif path.suffix.lower() == ".zip": - package_dir = Path(tempfile.mkdtemp(prefix="zip-", dir=self._tmp)) - with ZipFile(path) as archive: - archive.extractall(package_dir) - target = _find_load_target(package_dir) + if is_package: + if path.is_dir(): + package_dir = path + else: + package_dir = Path(tempfile.mkdtemp(prefix="zip-", dir=self._tmp)) + with ZipFile(path) as archive: + archive.extractall(package_dir) + report = None if entry_point else _find_load_target(package_dir) + if report is None: + # No report in the package: a taxonomy published on its own, which + # loads from one of its schemas and answers with concepts and networks. + taxonomy = _choose_entry_point(package_dir, entry_point) + target = taxonomy.entry_point.path + else: + target = report else: target = path package_dir = path.parent @@ -417,11 +465,23 @@ def _load_local(self, path: Path, source: str) -> LoadedFiling: model = self._parse( target, accession=accession, filing=None, entity=None, packages=packages ) - except NoXbrlFound: + except NoXbrlFound as exc: + if taxonomy is not None: + # The model holds the concepts its networks and facts reach, so an + # elements-only schema — no linkbases, nothing to reach them — reads + # as empty. Say so, rather than that the package is not XBRL. + others = ", ".join(e.document for e in taxonomy.others) + raise SourceError( + f"Entry point {taxonomy.entry_point.document} declares no networks " + "for the tools to read (an elements-only schema); load an entry " + f"point with linkbases instead{': ' + others if others else ''}." + ) from exc if target.suffix.lower() not in _DOCUMENT_SUFFIXES: raise return self._document_only(target, source, accession, package_dir) - return self._finish(_local_id(path, model), source, model, target, package_dir) + loaded = self._finish(_local_id(path, model), source, model, target, package_dir) + loaded.taxonomy = taxonomy + return loaded def _load_json( self, path: Path, source: str, document: Path | None = None @@ -479,10 +539,21 @@ def _load_json( source_kind=kind if served else None, ) - def _load_url(self, url: str) -> LoadedFiling: + def _load_url(self, url: str, entry_point: str | None = None) -> LoadedFiling: clean = Path(url.split("?", 1)[0]) accession = clean.stem or url suffix = clean.suffix.lower() + if suffix == ".zip": + # A package by URL is downloaded and loaded as a local one: the report + # found inside it, or — for a taxonomy published on its own, which is + # how FASB, XBRL US and the IFRS Foundation distribute theirs — an + # entry point. + archive = self._fetch( + url, into=Path(tempfile.mkdtemp(prefix="url-", dir=self._tmp)) + ) + return self._load_local(archive, url, entry_point) + if entry_point: + raise SourceError(_ENTRY_POINT_NEEDS_A_PACKAGE.format(source=url)) if suffix in _JSON_SUFFIXES: # A JSON report is read here, not by Arelle, which cannot load one. This # is how a report published as an artifact — a holon or a TAVI on a CDN — @@ -1437,12 +1508,14 @@ def _is_inline(head: bytes) -> bool: return b"ix:nonNumeric" in head or b"ix:nonFraction" in head or b"ix:header" in head -def _find_load_target(package_dir: Path) -> Path: - """The file Arelle should load from a filing directory: the inline +def _find_load_target(package_dir: Path) -> Path | None: + """The report Arelle should load from a filing directory: the inline document (the largest ``.htm`` carrying ``ix:`` markup), else the XBRL instance — recognised by its root element, since a package need not follow EDGAR's naming (an instance called ``instance.xml`` beside - ``report.xsd`` and hyphenated linkbases is a valid package too). + ``report.xsd`` and hyphenated linkbases is a valid package too). ``None`` + when the package holds no report: a taxonomy published on its own, which + :func:`_choose_entry_point` picks a schema from. The whole tree is searched, not just the top level. A conformant XBRL taxonomy package puts nothing at its root: an ESEF report sits under @@ -1479,10 +1552,174 @@ def _find_load_target(package_dir: Path) -> Path: raise SourceError( f"{len(instances)} XBRL instances in {package_dir} ({names}); point at one." ) - raise SourceError( - f"No inline document or XBRL instance found in {package_dir}; " - "point at the file to load." + return None + + +_ENTRY_POINT_NEEDS_A_PACKAGE = ( + "entry_point picks a schema inside a taxonomy package — a .zip or a " + "directory, local or by URL; {source} is not one." +) +_SCHEMA_LOCATION_RE = re.compile(rb"""schemaLocation\s*=\s*(["'])(.*?)\1""", re.DOTALL) + + +def _local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def _choose_entry_point(package_dir: Path, wanted: str | None) -> TaxonomyEntry: + """The schema to load a taxonomy package from, when it holds no report. + + A package that declares its entry points (``META-INF/taxonomyPackage.xml``) + loads the one ``wanted`` names, else the first it lists: FASB lists the whole + taxonomy first — its US GAAP and SRT packages open with + ``entire/…-entryPoint-all``, ahead of the elements-only, DQC and meta-model + entry points. A package without a manifest loads its one root + schema, the one no other schema in it imports; with several, the caller + names one. The others are returned beside the choice so it is never silent. + """ + # Resolved once, so a package unpacked under a symlinked temp directory + # (macOS's /var) still contains the resolved paths its entry points name. + package_dir = package_dir.resolve() + declared = _declared_entry_points(package_dir) + entry_points = declared or _root_schemas(package_dir) + if not entry_points: + raise SourceError( + f"No inline document, XBRL instance or taxonomy schema found in " + f"{package_dir}; point at the file to load." + ) + if wanted: + chosen = _match_entry_point(entry_points, wanted) + elif declared or len(entry_points) == 1: + chosen = entry_points[0] + else: + raise SourceError( + f"This taxonomy package has no manifest and {len(entry_points)} root " + f"schemas; pass entry_point to choose one: " + f"{', '.join(e.document for e in entry_points)}." + ) + return TaxonomyEntry( + entry_point=chosen, others=[e for e in entry_points if e is not chosen] + ) + + +def _match_entry_point(entry_points: list[EntryPoint], wanted: str) -> EntryPoint: + """The entry point ``wanted`` names: exactly, by its name, its path in the + package or its file name with or without ``.xsd``; else the one it is part + of.""" + key = wanted.strip().lower() + + def names(e: EntryPoint) -> set[str]: + document = Path(e.document) + return { + e.name.lower(), + e.document.lower(), + document.name.lower(), + document.stem.lower(), + } + + hits = [e for e in entry_points if key in names(e)] + if not hits: + hits = [e for e in entry_points if any(key in n for n in names(e))] + if len(hits) == 1: + return hits[0] + listed = ", ".join(e.document for e in hits or entry_points) + if hits: + raise SourceError(f"entry_point {wanted!r} matches {len(hits)}: {listed}.") + raise SourceError(f"No entry point matches {wanted!r}; this package has: {listed}.") + + +def _declared_entry_points(package_dir: Path) -> list[EntryPoint]: + """The entry points the package's manifests declare, in their order, that + resolve to a schema inside the package. + + An entry point names its schema by the URL it is published at + (``https://xbrl.fasb.org/us-gaap/2025/entire/…``); the package catalog's + ``rewriteURI`` maps that URL to the copy inside the package, and a relative + ``href`` resolves against the manifest itself. + """ + found: list[EntryPoint] = [] + for manifest in sorted(package_dir.rglob("META-INF/taxonomyPackage.xml")): + meta_inf = manifest.parent + try: + root = ElementTree.parse(manifest).getroot() + except ElementTree.ParseError as exc: + logger.warning("unreadable taxonomy package manifest %s: %s", manifest, exc) + continue + rewrites = _catalog_rewrites(meta_inf / "catalog.xml") + for element in root.iter(): + if _local_name(element.tag) != "entryPoint": + continue + name = "" + href = "" + for child in element: + local = _local_name(child.tag) + if local == "name" and not name: + name = (child.text or "").strip() + elif local == "entryPointDocument" and not href: + href = (child.get("href") or "").strip() + path = _resolve_package_href(href, meta_inf, rewrites) if href else None + if path is None or not path.is_file() or not path.is_relative_to(package_dir): + continue + document = path.relative_to(package_dir).as_posix() + found.append(EntryPoint(name=name or path.stem, document=document, path=path)) + return found + + +def _catalog_rewrites(catalog: Path) -> list[tuple[str, Path]]: + """A package catalog's URL prefixes and the directories they map to, + longest prefix first.""" + if not catalog.is_file(): + return [] + try: + root = ElementTree.parse(catalog).getroot() + except ElementTree.ParseError as exc: + logger.warning("unreadable taxonomy package catalog %s: %s", catalog, exc) + return [] + rewrites = [ + (start, catalog.parent / (element.get("rewritePrefix") or "")) + for element in root.iter() + if _local_name(element.tag) == "rewriteURI" + and (start := element.get("uriStartString")) + ] + return sorted(rewrites, key=lambda r: -len(r[0])) + + +def _resolve_package_href( + href: str, base: Path, rewrites: list[tuple[str, Path]] +) -> Path | None: + """Where an ``href`` in a package manifest sits on disk, or ``None`` when + it points outside the package.""" + if "://" not in href: + return (base / href).resolve() + for start, prefix in rewrites: + if href.startswith(start): + return (prefix / href[len(start) :]).resolve() + return None + + +def _root_schemas(package_dir: Path) -> list[EntryPoint]: + """The schemas in a package that no other schema in it imports or + includes — where a DTS starts when no manifest says so. A published + taxonomy with no manifest (GASB's exposure drafts) has one: the schema that + imports its roles and types and links its linkbases.""" + schemas = sorted( + p for p in package_dir.rglob("*.xsd") if p.is_file() and not p.name.startswith(".") ) + imported: set[Path] = set() + for schema in schemas: + for match in _SCHEMA_LOCATION_RE.finditer(schema.read_bytes()): + for location in match.group(2).decode("utf-8", "replace").split(): + if "://" not in location: + imported.add((schema.parent / location.split("#", 1)[0]).resolve()) + return [ + EntryPoint( + name=schema.stem, + document=schema.relative_to(package_dir).as_posix(), + path=schema, + ) + for schema in schemas + if schema.resolve() not in imported + ] _XML_IDENTITY = { diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index 69b8b49..8523f3b 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -34,7 +34,12 @@ from xbrlkit.config import CONFIG, Config from xbrlkit.model import Arc, Concept, Network, Period, Unit, XbrlFact, XbrlModel from xbrlkit.serialize import classify_network, root_qname -from xbrlkit.serve.session import FilingSession, LoadedFiling, TextSection +from xbrlkit.serve.session import ( + FilingSession, + LoadedFiling, + TaxonomyEntry, + TextSection, +) from xbrlkit.edgar.items import describe_items, is_earnings_release, items_note from xbrlkit.information_block import ( Disclosure, @@ -474,7 +479,7 @@ def list_filings(session: FilingSession) -> dict[str, Any]: return {"filings": rows, "count": len(rows)} -LOAD_RECEIPT_KEYS = ("profile", "filing", "entity", "counts") +LOAD_RECEIPT_KEYS = ("profile", "taxonomy", "filing", "entity", "counts") def load_receipt( @@ -615,6 +620,7 @@ def describe_filing( "text": "primary document" if whole and lf.has_document else "tagged text blocks", "xbrl": lf.has_xbrl, }, + **({"taxonomy": _describe_taxonomy(lf.taxonomy)} if lf.taxonomy else {}), "filing": { "id": lf.id, "source": lf.source, @@ -690,6 +696,25 @@ def describe_filing( } +def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]: + """Which entry point a taxonomy package was loaded from, and the others it + offers — the choice stated, never silent.""" + return { + "entry_point": { + "name": taxonomy.entry_point.name, + "document": taxonomy.entry_point.document, + }, + "other_entry_points": [ + {"name": e.name, "document": e.document} for e in taxonomy.others + ], + "note": ( + "a taxonomy with no report: concepts and networks, no facts, periods or " + "units. load_filing with the same source and `entry_point` set to another " + "entry point's document loads that one instead." + ), + } + + def _next_steps(lf: LoadedFiling) -> list[str]: """What to call next, given what this filing actually is. @@ -698,6 +723,15 @@ def _next_steps(lf: LoadedFiling) -> list[str]: the item codes say the substance is elsewhere, that goes first — above the fact tools, which for an 8-K have almost nothing to work with. """ + if lf.taxonomy is not None: + return [ + "resolve_element to find concepts by phrase — each with its label, type, " + "balance, period type and the networks it sits in", + "disclosures for the taxonomy's networks as families", + "statement or information_block with a network from the list above to " + "read its tree", + "calculation for what sums to a total, where the taxonomy has calculation arcs", + ] exhibit_first: list[str] = [] if is_earnings_release(lf.model.filing.items): exhibit_first = [ From 9ecb29e52e5ae2f7079781686b22562df11390a8 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Wed, 16 Sep 2026 15:51:30 -0500 Subject: [PATCH 2/3] feat(serve): page statement and information_block past max_rows A taxonomy's own networks outrun the 400-row cap: US GAAP's classified balance sheet walks to 691 rows, and `statement` stopped in current liabilities with no way to reach Liabilities, StockholdersEquity or LiabilitiesAndStockholdersEquity. - `statement` and `information_block` take `offset`. A truncated response returns `next_offset`; passing it back returns the next page, and the pages join into the unpaged rows with their depths intact. - A later page carries `ancestors`, the headers above its first row, so a page that opens deep in the tree says what it sits under. - `information_block` returns the section's axes, calculation and text with the first page only; later pages carry rows and their columns. - An offset past the end is refused with the row count. --- tests/test_information_block.py | 26 ++++++++ tests/test_serve.py | 24 ++++++++ xbrlkit/serve/server.py | 29 ++++++++- xbrlkit/serve/tools.py | 101 +++++++++++++++++++++++++------- 4 files changed, 157 insertions(+), 23 deletions(-) diff --git a/tests/test_information_block.py b/tests/test_information_block.py index 345c5da..8b84150 100644 --- a/tests/test_information_block.py +++ b/tests/test_information_block.py @@ -676,6 +676,32 @@ def test_information_block_pivots_a_details_table_by_its_own_axis(loaded): assert "text" not in out and out["truncated"] is False +def test_information_block_pages_a_block_longer_than_max_rows(loaded): + whole = tools.information_block(loaded, "Lease Cost", whole=False) + pages = [tools.information_block(loaded, "Lease Cost", max_rows=3, whole=False)] + while pages[-1]["truncated"]: + pages.append( + tools.information_block( + loaded, + "Lease Cost", + max_rows=3, + offset=pages[-1]["next_offset"], + whole=False, + ) + ) + assert len(pages) > 1 + assert [r for p in pages for r in p["rows"]] == whole["rows"] + # The section's axes and calculation come once, with the first page. + assert pages[0]["axes"] == whole["axes"] + assert pages[0]["calculation"] == whole["calculation"] + assert all("axes" not in p and "calculation" not in p for p in pages[1:]) + assert pages[1]["ancestors"][0]["concept"] == "us-gaap:LeaseCostTable" + with pytest.raises(tools.ToolError, match="past the end of this block"): + tools.information_block( + loaded, "Lease Cost", offset=len(whole["rows"]), whole=False + ) + + def test_information_block_reports_a_total_that_does_not_foot(loaded): out = tools.information_block(loaded, "income statement", whole=False) assert out["block"]["kind"] == "income_statement" diff --git a/tests/test_serve.py b/tests/test_serve.py index 95593a6..8b3cc73 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -497,6 +497,30 @@ def test_statement_by_kind_renders_rows_in_order(loaded: LoadedFiling) -> None: ] +def test_statement_pages_a_network_longer_than_max_rows(loaded: LoadedFiling) -> None: + whole = tools.statement(loaded, "income statement") + assert whole["truncated"] is False and "next_offset" not in whole + + first = tools.statement(loaded, "income statement", max_rows=2) + assert first["truncated"] is True and first["next_offset"] == 2 + assert "offset" not in first and "ancestors" not in first + rest = tools.statement( + loaded, "income statement", max_rows=2, offset=first["next_offset"] + ) + assert rest["offset"] == 2 and rest["truncated"] is False + assert "next_offset" not in rest + # Pages join back into the whole, depths intact, and a later page says + # which headers it opens under. + assert first["rows"] + rest["rows"] == whole["rows"] + assert rest["ancestors"] == [ + {"concept": "us-gaap:IncomeStatementAbstract", "label": "Income Statement Abstract"} + ] + with pytest.raises( + tools.ToolError, match="past the end of this network \\(4 rows\\)" + ): + tools.statement(loaded, "income statement", offset=4) + + def test_statement_columns_put_the_year_before_its_fourth_quarter() -> None: model = _model() model.periods.append( diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index 9224469..8a4cd78 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -491,7 +491,9 @@ def fact_grid( "describe_filing; under the product profile a kind also works " "(balance_sheet, income_statement, cash_flow_statement, equity_statement, " "or a phrase like 'balance sheet'). `periods` limits the columns to those " - "keys, end dates, or years; otherwise the most recent eight." + "keys, end dates, or years; otherwise the most recent eight. A network " + "longer than `max_rows` — a taxonomy's statements run to hundreds of " + "rows — is `truncated`; pass its `next_offset` as `offset` for the rest." ), structured_output=False, ) @@ -508,10 +510,22 @@ def statement( max_rows: Annotated[ int, Field(description="Rows to return (max 400).", ge=1, le=400) ] = 400, + offset: Annotated[ + int, + Field( + description="Rows to skip: the `next_offset` a truncated response returned.", + ge=0, + ), + ] = 0, ) -> str: return run( lambda: tools.statement( - session.get(filing), statement, periods=periods, max_rows=max_rows, pure=pure + session.get(filing), + statement, + periods=periods, + max_rows=max_rows, + pure=pure, + offset=offset, ) ) @@ -586,7 +600,8 @@ def disclosures( "disclosures first for the family index, and narrow this one with " "`member` or `periods` when part of the block answers the question. " "`block` is an id from disclosures or describe_filing, a name or " - "part of one, or a role URI." + "part of one, or a role URI. A block longer than `max_rows` is " + "`truncated`; pass its `next_offset` as `offset` for the rest." ), structured_output=False, ) @@ -618,6 +633,13 @@ def information_block( le=200, ), ] = None, + offset: Annotated[ + int, + Field( + description="Rows to skip: the `next_offset` a truncated response returned.", + ge=0, + ), + ] = 0, ) -> str: return run( lambda: tools.information_block( @@ -627,6 +649,7 @@ def information_block( member=member, max_rows=max_rows, max_members=max_members, + offset=offset, pure=pure, whole=whole, ) diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index 8523f3b..90f5c05 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -1208,11 +1208,13 @@ def statement( periods: list[str] | None = None, max_rows: int = MAX_STATEMENT_ROWS, pure: bool = False, + offset: int = 0, ) -> dict[str, Any]: _require_xbrl(lf, "presentation networks") model, idx = lf.model, index_for(lf) network = _find_network(idx, statement, pure=pure) max_rows = max(1, min(int(max_rows or MAX_STATEMENT_ROWS), MAX_STATEMENT_ROWS)) + offset = max(0, int(offset or 0)) children: dict[str, list[Arc]] = defaultdict(list) parents: set[str] = set() @@ -1228,14 +1230,10 @@ def statement( rows: list[dict[str, Any]] = [] used_periods: dict[str, int] = defaultdict(int) truncated = False + walked = 0 + page_trail: tuple[str, ...] = () - def visit( - qname: str, depth: int, label_role: str | None, trail: tuple[str, ...] - ) -> None: - nonlocal truncated - if len(rows) >= max_rows: - truncated = True - return + def row_at(qname: str, depth: int, label_role: str | None) -> dict[str, Any]: concept = model.concepts.get(qname) row: dict[str, Any] = { "depth": depth, @@ -1263,7 +1261,20 @@ def visit( used_periods[key] += 1 if values: row["values"] = values - rows.append(row) + return row + + def visit( + qname: str, depth: int, label_role: str | None, trail: tuple[str, ...] + ) -> None: + nonlocal truncated, walked, page_trail + if len(rows) >= max_rows: + truncated = True + return + walked += 1 + if walked > offset: + if not rows: + page_trail = trail + rows.append(row_at(qname, depth, label_role)) if qname in trail or depth > 14: return for arc in children.get(qname, []): @@ -1271,6 +1282,8 @@ def visit( for root in roots: visit(root, 0, None, ()) + if offset and not rows: + raise ToolError(f"offset {offset} is past the end of this network ({walked} rows)") period_by_key = {_period_key(p): p for p in model.periods} wanted = None @@ -1312,14 +1325,38 @@ def visit( "rows": rows, "row_count": len(rows), "truncated": truncated, + **_page_fields(model, offset, len(rows), truncated, page_trail), "note": ( "consolidated values only, most precise duplicate kept; depth is the " "presentation nesting; a label like 'Total' or a negated label reflects " - "the preferred label on the arc" + "the preferred label on the arc; a truncated network continues from " + "`next_offset` passed as `offset`" ), } +def _page_fields( + model: XbrlModel, + offset: int, + returned: int, + truncated: bool, + trail: tuple[str, ...], +) -> dict[str, Any]: + """Where a page of a presentation walk sits in the whole: the offset it + starts at, the headers above its first row — a later page opens deep in + the tree, and its depths alone do not say under what — and where the next + page starts.""" + out: dict[str, Any] = {} + if offset: + out["offset"] = offset + out["ancestors"] = [ + {"concept": q, "label": _pref_label(model.concepts.get(q), q)} for q in trail + ] + if truncated: + out["next_offset"] = offset + returned + return out + + def calculation( lf: LoadedFiling, concept: str, @@ -1618,6 +1655,7 @@ def information_block( member: str | None = None, max_rows: int = MAX_BLOCK_ROWS, max_members: int | None = None, + offset: int = 0, *, pure: bool = False, whole: bool = True, @@ -1628,12 +1666,18 @@ def information_block( Member breakdowns are kept most-reported first, up to the response budget (or ``max_members`` when the caller sets one); a row is never left blank - by that cut, and every row says how many breakdowns it lost.""" + by that cut, and every row says how many breakdowns it lost. + + A section longer than ``max_rows`` — a taxonomy's own networks run to + hundreds of concepts — pages by ``offset``. The axes, calculation arcs and + text blocks describe the section, not a page, and come with the first page + alone; a later page carries its rows and their columns.""" _require_xbrl(lf, "presentation networks") model, idx = lf.model, index_for(lf) blocks, membership, families = blocks_for(lf) st = _find_block(idx, blocks, block, pure=pure) max_rows = max(1, min(int(max_rows or MAX_BLOCK_ROWS), MAX_BLOCK_ROWS)) + offset = max(0, int(offset or 0)) member_limit = ( max(1, min(int(max_members), MAX_BLOCK_MEMBERS_CAP)) if max_members is not None @@ -1707,6 +1751,8 @@ def information_block( rows: list[dict[str, Any]] = [] used_periods: dict[str, int] = defaultdict(int) truncated = False + walked = 0 + page_trail: tuple[str, ...] = () def cell(f: XbrlFact) -> Any: if f.is_nil: @@ -1719,10 +1765,21 @@ def cell(f: XbrlFact) -> Any: def visit( qname: str, depth: int, label_role: str | None, trail: tuple[str, ...] ) -> None: - nonlocal truncated + nonlocal truncated, walked, page_trail if len(rows) >= max_rows: truncated = True return + walked += 1 + if walked > offset: + if not rows: + page_trail = trail + rows.append(row_at(qname, depth, label_role)) + if qname in trail or depth > 14: + return + for arc in children.get(qname, []): + visit(arc.to_qname, depth + 1, arc.preferred_label, trail + (qname,)) + + def row_at(qname: str, depth: int, label_role: str | None) -> dict[str, Any]: concept = model.concepts.get(qname) row: dict[str, Any] = { "depth": depth, @@ -1771,14 +1828,12 @@ def visit( row["values"] = values if by_member: row["members"] = by_member - rows.append(row) - if qname in trail or depth > 14: - return - for arc in children.get(qname, []): - visit(arc.to_qname, depth + 1, arc.preferred_label, trail + (qname,)) + return row for root in roots: visit(root, 0, None, ()) + if offset and not rows: + raise ToolError(f"offset {offset} is past the end of this block ({walked} rows)") period_by_key = {_period_key(p): p for p in model.periods} wanted = {w.strip() for w in periods if w and w.strip()} if periods else None @@ -1993,15 +2048,19 @@ def newest(period_keys: set[str]) -> str: column.pop("duration", None) column.pop("calendar", None) out: dict[str, Any] = {"block": head, "columns": columns} - if axes_out: + # The axes, calculation and text describe the section; a later page of it + # would only repeat them. + first_page = not offset + if axes_out and first_page: out["axes"] = axes_out out["rows"] = rows - if calc_out: + if calc_out and first_page: out["calculation"] = calc_out - if text_out: + if text_out and first_page: out["text"] = text_out out["row_count"] = len(rows) out["truncated"] = truncated + out.update(_page_fields(model, offset, len(rows), truncated, page_trail)) if members_omitted: out["members_omitted"] = members_omitted if columns_omitted: @@ -2019,7 +2078,9 @@ def newest(period_keys: set[str]) -> str: "and columns dropped from it, and a row is never left blank by either cut; " "`calculation` lists each total's children with weights, how many of the " "shown periods foot on consolidated values, and any difference; `text` " - "entries are tagged text blocks — read one with read_text from its offset" + "entries are tagged text blocks — read one with read_text from its offset; " + "a truncated block continues from `next_offset` passed as `offset`, and " + "axes, calculation and text come with the first page only" ) return out From 9db4aaf00815de2568b185d6f9b8ec521ec054b4 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Wed, 16 Sep 2026 15:54:15 -0500 Subject: [PATCH 3/3] docs(serve): trim the taxonomy and paging tool text to one sentence each Tool descriptions ship in every client's prompt, so every xbrlkit user pays for them. The taxonomy loading detail stays in serve/README.md. --- xbrlkit/serve/server.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index 8a4cd78..2383b1f 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -59,8 +59,7 @@ START - list_filings says what is loaded. Nothing? load_filing takes a local path \ -(an inline .htm, an instance .xml, a filing directory or zip, or a taxonomy \ -package with no report in it), a URL, an \ +(an inline .htm, an instance .xml, a filing directory or zip), a URL, an \ EDGAR `cik:accession`, or a ticker (`NVDA`, `NVDA 10-Q`). Outside the SEC it \ takes `lei:` for a filer's latest filing on filings.xbrl.org, or that \ index's own filing id — ESEF and the national regimes, identified by LEI \ @@ -312,12 +311,9 @@ async def search_filings( "the SEC — `lei:` for that filer's latest filing on " "filings.xbrl.org, or one of that index's filing ids (e.g. " "`213800H2PQMIF3OVZY47-2022-03-31-ESEF-GB-0`). Any XBRL " - "taxonomy loads: US GAAP, IFRS, ESEF, ACFR. A taxonomy published on its " - "own loads too — a .zip or directory with schemas and linkbases and no " - "report, local or by URL (`https://xbrl.fasb.org/us-gaap/2026/" - "us-gaap-2026.zip`): its concepts and networks, no facts. The receipt's " - "`taxonomy` names the entry point loaded — the first the package's " - "manifest lists, or its one root schema — and the others it offers. " + "taxonomy loads: US GAAP, IFRS, ESEF, ACFR. A zip or directory with no " + "report loads as a taxonomy; the receipt names the entry point, and " + "`entry_point` picks another. " "Takes seconds to a minute; the taxonomy cache makes repeat loads fast, " "and a JSON report loads at once, with no Arelle and no taxonomy fetch." ), @@ -337,9 +333,8 @@ async def load_filing( str | None, Field( description=( - "For a taxonomy package: the entry point to load, by its document " - "path, file name or name from the receipt's `taxonomy` " - "(e.g. `us-gaap-entryPoint-all-2025`). Omit for the default." + "A taxonomy's entry point, by path, file name or name from the " + "receipt's `taxonomy`; omit for the default." ) ), ] = None, @@ -491,9 +486,8 @@ def fact_grid( "describe_filing; under the product profile a kind also works " "(balance_sheet, income_statement, cash_flow_statement, equity_statement, " "or a phrase like 'balance sheet'). `periods` limits the columns to those " - "keys, end dates, or years; otherwise the most recent eight. A network " - "longer than `max_rows` — a taxonomy's statements run to hundreds of " - "rows — is `truncated`; pass its `next_offset` as `offset` for the rest." + "keys, end dates, or years; otherwise the most recent eight. A " + "`truncated` response continues from its `next_offset` as `offset`." ), structured_output=False, ) @@ -600,8 +594,8 @@ def disclosures( "disclosures first for the family index, and narrow this one with " "`member` or `periods` when part of the block answers the question. " "`block` is an id from disclosures or describe_filing, a name or " - "part of one, or a role URI. A block longer than `max_rows` is " - "`truncated`; pass its `next_offset` as `offset` for the rest." + "part of one, or a role URI. A `truncated` response continues from its " + "`next_offset` as `offset`." ), structured_output=False, )