diff --git a/tests/test_information_block.py b/tests/test_information_block.py index 345c5da..8b84150 100644 --- a/tests/test_information_block.py +++ b/tests/test_information_block.py @@ -676,6 +676,32 @@ def test_information_block_pivots_a_details_table_by_its_own_axis(loaded): assert "text" not in out and out["truncated"] is False +def test_information_block_pages_a_block_longer_than_max_rows(loaded): + whole = tools.information_block(loaded, "Lease Cost", whole=False) + pages = [tools.information_block(loaded, "Lease Cost", max_rows=3, whole=False)] + while pages[-1]["truncated"]: + pages.append( + tools.information_block( + loaded, + "Lease Cost", + max_rows=3, + offset=pages[-1]["next_offset"], + whole=False, + ) + ) + assert len(pages) > 1 + assert [r for p in pages for r in p["rows"]] == whole["rows"] + # The section's axes and calculation come once, with the first page. + assert pages[0]["axes"] == whole["axes"] + assert pages[0]["calculation"] == whole["calculation"] + assert all("axes" not in p and "calculation" not in p for p in pages[1:]) + assert pages[1]["ancestors"][0]["concept"] == "us-gaap:LeaseCostTable" + with pytest.raises(tools.ToolError, match="past the end of this block"): + tools.information_block( + loaded, "Lease Cost", offset=len(whole["rows"]), whole=False + ) + + def test_information_block_reports_a_total_that_does_not_foot(loaded): out = tools.information_block(loaded, "income statement", whole=False) assert out["block"]["kind"] == "income_statement" diff --git a/tests/test_serve.py b/tests/test_serve.py index aa544ab..8b3cc73 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -13,6 +13,7 @@ import json import os +import shutil import textwrap from datetime import date from pathlib import Path @@ -496,6 +497,30 @@ def test_statement_by_kind_renders_rows_in_order(loaded: LoadedFiling) -> None: ] +def test_statement_pages_a_network_longer_than_max_rows(loaded: LoadedFiling) -> None: + whole = tools.statement(loaded, "income statement") + assert whole["truncated"] is False and "next_offset" not in whole + + first = tools.statement(loaded, "income statement", max_rows=2) + assert first["truncated"] is True and first["next_offset"] == 2 + assert "offset" not in first and "ancestors" not in first + rest = tools.statement( + loaded, "income statement", max_rows=2, offset=first["next_offset"] + ) + assert rest["offset"] == 2 and rest["truncated"] is False + assert "next_offset" not in rest + # Pages join back into the whole, depths intact, and a later page says + # which headers it opens under. + assert first["rows"] + rest["rows"] == whole["rows"] + assert rest["ancestors"] == [ + {"concept": "us-gaap:IncomeStatementAbstract", "label": "Income Statement Abstract"} + ] + with pytest.raises( + tools.ToolError, match="past the end of this network \\(4 rows\\)" + ): + tools.statement(loaded, "income statement", offset=4) + + def test_statement_columns_put_the_year_before_its_fourth_quarter() -> None: model = _model() model.periods.append( @@ -827,6 +852,217 @@ def test_find_load_target_prefers_inline_document(tmp_path: Path) -> None: assert _find_load_target(tmp_path).name == "acme-20241231.xml" +# -- a taxonomy published on its own ------------------------------------------------ + + +XS = "xmlns:xs='http://www.w3.org/2001/XMLSchema'" +GAAP_URL = "https://taxonomies.example/gaap/2025/" + + +def _bare_taxonomy(root: Path) -> Path: + """A taxonomy shipped the way GASB's exposure draft is: no manifest, one + schema importing its roles and types, its linkbases beside it pointing back + at it.""" + root.mkdir(parents=True, exist_ok=True) + (root / "gov-2026.xsd").write_text( + f"" + "" + "" + "" + "" + ) + (root / "gov-roles.xsd").write_text(f"") + (root / "gov-types.xsd").write_text(f"") + (root / "gov-2026-pre.xml").write_text( + "" + "" + "" + ) + return root + + +def _published_package(root: Path) -> Path: + """A taxonomy package shipped the way FASB ships US GAAP: a manifest listing + its entry points by their published URLs, a catalog mapping those URLs into + the package, and no report.""" + pkg = root / "gaap-2025" + (pkg / "META-INF").mkdir(parents=True) + + def entry(name: str, href: str) -> str: + return ( + f"{name}" + f"" + ) + + (pkg / "META-INF" / "taxonomyPackage.xml").write_text( + "" + "" + + entry("Everything", f"{GAAP_URL}entire/gaap-entryPoint-all-2025.xsd") + + entry("Published elsewhere", "https://elsewhere.example/other.xsd") + + entry("Elements only", f"{GAAP_URL}elts/gaap-2025.xsd") + + entry("Meta model", "../meta/gaap-meta-2025.xsd") + + "" + ) + (pkg / "META-INF" / "catalog.xml").write_text( + "" + f"" + ) + for rel in ( + "entire/gaap-entryPoint-all-2025.xsd", + "elts/gaap-2025.xsd", + "meta/gaap-meta-2025.xsd", + ): + (pkg / rel).parent.mkdir(parents=True, exist_ok=True) + (pkg / rel).write_text(f"") + return root + + +def _zip(tree: Path, archive: Path) -> Path: + import zipfile + + with zipfile.ZipFile(archive, "w") as zf: + for path in sorted(tree.rglob("*")): + if path.is_file(): + zf.write(path, path.relative_to(tree)) + return archive + + +def test_a_taxonomy_holds_no_report(tmp_path: Path) -> None: + assert _find_load_target(_bare_taxonomy(tmp_path)) is None + assert _find_load_target(_published_package(tmp_path / "pkg")) is None + + +def test_a_bare_taxonomy_loads_from_the_schema_nothing_imports(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + # The roles and types are imported, so they are not where the DTS starts; + # the linkbase pointing back at the main schema does not make it an import. + chosen = _choose_entry_point(_bare_taxonomy(tmp_path), None) + assert chosen.entry_point.document == "gov-2026.xsd" + assert chosen.others == [] + + +def test_a_bare_taxonomy_with_several_roots_asks_for_one(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + root = _bare_taxonomy(tmp_path) + (root / "gov-2026-alt.xsd").write_text(f"") + with pytest.raises(SourceError, match="entry_point"): + _choose_entry_point(root, None) + chosen = _choose_entry_point(root, "gov-2026-alt") + assert chosen.entry_point.document == "gov-2026-alt.xsd" + assert [e.document for e in chosen.others] == ["gov-2026.xsd"] + + +def test_a_manifest_loads_the_first_entry_point_it_lists(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + chosen = _choose_entry_point(_published_package(tmp_path), None) + assert chosen.entry_point.name == "Everything" + assert chosen.entry_point.document == "gaap-2025/entire/gaap-entryPoint-all-2025.xsd" + # The one published outside the package is not offered; a relative href + # resolves against the manifest. + assert [e.document for e in chosen.others] == [ + "gaap-2025/elts/gaap-2025.xsd", + "gaap-2025/meta/gaap-meta-2025.xsd", + ] + + +def test_an_entry_point_is_named_by_path_file_or_name(tmp_path: Path) -> None: + from xbrlkit.serve.session import _choose_entry_point + + root = _published_package(tmp_path) + + def chosen(wanted: str) -> str: + return _choose_entry_point(root, wanted).entry_point.document + + elements = "gaap-2025/elts/gaap-2025.xsd" + assert chosen("gaap-2025/elts/gaap-2025.xsd") == elements + assert chosen("gaap-2025.xsd") == elements + assert chosen("Elements only") == elements + assert chosen("gaap-entryPoint-all-2025").endswith("entryPoint-all-2025.xsd") + assert chosen("meta") == "gaap-2025/meta/gaap-meta-2025.xsd" + with pytest.raises(SourceError, match="matches 3"): + chosen("gaap") + with pytest.raises(SourceError, match="No entry point matches 'nope'"): + chosen("nope") + + +def test_a_taxonomy_zip_loads_with_its_entry_point_stated( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from xbrlkit.serve.session import NoXbrlFound + + parsed: list[Path] = [] + + def fake_parse(self, target, accession, filing, entity, packages=None): + parsed.append(Path(target)) + if Path(target).name == "gaap-2025.xsd": + raise NoXbrlFound(f"{target} holds no XBRL facts or concepts") + return _model() + + monkeypatch.setattr(FilingSession, "_parse", fake_parse) + archive = _zip(_published_package(tmp_path / "tree"), tmp_path / "gaap-2025.zip") + session = FilingSession() + try: + lf = session.load(str(archive)) + assert ( + parsed[-1].as_posix().endswith("gaap-2025/entire/gaap-entryPoint-all-2025.xsd") + ) + receipt = tools.load_receipt(lf) + taxonomy = receipt["taxonomy"] + assert taxonomy["entry_point"]["name"] == "Everything" + assert [e["document"] for e in taxonomy["other_entry_points"]] == [ + "gaap-2025/elts/gaap-2025.xsd", + "gaap-2025/meta/gaap-meta-2025.xsd", + ] + assert tools.describe_filing(lf)["next"][0].startswith("resolve_element") + + other = session.load(str(archive), entry_point="meta") + assert other.taxonomy is not None + assert other.taxonomy.entry_point.document == "gaap-2025/meta/gaap-meta-2025.xsd" + + # An elements-only schema has no networks to read; the error says so and + # names the entry points that do, rather than that this is not XBRL. + with pytest.raises(SourceError, match="declares no networks"): + session.load(str(archive), entry_point="Elements only") + finally: + session.close() + + +def test_a_taxonomy_zip_loads_by_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(FilingSession, "_parse", lambda self, *a, **kw: _model()) + archive = _zip(_bare_taxonomy(tmp_path / "tree"), tmp_path / "gov-2026.zip") + session = FilingSession() + + def fake_fetch(url: str, into: Path | None = None) -> Path: + assert into is not None + return Path(shutil.copy(archive, into / Path(url).name)) + + monkeypatch.setattr(session, "_fetch", fake_fetch) + try: + lf = session.load("https://taxonomies.example/gov/gov-2026.zip") + assert lf.taxonomy is not None + assert lf.taxonomy.entry_point.document == "gov-2026.xsd" + finally: + session.close() + + +def test_entry_point_is_refused_where_there_is_no_package(tmp_path: Path) -> None: + schema = _bare_taxonomy(tmp_path) / "gov-2026.xsd" + session = FilingSession() + try: + for source in (str(schema), "ACME", "https://example.test/report.htm"): + with pytest.raises(SourceError, match="taxonomy package"): + session.load(source, entry_point="gov-2026") + finally: + session.close() + + # -- the pure profile and the document toggle ------------------------------------- diff --git a/xbrlkit/serve/README.md b/xbrlkit/serve/README.md index e7d0231..ae8ae88 100644 --- a/xbrlkit/serve/README.md +++ b/xbrlkit/serve/README.md @@ -25,6 +25,7 @@ chat: *"load NVIDIA's latest 10-K"*, *"load `1045810:0001045810-26-000021`"*, | an EDGAR `cik:accession` | `1045810:0001045810-26-000021` | | a **LEI**, or a filings.xbrl.org filing id | `lei:213800H2PQMIF3OVZY47` | | a local filing | an inline `.htm`, an instance `.xml`, a directory, a `.zip` package | +| a **taxonomy** published on its own, by path or URL | `us-gaap-2025.zip`, `https://xbrl.fasb.org/us-gaap/2026/us-gaap-2026.zip`, an unpacked taxonomy directory | | a **JSON report**, by path or URL | `.clawdog.jsonld`, `.tavi.json`, `.holon.jsonld`, or a `model.json` from `export_filing` | | a URL Arelle can load | any of the above on the web | @@ -34,6 +35,17 @@ The two indexes behind the first three are [`edgar/`](../edgar/README.md) and fetch. Several filings load at once, each under an id; `unload_filing` drops one. +A zip or directory with no report in it is a taxonomy, and loads from an +entry point: the first one its `META-INF/taxonomyPackage.xml` lists +(FASB's US GAAP package lists `entire/us-gaap-entryPoint-all` first), or — +with no manifest, as GASB's exposure drafts ship — the one schema nothing else +in the package imports. The receipt's `taxonomy` names the entry point loaded +and the others on offer; `entry_point` on `load_filing` picks another, by its +path, file name or name. A taxonomy answers with its concepts and networks — +`resolve_element`, `disclosures`, `statement`, `information_block` — and no +facts. An elements-only entry point (a schema with no linkbases) has no +networks to read, and is refused as such. + ## The tools The tools are the shapes a reader needs, not a query language. diff --git a/xbrlkit/serve/server.py b/xbrlkit/serve/server.py index c8e3127..2383b1f 100644 --- a/xbrlkit/serve/server.py +++ b/xbrlkit/serve/server.py @@ -311,9 +311,11 @@ async def search_filings( "the SEC — `lei:` for that filer's latest filing on " "filings.xbrl.org, or one of that index's filing ids (e.g. " "`213800H2PQMIF3OVZY47-2022-03-31-ESEF-GB-0`). Any XBRL " - "taxonomy loads: US GAAP, IFRS, ESEF, ACFR. Takes seconds to a minute; " - "the taxonomy cache makes repeat loads fast, and a JSON report loads at " - "once, with no Arelle and no taxonomy fetch." + "taxonomy loads: US GAAP, IFRS, ESEF, ACFR. A zip or directory with no " + "report loads as a taxonomy; the receipt names the entry point, and " + "`entry_point` picks another. " + "Takes seconds to a minute; the taxonomy cache makes repeat loads fast, " + "and a JSON report loads at once, with no Arelle and no taxonomy fetch." ), structured_output=False, ) @@ -327,9 +329,20 @@ async def load_filing( ) ), ] = None, + entry_point: Annotated[ + str | None, + Field( + description=( + "A taxonomy's entry point, by path, file name or name from the " + "receipt's `taxonomy`; omit for the default." + ) + ), + ] = None, ) -> str: try: - loaded = await anyio.to_thread.run_sync(session.load, source, filing_id) + loaded = await anyio.to_thread.run_sync( + session.load, source, filing_id, entry_point + ) except (SourceError, FileNotFoundError, ValueError) as exc: return _error(str(exc)) return run(receipt, loaded) @@ -473,7 +486,8 @@ def fact_grid( "describe_filing; under the product profile a kind also works " "(balance_sheet, income_statement, cash_flow_statement, equity_statement, " "or a phrase like 'balance sheet'). `periods` limits the columns to those " - "keys, end dates, or years; otherwise the most recent eight." + "keys, end dates, or years; otherwise the most recent eight. A " + "`truncated` response continues from its `next_offset` as `offset`." ), structured_output=False, ) @@ -490,10 +504,22 @@ def statement( max_rows: Annotated[ int, Field(description="Rows to return (max 400).", ge=1, le=400) ] = 400, + offset: Annotated[ + int, + Field( + description="Rows to skip: the `next_offset` a truncated response returned.", + ge=0, + ), + ] = 0, ) -> str: return run( lambda: tools.statement( - session.get(filing), statement, periods=periods, max_rows=max_rows, pure=pure + session.get(filing), + statement, + periods=periods, + max_rows=max_rows, + pure=pure, + offset=offset, ) ) @@ -568,7 +594,8 @@ def disclosures( "disclosures first for the family index, and narrow this one with " "`member` or `periods` when part of the block answers the question. " "`block` is an id from disclosures or describe_filing, a name or " - "part of one, or a role URI." + "part of one, or a role URI. A `truncated` response continues from its " + "`next_offset` as `offset`." ), structured_output=False, ) @@ -600,6 +627,13 @@ def information_block( le=200, ), ] = None, + offset: Annotated[ + int, + Field( + description="Rows to skip: the `next_offset` a truncated response returned.", + ge=0, + ), + ] = 0, ) -> str: return run( lambda: tools.information_block( @@ -609,6 +643,7 @@ def information_block( member=member, max_rows=max_rows, max_members=max_members, + offset=offset, pure=pure, whole=whole, ) diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index ee761fd..0f43816 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -99,6 +99,28 @@ class TextSection: heading_chars: int = 0 +@dataclass +class EntryPoint: + """One way into a taxonomy: the schema its DTS is discovered from. + + ``document`` is the schema's path inside the package, as a caller names it + back through ``entry_point``; ``path`` is where it sits on disk. + """ + + name: str + document: str + path: Path + + +@dataclass +class TaxonomyEntry: + """The entry point a taxonomy package with no report was loaded from, and + the ones it offers that were not.""" + + entry_point: EntryPoint + others: list[EntryPoint] = field(default_factory=list) + + @dataclass class LoadedFiling: """One filing the server holds: the model plus its readable text. @@ -134,6 +156,10 @@ class LoadedFiling: # for, and the reader who asked to see the report asked to see that one. source_document: str | None = None source_kind: str | None = None + # Set when the source was a taxonomy published on its own — a package with + # schemas and linkbases and no report — and so holds concepts and networks + # but no facts. + taxonomy: TaxonomyEntry | None = None @property def has_xbrl(self) -> bool: @@ -319,13 +345,20 @@ def get(self, filing: str | None = None) -> LoadedFiling: # -- loading --------------------------------------------------------------- - def load(self, source: str, filing_id: str | None = None) -> LoadedFiling: - """Resolve ``source`` and load it; returns the new :class:`LoadedFiling`.""" + def load( + self, source: str, filing_id: str | None = None, entry_point: str | None = None + ) -> LoadedFiling: + """Resolve ``source`` and load it; returns the new :class:`LoadedFiling`. + + ``entry_point`` names the schema to load from a taxonomy package — a zip + or directory, local or by URL — in place of the one chosen by default. + """ source = source.strip() if not source: raise SourceError("An empty source.") + entry_point = (entry_point or "").strip() or None with self._lock: - loaded = self._load(source) + loaded = self._load(source, entry_point) wanted = filing_id or loaded.id loaded.id = self._unique_id(wanted) self._filings[loaded.id] = loaded @@ -350,12 +383,14 @@ def _unique_id(self, wanted: str) -> str: n += 1 return f"{wanted}-{n}" - def _load(self, source: str) -> LoadedFiling: + def _load(self, source: str, entry_point: str | None = None) -> LoadedFiling: path = Path(source).expanduser() if path.exists(): - return self._load_local(path, source) + return self._load_local(path, source, entry_point) if source.startswith(("http://", "https://")): - return self._load_url(source) + return self._load_url(source, entry_point) + if entry_point: + raise SourceError(_ENTRY_POINT_NEEDS_A_PACKAGE.format(source=source)) m = _CIK_ACCESSION_RE.match(source) if m: return self._load_edgar(m.group(1), m.group(2), source) @@ -385,18 +420,31 @@ def _load(self, source: str) -> LoadedFiling: "filing outside EDGAR — `lei:` or a filings.xbrl.org filing id." ) - def _load_local(self, path: Path, source: str) -> LoadedFiling: + def _load_local( + self, path: Path, source: str, entry_point: str | None = None + ) -> LoadedFiling: package_dir: Path | None = None + taxonomy: TaxonomyEntry | None = None + is_package = path.is_dir() or path.suffix.lower() == ".zip" + if entry_point and not is_package: + raise SourceError(_ENTRY_POINT_NEEDS_A_PACKAGE.format(source=source)) if path.is_file() and path.suffix.lower() in _JSON_SUFFIXES: return self._load_json(path, source) - if path.is_dir(): - package_dir = path - target = _find_load_target(path) - elif path.suffix.lower() == ".zip": - package_dir = Path(tempfile.mkdtemp(prefix="zip-", dir=self._tmp)) - with ZipFile(path) as archive: - archive.extractall(package_dir) - target = _find_load_target(package_dir) + if is_package: + if path.is_dir(): + package_dir = path + else: + package_dir = Path(tempfile.mkdtemp(prefix="zip-", dir=self._tmp)) + with ZipFile(path) as archive: + archive.extractall(package_dir) + report = None if entry_point else _find_load_target(package_dir) + if report is None: + # No report in the package: a taxonomy published on its own, which + # loads from one of its schemas and answers with concepts and networks. + taxonomy = _choose_entry_point(package_dir, entry_point) + target = taxonomy.entry_point.path + else: + target = report else: target = path package_dir = path.parent @@ -417,11 +465,23 @@ def _load_local(self, path: Path, source: str) -> LoadedFiling: model = self._parse( target, accession=accession, filing=None, entity=None, packages=packages ) - except NoXbrlFound: + except NoXbrlFound as exc: + if taxonomy is not None: + # The model holds the concepts its networks and facts reach, so an + # elements-only schema — no linkbases, nothing to reach them — reads + # as empty. Say so, rather than that the package is not XBRL. + others = ", ".join(e.document for e in taxonomy.others) + raise SourceError( + f"Entry point {taxonomy.entry_point.document} declares no networks " + "for the tools to read (an elements-only schema); load an entry " + f"point with linkbases instead{': ' + others if others else ''}." + ) from exc if target.suffix.lower() not in _DOCUMENT_SUFFIXES: raise return self._document_only(target, source, accession, package_dir) - return self._finish(_local_id(path, model), source, model, target, package_dir) + loaded = self._finish(_local_id(path, model), source, model, target, package_dir) + loaded.taxonomy = taxonomy + return loaded def _load_json( self, path: Path, source: str, document: Path | None = None @@ -479,10 +539,21 @@ def _load_json( source_kind=kind if served else None, ) - def _load_url(self, url: str) -> LoadedFiling: + def _load_url(self, url: str, entry_point: str | None = None) -> LoadedFiling: clean = Path(url.split("?", 1)[0]) accession = clean.stem or url suffix = clean.suffix.lower() + if suffix == ".zip": + # A package by URL is downloaded and loaded as a local one: the report + # found inside it, or — for a taxonomy published on its own, which is + # how FASB, XBRL US and the IFRS Foundation distribute theirs — an + # entry point. + archive = self._fetch( + url, into=Path(tempfile.mkdtemp(prefix="url-", dir=self._tmp)) + ) + return self._load_local(archive, url, entry_point) + if entry_point: + raise SourceError(_ENTRY_POINT_NEEDS_A_PACKAGE.format(source=url)) if suffix in _JSON_SUFFIXES: # A JSON report is read here, not by Arelle, which cannot load one. This # is how a report published as an artifact — a holon or a TAVI on a CDN — @@ -1437,12 +1508,14 @@ def _is_inline(head: bytes) -> bool: return b"ix:nonNumeric" in head or b"ix:nonFraction" in head or b"ix:header" in head -def _find_load_target(package_dir: Path) -> Path: - """The file Arelle should load from a filing directory: the inline +def _find_load_target(package_dir: Path) -> Path | None: + """The report Arelle should load from a filing directory: the inline document (the largest ``.htm`` carrying ``ix:`` markup), else the XBRL instance — recognised by its root element, since a package need not follow EDGAR's naming (an instance called ``instance.xml`` beside - ``report.xsd`` and hyphenated linkbases is a valid package too). + ``report.xsd`` and hyphenated linkbases is a valid package too). ``None`` + when the package holds no report: a taxonomy published on its own, which + :func:`_choose_entry_point` picks a schema from. The whole tree is searched, not just the top level. A conformant XBRL taxonomy package puts nothing at its root: an ESEF report sits under @@ -1479,10 +1552,174 @@ def _find_load_target(package_dir: Path) -> Path: raise SourceError( f"{len(instances)} XBRL instances in {package_dir} ({names}); point at one." ) - raise SourceError( - f"No inline document or XBRL instance found in {package_dir}; " - "point at the file to load." + return None + + +_ENTRY_POINT_NEEDS_A_PACKAGE = ( + "entry_point picks a schema inside a taxonomy package — a .zip or a " + "directory, local or by URL; {source} is not one." +) +_SCHEMA_LOCATION_RE = re.compile(rb"""schemaLocation\s*=\s*(["'])(.*?)\1""", re.DOTALL) + + +def _local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def _choose_entry_point(package_dir: Path, wanted: str | None) -> TaxonomyEntry: + """The schema to load a taxonomy package from, when it holds no report. + + A package that declares its entry points (``META-INF/taxonomyPackage.xml``) + loads the one ``wanted`` names, else the first it lists: FASB lists the whole + taxonomy first — its US GAAP and SRT packages open with + ``entire/…-entryPoint-all``, ahead of the elements-only, DQC and meta-model + entry points. A package without a manifest loads its one root + schema, the one no other schema in it imports; with several, the caller + names one. The others are returned beside the choice so it is never silent. + """ + # Resolved once, so a package unpacked under a symlinked temp directory + # (macOS's /var) still contains the resolved paths its entry points name. + package_dir = package_dir.resolve() + declared = _declared_entry_points(package_dir) + entry_points = declared or _root_schemas(package_dir) + if not entry_points: + raise SourceError( + f"No inline document, XBRL instance or taxonomy schema found in " + f"{package_dir}; point at the file to load." + ) + if wanted: + chosen = _match_entry_point(entry_points, wanted) + elif declared or len(entry_points) == 1: + chosen = entry_points[0] + else: + raise SourceError( + f"This taxonomy package has no manifest and {len(entry_points)} root " + f"schemas; pass entry_point to choose one: " + f"{', '.join(e.document for e in entry_points)}." + ) + return TaxonomyEntry( + entry_point=chosen, others=[e for e in entry_points if e is not chosen] + ) + + +def _match_entry_point(entry_points: list[EntryPoint], wanted: str) -> EntryPoint: + """The entry point ``wanted`` names: exactly, by its name, its path in the + package or its file name with or without ``.xsd``; else the one it is part + of.""" + key = wanted.strip().lower() + + def names(e: EntryPoint) -> set[str]: + document = Path(e.document) + return { + e.name.lower(), + e.document.lower(), + document.name.lower(), + document.stem.lower(), + } + + hits = [e for e in entry_points if key in names(e)] + if not hits: + hits = [e for e in entry_points if any(key in n for n in names(e))] + if len(hits) == 1: + return hits[0] + listed = ", ".join(e.document for e in hits or entry_points) + if hits: + raise SourceError(f"entry_point {wanted!r} matches {len(hits)}: {listed}.") + raise SourceError(f"No entry point matches {wanted!r}; this package has: {listed}.") + + +def _declared_entry_points(package_dir: Path) -> list[EntryPoint]: + """The entry points the package's manifests declare, in their order, that + resolve to a schema inside the package. + + An entry point names its schema by the URL it is published at + (``https://xbrl.fasb.org/us-gaap/2025/entire/…``); the package catalog's + ``rewriteURI`` maps that URL to the copy inside the package, and a relative + ``href`` resolves against the manifest itself. + """ + found: list[EntryPoint] = [] + for manifest in sorted(package_dir.rglob("META-INF/taxonomyPackage.xml")): + meta_inf = manifest.parent + try: + root = ElementTree.parse(manifest).getroot() + except ElementTree.ParseError as exc: + logger.warning("unreadable taxonomy package manifest %s: %s", manifest, exc) + continue + rewrites = _catalog_rewrites(meta_inf / "catalog.xml") + for element in root.iter(): + if _local_name(element.tag) != "entryPoint": + continue + name = "" + href = "" + for child in element: + local = _local_name(child.tag) + if local == "name" and not name: + name = (child.text or "").strip() + elif local == "entryPointDocument" and not href: + href = (child.get("href") or "").strip() + path = _resolve_package_href(href, meta_inf, rewrites) if href else None + if path is None or not path.is_file() or not path.is_relative_to(package_dir): + continue + document = path.relative_to(package_dir).as_posix() + found.append(EntryPoint(name=name or path.stem, document=document, path=path)) + return found + + +def _catalog_rewrites(catalog: Path) -> list[tuple[str, Path]]: + """A package catalog's URL prefixes and the directories they map to, + longest prefix first.""" + if not catalog.is_file(): + return [] + try: + root = ElementTree.parse(catalog).getroot() + except ElementTree.ParseError as exc: + logger.warning("unreadable taxonomy package catalog %s: %s", catalog, exc) + return [] + rewrites = [ + (start, catalog.parent / (element.get("rewritePrefix") or "")) + for element in root.iter() + if _local_name(element.tag) == "rewriteURI" + and (start := element.get("uriStartString")) + ] + return sorted(rewrites, key=lambda r: -len(r[0])) + + +def _resolve_package_href( + href: str, base: Path, rewrites: list[tuple[str, Path]] +) -> Path | None: + """Where an ``href`` in a package manifest sits on disk, or ``None`` when + it points outside the package.""" + if "://" not in href: + return (base / href).resolve() + for start, prefix in rewrites: + if href.startswith(start): + return (prefix / href[len(start) :]).resolve() + return None + + +def _root_schemas(package_dir: Path) -> list[EntryPoint]: + """The schemas in a package that no other schema in it imports or + includes — where a DTS starts when no manifest says so. A published + taxonomy with no manifest (GASB's exposure drafts) has one: the schema that + imports its roles and types and links its linkbases.""" + schemas = sorted( + p for p in package_dir.rglob("*.xsd") if p.is_file() and not p.name.startswith(".") ) + imported: set[Path] = set() + for schema in schemas: + for match in _SCHEMA_LOCATION_RE.finditer(schema.read_bytes()): + for location in match.group(2).decode("utf-8", "replace").split(): + if "://" not in location: + imported.add((schema.parent / location.split("#", 1)[0]).resolve()) + return [ + EntryPoint( + name=schema.stem, + document=schema.relative_to(package_dir).as_posix(), + path=schema, + ) + for schema in schemas + if schema.resolve() not in imported + ] _XML_IDENTITY = { diff --git a/xbrlkit/serve/tools.py b/xbrlkit/serve/tools.py index 69b8b49..90f5c05 100644 --- a/xbrlkit/serve/tools.py +++ b/xbrlkit/serve/tools.py @@ -34,7 +34,12 @@ from xbrlkit.config import CONFIG, Config from xbrlkit.model import Arc, Concept, Network, Period, Unit, XbrlFact, XbrlModel from xbrlkit.serialize import classify_network, root_qname -from xbrlkit.serve.session import FilingSession, LoadedFiling, TextSection +from xbrlkit.serve.session import ( + FilingSession, + LoadedFiling, + TaxonomyEntry, + TextSection, +) from xbrlkit.edgar.items import describe_items, is_earnings_release, items_note from xbrlkit.information_block import ( Disclosure, @@ -474,7 +479,7 @@ def list_filings(session: FilingSession) -> dict[str, Any]: return {"filings": rows, "count": len(rows)} -LOAD_RECEIPT_KEYS = ("profile", "filing", "entity", "counts") +LOAD_RECEIPT_KEYS = ("profile", "taxonomy", "filing", "entity", "counts") def load_receipt( @@ -615,6 +620,7 @@ def describe_filing( "text": "primary document" if whole and lf.has_document else "tagged text blocks", "xbrl": lf.has_xbrl, }, + **({"taxonomy": _describe_taxonomy(lf.taxonomy)} if lf.taxonomy else {}), "filing": { "id": lf.id, "source": lf.source, @@ -690,6 +696,25 @@ def describe_filing( } +def _describe_taxonomy(taxonomy: TaxonomyEntry) -> dict[str, Any]: + """Which entry point a taxonomy package was loaded from, and the others it + offers — the choice stated, never silent.""" + return { + "entry_point": { + "name": taxonomy.entry_point.name, + "document": taxonomy.entry_point.document, + }, + "other_entry_points": [ + {"name": e.name, "document": e.document} for e in taxonomy.others + ], + "note": ( + "a taxonomy with no report: concepts and networks, no facts, periods or " + "units. load_filing with the same source and `entry_point` set to another " + "entry point's document loads that one instead." + ), + } + + def _next_steps(lf: LoadedFiling) -> list[str]: """What to call next, given what this filing actually is. @@ -698,6 +723,15 @@ def _next_steps(lf: LoadedFiling) -> list[str]: the item codes say the substance is elsewhere, that goes first — above the fact tools, which for an 8-K have almost nothing to work with. """ + if lf.taxonomy is not None: + return [ + "resolve_element to find concepts by phrase — each with its label, type, " + "balance, period type and the networks it sits in", + "disclosures for the taxonomy's networks as families", + "statement or information_block with a network from the list above to " + "read its tree", + "calculation for what sums to a total, where the taxonomy has calculation arcs", + ] exhibit_first: list[str] = [] if is_earnings_release(lf.model.filing.items): exhibit_first = [ @@ -1174,11 +1208,13 @@ def statement( periods: list[str] | None = None, max_rows: int = MAX_STATEMENT_ROWS, pure: bool = False, + offset: int = 0, ) -> dict[str, Any]: _require_xbrl(lf, "presentation networks") model, idx = lf.model, index_for(lf) network = _find_network(idx, statement, pure=pure) max_rows = max(1, min(int(max_rows or MAX_STATEMENT_ROWS), MAX_STATEMENT_ROWS)) + offset = max(0, int(offset or 0)) children: dict[str, list[Arc]] = defaultdict(list) parents: set[str] = set() @@ -1194,14 +1230,10 @@ def statement( rows: list[dict[str, Any]] = [] used_periods: dict[str, int] = defaultdict(int) truncated = False + walked = 0 + page_trail: tuple[str, ...] = () - def visit( - qname: str, depth: int, label_role: str | None, trail: tuple[str, ...] - ) -> None: - nonlocal truncated - if len(rows) >= max_rows: - truncated = True - return + def row_at(qname: str, depth: int, label_role: str | None) -> dict[str, Any]: concept = model.concepts.get(qname) row: dict[str, Any] = { "depth": depth, @@ -1229,7 +1261,20 @@ def visit( used_periods[key] += 1 if values: row["values"] = values - rows.append(row) + return row + + def visit( + qname: str, depth: int, label_role: str | None, trail: tuple[str, ...] + ) -> None: + nonlocal truncated, walked, page_trail + if len(rows) >= max_rows: + truncated = True + return + walked += 1 + if walked > offset: + if not rows: + page_trail = trail + rows.append(row_at(qname, depth, label_role)) if qname in trail or depth > 14: return for arc in children.get(qname, []): @@ -1237,6 +1282,8 @@ def visit( for root in roots: visit(root, 0, None, ()) + if offset and not rows: + raise ToolError(f"offset {offset} is past the end of this network ({walked} rows)") period_by_key = {_period_key(p): p for p in model.periods} wanted = None @@ -1278,14 +1325,38 @@ def visit( "rows": rows, "row_count": len(rows), "truncated": truncated, + **_page_fields(model, offset, len(rows), truncated, page_trail), "note": ( "consolidated values only, most precise duplicate kept; depth is the " "presentation nesting; a label like 'Total' or a negated label reflects " - "the preferred label on the arc" + "the preferred label on the arc; a truncated network continues from " + "`next_offset` passed as `offset`" ), } +def _page_fields( + model: XbrlModel, + offset: int, + returned: int, + truncated: bool, + trail: tuple[str, ...], +) -> dict[str, Any]: + """Where a page of a presentation walk sits in the whole: the offset it + starts at, the headers above its first row — a later page opens deep in + the tree, and its depths alone do not say under what — and where the next + page starts.""" + out: dict[str, Any] = {} + if offset: + out["offset"] = offset + out["ancestors"] = [ + {"concept": q, "label": _pref_label(model.concepts.get(q), q)} for q in trail + ] + if truncated: + out["next_offset"] = offset + returned + return out + + def calculation( lf: LoadedFiling, concept: str, @@ -1584,6 +1655,7 @@ def information_block( member: str | None = None, max_rows: int = MAX_BLOCK_ROWS, max_members: int | None = None, + offset: int = 0, *, pure: bool = False, whole: bool = True, @@ -1594,12 +1666,18 @@ def information_block( Member breakdowns are kept most-reported first, up to the response budget (or ``max_members`` when the caller sets one); a row is never left blank - by that cut, and every row says how many breakdowns it lost.""" + by that cut, and every row says how many breakdowns it lost. + + A section longer than ``max_rows`` — a taxonomy's own networks run to + hundreds of concepts — pages by ``offset``. The axes, calculation arcs and + text blocks describe the section, not a page, and come with the first page + alone; a later page carries its rows and their columns.""" _require_xbrl(lf, "presentation networks") model, idx = lf.model, index_for(lf) blocks, membership, families = blocks_for(lf) st = _find_block(idx, blocks, block, pure=pure) max_rows = max(1, min(int(max_rows or MAX_BLOCK_ROWS), MAX_BLOCK_ROWS)) + offset = max(0, int(offset or 0)) member_limit = ( max(1, min(int(max_members), MAX_BLOCK_MEMBERS_CAP)) if max_members is not None @@ -1673,6 +1751,8 @@ def information_block( rows: list[dict[str, Any]] = [] used_periods: dict[str, int] = defaultdict(int) truncated = False + walked = 0 + page_trail: tuple[str, ...] = () def cell(f: XbrlFact) -> Any: if f.is_nil: @@ -1685,10 +1765,21 @@ def cell(f: XbrlFact) -> Any: def visit( qname: str, depth: int, label_role: str | None, trail: tuple[str, ...] ) -> None: - nonlocal truncated + nonlocal truncated, walked, page_trail if len(rows) >= max_rows: truncated = True return + walked += 1 + if walked > offset: + if not rows: + page_trail = trail + rows.append(row_at(qname, depth, label_role)) + if qname in trail or depth > 14: + return + for arc in children.get(qname, []): + visit(arc.to_qname, depth + 1, arc.preferred_label, trail + (qname,)) + + def row_at(qname: str, depth: int, label_role: str | None) -> dict[str, Any]: concept = model.concepts.get(qname) row: dict[str, Any] = { "depth": depth, @@ -1737,14 +1828,12 @@ def visit( row["values"] = values if by_member: row["members"] = by_member - rows.append(row) - if qname in trail or depth > 14: - return - for arc in children.get(qname, []): - visit(arc.to_qname, depth + 1, arc.preferred_label, trail + (qname,)) + return row for root in roots: visit(root, 0, None, ()) + if offset and not rows: + raise ToolError(f"offset {offset} is past the end of this block ({walked} rows)") period_by_key = {_period_key(p): p for p in model.periods} wanted = {w.strip() for w in periods if w and w.strip()} if periods else None @@ -1959,15 +2048,19 @@ def newest(period_keys: set[str]) -> str: column.pop("duration", None) column.pop("calendar", None) out: dict[str, Any] = {"block": head, "columns": columns} - if axes_out: + # The axes, calculation and text describe the section; a later page of it + # would only repeat them. + first_page = not offset + if axes_out and first_page: out["axes"] = axes_out out["rows"] = rows - if calc_out: + if calc_out and first_page: out["calculation"] = calc_out - if text_out: + if text_out and first_page: out["text"] = text_out out["row_count"] = len(rows) out["truncated"] = truncated + out.update(_page_fields(model, offset, len(rows), truncated, page_trail)) if members_omitted: out["members_omitted"] = members_omitted if columns_omitted: @@ -1985,7 +2078,9 @@ def newest(period_keys: set[str]) -> str: "and columns dropped from it, and a row is never left blank by either cut; " "`calculation` lists each total's children with weights, how many of the " "shown periods foot on consolidated values, and any difference; `text` " - "entries are tagged text blocks — read one with read_text from its offset" + "entries are tagged text blocks — read one with read_text from its offset; " + "a truncated block continues from `next_offset` passed as `offset`, and " + "axes, calculation and text come with the first page only" ) return out