From 7f0a4777af1b65cdbd1c2cb81dc5fc231b792c5e Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Thu, 24 Sep 2026 18:12:36 -0500 Subject: [PATCH 1/2] fix(tavi): read INF precision and abstract members back A TAVI numeric fact with no decimals is infinitely precise (the fact value object leaves the property out for INF), but the reader read it as unknown, so an exact fact lost to a rounded duplicate of itself: JPM's shares issued at 2024-12-31 read 4,104,900,000 instead of 4,104,933,895. Domains and members are objects of their own in TAVI, not concepts, so no fact can report against one; they now read back abstract, as the parse has them. --- tests/test_deserialize.py | 12 ++++++++++++ xbrlkit/deserialize/tavi.py | 21 +++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py index 33b1a4e..ccb9191 100644 --- a/tests/test_deserialize.py +++ b/tests/test_deserialize.py @@ -601,6 +601,18 @@ def test_round_trip_keeps_the_facts(model: XbrlModel, fmt: str) -> None: assert cash.unit_id == next(u.id for u in got.units if u.measure == "iso4217:USD") +def test_tavi_reads_an_exact_fact_as_infinitely_precise(model: XbrlModel) -> None: + """TAVI writes INF by leaving `decimals` out; the reader must not read that + as unknown, or an exact fact loses to a rounded duplicate of itself.""" + assets = next(f for f in model.facts if f.concept_qname == "us-gaap:Assets") + assets.decimals = "INF" + got = from_tavi_json(to_tavi(model)) + back = next(f for f in got.facts if f.concept_qname == "us-gaap:Assets") + assert back.decimals == "INF" + text = next(f for f in got.facts if f.concept_qname == "dei:DocumentType") + assert text.decimals is None + + @pytest.mark.parametrize("fmt", ["tavi", "holon"]) def test_round_trip_keeps_the_networks(model: XbrlModel, fmt: str) -> None: got = _through(model, fmt) diff --git a/xbrlkit/deserialize/tavi.py b/xbrlkit/deserialize/tavi.py index 895a288..9d842e6 100644 --- a/xbrlkit/deserialize/tavi.py +++ b/xbrlkit/deserialize/tavi.py @@ -22,9 +22,9 @@ - the derived period semantics (duration bucket, calendar placement) — the one exception, recomputed by :mod:`xbrlkit.periods` from the dates themselves, which is where the parse gets them too; -- the abstractness of axes, domains and members: the emitter turns those - elements into dimensional objects, and TAVI has no flag for it (a hypercube - is marked ``is_hypercube_item`` when its role presents the table); +- nothing further on the dimensional elements: axes, domains and members are + objects of their own in TAVI, so they read back abstract, and a hypercube is + marked ``is_hypercube_item`` when its role presents the table; - reference linkbase entries, ``Network.role_id``, a fact's source hash and raw lexical value, and a dimension's segment/scenario axis; - a fact's own entity when it differs from the report's — the emitter writes @@ -200,7 +200,6 @@ def _read(document: Mapping[str, Any]) -> tuple[XbrlModel, ImportGaps]: } gaps = ImportGaps( missing=[ - "abstract flag on axes, domains and members", "a hypercube's name and primary items where its role presents no table", "an axis's default member, read as its domain", "concept references", @@ -352,13 +351,19 @@ def _concepts( concepts[qname] = _bare(qname, namespaces, is_abstract=True, is_dimension_item=True) domain = obj.get("domainClass") if isinstance(domain, str) and domain and domain not in concepts: - concepts[domain] = _bare(domain, namespaces, is_domain_member=True) + concepts[domain] = _bare( + domain, namespaces, is_abstract=True, is_domain_member=True + ) + # Domains and members are objects of their own in TAVI, not concepts, so no + # fact can report against one: abstract is what the model says they are. for key in ("domainClasses", "members"): for entry in _sequence(xbrl_model.get(key)): qname = str(_mapping(entry).get("name", "")) if qname and qname not in concepts: - concepts[qname] = _bare(qname, namespaces, is_domain_member=True) + concepts[qname] = _bare( + qname, namespaces, is_abstract=True, is_domain_member=True + ) return concepts @@ -829,6 +834,10 @@ def _facts( numeric_value = _float(value_str) if unit is not None else None decimals = value_obj.get("decimals") + if decimals is None and unit is not None and value_str is not None: + # Absent on a numeric fact means infinitely precise (the fact value + # object's `decimals`), which is how the emitter writes INF. + decimals = "INF" language = dimensions.get("xbrl:language") facts.append( XbrlFact( From d7ff1ab346b1f66155716b58f383c19d176ef06b Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Thu, 24 Sep 2026 18:12:55 -0500 Subject: [PATCH 2/2] feat(serve): load a published filing from its TAVI model first The public catalog lists each filing's tavi.json beside its holon. PublishedFiling carries both (holon_url is now optional, and a filing published as a TAVI alone is still a published filing), and the load tries the TAVI first and falls back to the holon when it cannot be fetched or read. The TAVI carries its text blocks inline, so there is nothing to fetch after it, and it loads in about half the time: NVDA 1.0s against 1.7s, JPM 3.1s against 5.5s, from the live CDN. Its fact count matches the parse of the filing (NVDA 1,305), where the holon collapses repeated tags. --- tests/test_published.py | 66 ++++++++++++++++++++++++++++++++++++++-- xbrlkit/serve/session.py | 50 +++++++++++++++++++++--------- 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/tests/test_published.py b/tests/test_published.py index 3460bc9..d022c76 100644 --- a/tests/test_published.py +++ b/tests/test_published.py @@ -24,7 +24,7 @@ from tests.test_deserialize import _model from xbrlkit.config import Config from xbrlkit.model import Concept, Label, XbrlFact, XbrlModel -from xbrlkit.serialize import to_holon +from xbrlkit.serialize import to_holon, to_tavi from xbrlkit.serve import tools from xbrlkit.serve.session import FilingSession, PublishedFiling @@ -341,7 +341,7 @@ def test_a_missing_fragment_stays_a_url_and_the_load_succeeds( @pytest.mark.unit -def test_a_manifest_without_a_holon_is_not_a_published_filing() -> None: +def test_a_manifest_without_a_model_is_not_a_published_filing() -> None: from xbrlkit.serve.session import _published_from assert ( @@ -354,6 +354,68 @@ def test_a_manifest_without_a_holon_is_not_a_published_filing() -> None: assert published == PublishedFiling( accession="acc", holon_url="http://x/f/holon.jsonld" ) + # A filing published as a TAVI model alone is still a published filing. + only_tavi = _published_from( + "acc", [{"kind": "tavi", "name": "tavi.json"}], "http://x/f/" + ) + assert only_tavi == PublishedFiling(accession="acc", tavi_url="http://x/f/tavi.json") + assert only_tavi.model_urls == ["http://x/f/tavi.json"] + + +def _publish_tavi(tmp_path: Path, base: str, *, reachable: bool = True) -> None: + """List a TAVI model in the ACME catalog ahead of the holon; write the file + only when it should be reachable. The TAVI carries its text block inline.""" + folder = f"{base}/2024/{CIK}/{ACCESSION}" + root = tmp_path / "2024" / CIK / ACCESSION + if reachable: + model = _model_with_text() + for fact in model.facts: + if fact.value_str == "__FRAGMENT_URL__": + fact.value_str = FRAGMENT + (root / "tavi.json").write_text(to_tavi(model)) + catalog_path = tmp_path / "companies" / "acme.json" + catalog = json.loads(catalog_path.read_text()) + newest = catalog["filings"][0] + newest["representations"] = [ + {"kind": "tavi", "name": "tavi.json", "url": f"{folder}/tavi.json"}, + *newest["representations"], + ] + catalog_path.write_text(json.dumps(catalog)) + + +@pytest.mark.unit +def test_a_ticker_loads_the_published_tavi_first( + cdn: str, tmp_path: Path, no_edgar +) -> None: + _publish_tavi(tmp_path, cdn) + session = _session(cdn) + try: + loaded = session.load("ACME") + assert loaded.source_kind == "tavi" + assert loaded.has_document is True + assert tools.fact_grid(loaded, ["us-gaap:Assets"])["rows"][0]["value"] == 1000.0 + block = next( + f + for f in loaded.model.facts + if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock" + ) + assert block.value_str == FRAGMENT + finally: + session.close() + + +@pytest.mark.unit +def test_an_unreachable_tavi_falls_back_to_the_holon( + cdn: str, tmp_path: Path, no_edgar +) -> None: + _publish_tavi(tmp_path, cdn, reachable=False) + session = _session(cdn) + try: + loaded = session.load("ACME") + assert loaded.source_kind == "holon" + assert tools.fact_grid(loaded, ["us-gaap:Assets"])["rows"][0]["value"] == 1000.0 + finally: + session.close() # ── The filer's identity ─────────────────────────────────────────────────── diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index 0f43816..532c1d9 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -259,14 +259,21 @@ def _enrich_filer(loaded: LoadedFiling, fields: Mapping[str, Any]) -> None: @dataclass class PublishedFiling: - """A filing as the public data CDN lists it: the holon to load, the document - as filed when it was published beside it, and the filer's ticker — the key - the catalog holds that filer's identity under.""" + """A filing as the public data CDN lists it: the representations to load + (the TAVI model first, the holon beside it), the document as filed when it + was published too, and the filer's ticker — the key the catalog holds that + filer's identity under.""" accession: str - holon_url: str + holon_url: str | None = None document_url: str | None = None ticker: str | None = None + tavi_url: str | None = None + + @property + def model_urls(self) -> list[str]: + """The representations to try, in order: the TAVI model, then the holon.""" + return [url for url in (self.tavi_url, self.holon_url) if url] def _published_from( @@ -276,8 +283,8 @@ def _published_from( ticker: str | None = None, ) -> PublishedFiling | None: """The published filing a catalog entry or manifest describes, or ``None`` - when it lists no holon.""" - holon = document = None + when it lists neither a TAVI model nor a holon.""" + holon = document = tavi = None for rep in representations if isinstance(representations, list) else []: if not isinstance(rep, dict): continue @@ -288,15 +295,18 @@ def _published_from( continue if rep.get("kind") == "holon": holon = url + elif rep.get("kind") == "tavi": + tavi = url elif rep.get("kind") == "document": document = url - if not holon: + if not (holon or tavi): return None return PublishedFiling( accession=accession or "", holon_url=holon, document_url=document, ticker=ticker, + tavi_url=tavi, ) @@ -998,19 +1008,31 @@ def _filer_from_edgar(self, cik: str, client: Any = None) -> dict[str, Any]: } def _load_published(self, published: PublishedFiling, source: str) -> LoadedFiling: - """The filing from its published holon, with the document as filed beside - it when the CDN has that too — the same shape an EDGAR load gives, in a - fraction of the time and with no Arelle.""" - into = self._tmp / (published.accession or Path(published.holon_url).stem) - holon = self._fetch(published.holon_url, into=into) + """The filing from its published TAVI model (or its holon when there is no + TAVI, or it cannot be read), with the document as filed beside it when the + CDN has that too — the same shape an EDGAR load gives, in a fraction of the + time and with no Arelle. The TAVI carries its text blocks inline, so there + are no fragments to fetch after it.""" + urls = published.model_urls + into = self._tmp / (published.accession or Path(urls[0].split("?", 1)[0]).stem) document: Path | None = None if published.document_url: try: document = self._fetch(published.document_url, into=into) except requests.RequestException as exc: logger.warning("published document unavailable for %s: %s", source, exc) - logger.info("loading %s from its published holon", source) - loaded = self._load_json(holon, source, document=document) + loaded: LoadedFiling | None = None + for position, url in enumerate(urls): + try: + path = self._fetch(url, into=into) + logger.info("loading %s from %s", source, Path(url.split("?", 1)[0]).name) + loaded = self._load_json(path, source, document=document) + break + except (requests.RequestException, SourceError) as exc: + if position == len(urls) - 1: + raise + logger.warning("%s unavailable for %s (%s); trying the next", url, source, exc) + assert loaded is not None _enrich_filer( loaded, self._filer_metadata(loaded.model.entity.cik, ticker=published.ticker) )