diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py index 33b1a4e..ccb9191 100644 --- a/tests/test_deserialize.py +++ b/tests/test_deserialize.py @@ -601,6 +601,18 @@ def test_round_trip_keeps_the_facts(model: XbrlModel, fmt: str) -> None: assert cash.unit_id == next(u.id for u in got.units if u.measure == "iso4217:USD") +def test_tavi_reads_an_exact_fact_as_infinitely_precise(model: XbrlModel) -> None: + """TAVI writes INF by leaving `decimals` out; the reader must not read that + as unknown, or an exact fact loses to a rounded duplicate of itself.""" + assets = next(f for f in model.facts if f.concept_qname == "us-gaap:Assets") + assets.decimals = "INF" + got = from_tavi_json(to_tavi(model)) + back = next(f for f in got.facts if f.concept_qname == "us-gaap:Assets") + assert back.decimals == "INF" + text = next(f for f in got.facts if f.concept_qname == "dei:DocumentType") + assert text.decimals is None + + @pytest.mark.parametrize("fmt", ["tavi", "holon"]) def test_round_trip_keeps_the_networks(model: XbrlModel, fmt: str) -> None: got = _through(model, fmt) diff --git a/tests/test_published.py b/tests/test_published.py index 3460bc9..d022c76 100644 --- a/tests/test_published.py +++ b/tests/test_published.py @@ -24,7 +24,7 @@ from tests.test_deserialize import _model from xbrlkit.config import Config from xbrlkit.model import Concept, Label, XbrlFact, XbrlModel -from xbrlkit.serialize import to_holon +from xbrlkit.serialize import to_holon, to_tavi from xbrlkit.serve import tools from xbrlkit.serve.session import FilingSession, PublishedFiling @@ -341,7 +341,7 @@ def test_a_missing_fragment_stays_a_url_and_the_load_succeeds( @pytest.mark.unit -def test_a_manifest_without_a_holon_is_not_a_published_filing() -> None: +def test_a_manifest_without_a_model_is_not_a_published_filing() -> None: from xbrlkit.serve.session import _published_from assert ( @@ -354,6 +354,68 @@ def test_a_manifest_without_a_holon_is_not_a_published_filing() -> None: assert published == PublishedFiling( accession="acc", holon_url="http://x/f/holon.jsonld" ) + # A filing published as a TAVI model alone is still a published filing. + only_tavi = _published_from( + "acc", [{"kind": "tavi", "name": "tavi.json"}], "http://x/f/" + ) + assert only_tavi == PublishedFiling(accession="acc", tavi_url="http://x/f/tavi.json") + assert only_tavi.model_urls == ["http://x/f/tavi.json"] + + +def _publish_tavi(tmp_path: Path, base: str, *, reachable: bool = True) -> None: + """List a TAVI model in the ACME catalog ahead of the holon; write the file + only when it should be reachable. The TAVI carries its text block inline.""" + folder = f"{base}/2024/{CIK}/{ACCESSION}" + root = tmp_path / "2024" / CIK / ACCESSION + if reachable: + model = _model_with_text() + for fact in model.facts: + if fact.value_str == "__FRAGMENT_URL__": + fact.value_str = FRAGMENT + (root / "tavi.json").write_text(to_tavi(model)) + catalog_path = tmp_path / "companies" / "acme.json" + catalog = json.loads(catalog_path.read_text()) + newest = catalog["filings"][0] + newest["representations"] = [ + {"kind": "tavi", "name": "tavi.json", "url": f"{folder}/tavi.json"}, + *newest["representations"], + ] + catalog_path.write_text(json.dumps(catalog)) + + +@pytest.mark.unit +def test_a_ticker_loads_the_published_tavi_first( + cdn: str, tmp_path: Path, no_edgar +) -> None: + _publish_tavi(tmp_path, cdn) + session = _session(cdn) + try: + loaded = session.load("ACME") + assert loaded.source_kind == "tavi" + assert loaded.has_document is True + assert tools.fact_grid(loaded, ["us-gaap:Assets"])["rows"][0]["value"] == 1000.0 + block = next( + f + for f in loaded.model.facts + if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock" + ) + assert block.value_str == FRAGMENT + finally: + session.close() + + +@pytest.mark.unit +def test_an_unreachable_tavi_falls_back_to_the_holon( + cdn: str, tmp_path: Path, no_edgar +) -> None: + _publish_tavi(tmp_path, cdn, reachable=False) + session = _session(cdn) + try: + loaded = session.load("ACME") + assert loaded.source_kind == "holon" + assert tools.fact_grid(loaded, ["us-gaap:Assets"])["rows"][0]["value"] == 1000.0 + finally: + session.close() # ── The filer's identity ─────────────────────────────────────────────────── diff --git a/xbrlkit/deserialize/tavi.py b/xbrlkit/deserialize/tavi.py index 895a288..9d842e6 100644 --- a/xbrlkit/deserialize/tavi.py +++ b/xbrlkit/deserialize/tavi.py @@ -22,9 +22,9 @@ - the derived period semantics (duration bucket, calendar placement) — the one exception, recomputed by :mod:`xbrlkit.periods` from the dates themselves, which is where the parse gets them too; -- the abstractness of axes, domains and members: the emitter turns those - elements into dimensional objects, and TAVI has no flag for it (a hypercube - is marked ``is_hypercube_item`` when its role presents the table); +- nothing further on the dimensional elements: axes, domains and members are + objects of their own in TAVI, so they read back abstract, and a hypercube is + marked ``is_hypercube_item`` when its role presents the table; - reference linkbase entries, ``Network.role_id``, a fact's source hash and raw lexical value, and a dimension's segment/scenario axis; - a fact's own entity when it differs from the report's — the emitter writes @@ -200,7 +200,6 @@ def _read(document: Mapping[str, Any]) -> tuple[XbrlModel, ImportGaps]: } gaps = ImportGaps( missing=[ - "abstract flag on axes, domains and members", "a hypercube's name and primary items where its role presents no table", "an axis's default member, read as its domain", "concept references", @@ -352,13 +351,19 @@ def _concepts( concepts[qname] = _bare(qname, namespaces, is_abstract=True, is_dimension_item=True) domain = obj.get("domainClass") if isinstance(domain, str) and domain and domain not in concepts: - concepts[domain] = _bare(domain, namespaces, is_domain_member=True) + concepts[domain] = _bare( + domain, namespaces, is_abstract=True, is_domain_member=True + ) + # Domains and members are objects of their own in TAVI, not concepts, so no + # fact can report against one: abstract is what the model says they are. for key in ("domainClasses", "members"): for entry in _sequence(xbrl_model.get(key)): qname = str(_mapping(entry).get("name", "")) if qname and qname not in concepts: - concepts[qname] = _bare(qname, namespaces, is_domain_member=True) + concepts[qname] = _bare( + qname, namespaces, is_abstract=True, is_domain_member=True + ) return concepts @@ -829,6 +834,10 @@ def _facts( numeric_value = _float(value_str) if unit is not None else None decimals = value_obj.get("decimals") + if decimals is None and unit is not None and value_str is not None: + # Absent on a numeric fact means infinitely precise (the fact value + # object's `decimals`), which is how the emitter writes INF. + decimals = "INF" language = dimensions.get("xbrl:language") facts.append( XbrlFact( diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index 0f43816..532c1d9 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -259,14 +259,21 @@ def _enrich_filer(loaded: LoadedFiling, fields: Mapping[str, Any]) -> None: @dataclass class PublishedFiling: - """A filing as the public data CDN lists it: the holon to load, the document - as filed when it was published beside it, and the filer's ticker — the key - the catalog holds that filer's identity under.""" + """A filing as the public data CDN lists it: the representations to load + (the TAVI model first, the holon beside it), the document as filed when it + was published too, and the filer's ticker — the key the catalog holds that + filer's identity under.""" accession: str - holon_url: str + holon_url: str | None = None document_url: str | None = None ticker: str | None = None + tavi_url: str | None = None + + @property + def model_urls(self) -> list[str]: + """The representations to try, in order: the TAVI model, then the holon.""" + return [url for url in (self.tavi_url, self.holon_url) if url] def _published_from( @@ -276,8 +283,8 @@ def _published_from( ticker: str | None = None, ) -> PublishedFiling | None: """The published filing a catalog entry or manifest describes, or ``None`` - when it lists no holon.""" - holon = document = None + when it lists neither a TAVI model nor a holon.""" + holon = document = tavi = None for rep in representations if isinstance(representations, list) else []: if not isinstance(rep, dict): continue @@ -288,15 +295,18 @@ def _published_from( continue if rep.get("kind") == "holon": holon = url + elif rep.get("kind") == "tavi": + tavi = url elif rep.get("kind") == "document": document = url - if not holon: + if not (holon or tavi): return None return PublishedFiling( accession=accession or "", holon_url=holon, document_url=document, ticker=ticker, + tavi_url=tavi, ) @@ -998,19 +1008,31 @@ def _filer_from_edgar(self, cik: str, client: Any = None) -> dict[str, Any]: } def _load_published(self, published: PublishedFiling, source: str) -> LoadedFiling: - """The filing from its published holon, with the document as filed beside - it when the CDN has that too — the same shape an EDGAR load gives, in a - fraction of the time and with no Arelle.""" - into = self._tmp / (published.accession or Path(published.holon_url).stem) - holon = self._fetch(published.holon_url, into=into) + """The filing from its published TAVI model (or its holon when there is no + TAVI, or it cannot be read), with the document as filed beside it when the + CDN has that too — the same shape an EDGAR load gives, in a fraction of the + time and with no Arelle. The TAVI carries its text blocks inline, so there + are no fragments to fetch after it.""" + urls = published.model_urls + into = self._tmp / (published.accession or Path(urls[0].split("?", 1)[0]).stem) document: Path | None = None if published.document_url: try: document = self._fetch(published.document_url, into=into) except requests.RequestException as exc: logger.warning("published document unavailable for %s: %s", source, exc) - logger.info("loading %s from its published holon", source) - loaded = self._load_json(holon, source, document=document) + loaded: LoadedFiling | None = None + for position, url in enumerate(urls): + try: + path = self._fetch(url, into=into) + logger.info("loading %s from %s", source, Path(url.split("?", 1)[0]).name) + loaded = self._load_json(path, source, document=document) + break + except (requests.RequestException, SourceError) as exc: + if position == len(urls) - 1: + raise + logger.warning("%s unavailable for %s (%s); trying the next", url, source, exc) + assert loaded is not None _enrich_filer( loaded, self._filer_metadata(loaded.model.entity.cik, ticker=published.ticker) )