From f8911143a5a60ff0cf120abdc0bf52b8ad01a051 Mon Sep 17 00:00:00 2001 From: "Joseph T. French" Date: Thu, 24 Sep 2026 17:55:59 -0500 Subject: [PATCH] feat(tavi): rebuild definition networks from cubes on read The TAVI emitter writes the definition linkbase as cube objects, and the reader skipped them, so a TAVI-loaded filing's information blocks had no hypercubes: on JPM's FY2025 10-K, 0 against the holon's 140. _definition_networks walks each cube back into all, hypercube-dimension, dimension-domain, domain-member and dimension-default arcs, one definition network per role. A cube does not name its hypercube element, its primary items or an axis's default member, so the reader takes the table from the role's own presentation tree (a table no role presents keeps the cube's name rather than a guess), the primary items from that table's other children, and an optional axis's default from its domain. Measured against the holon: JPM 136/140 hypercubes, BAC 112/116, with every axis, domain and member set identical. What remains is exactly what the cube cannot carry: tables the filer presents under other names, tables with no axes, and defaults that are not the domain. --- tests/test_deserialize.py | 104 +++++++++++++++++--- xbrlkit/deserialize/tavi.py | 188 +++++++++++++++++++++++++++++++++--- 2 files changed, 264 insertions(+), 28 deletions(-) diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py index 502547d..33b1a4e 100644 --- a/tests/test_deserialize.py +++ b/tests/test_deserialize.py @@ -695,32 +695,104 @@ def test_round_trip_keeps_the_concept_facts(model: XbrlModel, fmt: str) -> None: # -- what does not, and is reported rather than invented ------------------------- -def test_tavi_loses_the_definition_networks_the_holon_keeps( +def test_tavi_rebuilds_the_definition_network_from_its_cube( through_tavi: XbrlModel, through_holon: XbrlModel ) -> None: - """TAVI turns the dimensional wiring into cube objects, which do not come - back as arcs. The holon writes it as associations of its own kind, so the - definition networks survive there — and the axes and members survive both.""" - assert not [n for n in through_tavi.networks if n.kind == "definition"] - definition = [n for n in through_holon.networks if n.kind == "definition"] - assert [(a.from_qname, a.to_qname) for a in definition[0].arcs] == [ - ("us-gaap:AssetsAbstract", "us-gaap:SegmentTable"), - ("us-gaap:SegmentTable", "us-gaap:SegmentAxis"), - ("us-gaap:SegmentAxis", "us-gaap:SegmentDomain"), - ("us-gaap:SegmentDomain", "us-gaap:NorthAmerica"), - ] + """TAVI turns the dimensional wiring into cube objects; the reader walks them + back into a definition network, so the axes, the domain and the members + (followed across the ``targetRole`` hop) read as the holon's do. This + fixture's role presents no table, so the hypercube keeps the cube's name.""" + holon = [n for n in through_holon.networks if n.kind == "definition"] + tavi = [n for n in through_tavi.networks if n.kind == "definition"] + assert [n.role_uri for n in tavi] == [n.role_uri for n in holon] + by_arcrole = {a.arcrole.rsplit("/", 1)[-1]: a for a in tavi[0].arcs} + assert by_arcrole["hypercube-dimension"].to_qname == "us-gaap:SegmentAxis" + assert ( + by_arcrole["dimension-domain"].from_qname, + by_arcrole["dimension-domain"].to_qname, + ) == ("us-gaap:SegmentAxis", "us-gaap:SegmentDomain") + assert ( + by_arcrole["domain-member"].from_qname, + by_arcrole["domain-member"].to_qname, + ) == ("us-gaap:SegmentDomain", "us-gaap:NorthAmerica") + assert by_arcrole["all"].to_qname.startswith("rpt:cube-") for got in (through_tavi, through_holon): assert got.concepts["us-gaap:SegmentAxis"].is_dimension_item is True assert got.concepts["us-gaap:NorthAmerica"].is_domain_member is True - # Only the holon says a hypercube is one; TAVI has no flag for it. - assert through_holon.concepts["us-gaap:SegmentTable"].is_hypercube_item is True - assert through_tavi.concepts["us-gaap:SegmentTable"].is_hypercube_item is False + + +def _presented_segment_role(model: XbrlModel, *, defaulted: bool) -> XbrlModel: + """The fixture's segment role with the table presented, as EDGAR files it.""" + role = "http://example.com/role/Segments" + model.networks.append( + Network( + role_uri=role, + definition="Segments", + kind="presentation", + arcs=[ + Arc( + from_qname="us-gaap:AssetsAbstract", + to_qname="us-gaap:SegmentTable", + arcrole=PARENT_CHILD, + order=1.0, + is_root=True, + ), + Arc( + from_qname="us-gaap:SegmentTable", + to_qname="us-gaap:SegmentAxis", + arcrole=PARENT_CHILD, + order=1.0, + ), + Arc( + from_qname="us-gaap:SegmentTable", + to_qname="us-gaap:SegmentLineItems", + arcrole=PARENT_CHILD, + order=2.0, + ), + ], + ) + ) + if defaulted: + definition = next(n for n in model.networks if n.kind == "definition") + definition.arcs.append( + Arc( + from_qname="us-gaap:SegmentAxis", + to_qname="us-gaap:SegmentDomain", + arcrole=f"{DIM}/dimension-default", + ) + ) + return model + + +def test_tavi_names_the_table_its_role_presents(model: XbrlModel) -> None: + from xbrlkit.information_block import plan_blocks + + got = from_tavi_json(to_tavi(_presented_segment_role(model, defaulted=False))) + segments = next(b for b in plan_blocks(got) if b.role_uri.endswith("/Segments")) + (cube,) = segments.hypercubes + assert cube.qname == "us-gaap:SegmentTable" + assert cube.primary_items == ["us-gaap:SegmentLineItems"] + assert [a.qname for a in cube.axes] == ["us-gaap:SegmentAxis"] + assert cube.axes[0].members == ["us-gaap:NorthAmerica"] + assert cube.axes[0].default is None + assert got.concepts["us-gaap:SegmentTable"].is_hypercube_item is True + + +def test_tavi_reads_an_optional_axis_as_defaulting_to_its_domain( + model: XbrlModel, +) -> None: + """The cube records only that an axis may be omitted, not the default member; + the domain is the EDGAR convention, and the one the reader supplies.""" + from xbrlkit.information_block import plan_blocks + + got = from_tavi_json(to_tavi(_presented_segment_role(model, defaulted=True))) + segments = next(b for b in plan_blocks(got) if b.role_uri.endswith("/Segments")) + assert segments.hypercubes[0].axes[0].default == "us-gaap:SegmentDomain" def test_tavi_gaps_are_declared(model: XbrlModel) -> None: _, gaps = from_tavi_report(to_tavi(model)) reported = " ".join(gaps.missing) - assert "is_hypercube_item" in reported assert "source_hash" in reported assert gaps.unmapped_datatypes == {} assert gaps.unmapped_label_types == {} diff --git a/xbrlkit/deserialize/tavi.py b/xbrlkit/deserialize/tavi.py index a51866e..895a288 100644 --- a/xbrlkit/deserialize/tavi.py +++ b/xbrlkit/deserialize/tavi.py @@ -11,16 +11,20 @@ against this model — facts with their values, decimals, language and dimensions; concepts with their datatype, period type, balance and nillable flag; every label role; presentation and calculation networks with order, weight and -preferred label; the extended link roles as groups. Four things have no home in -it and are therefore *not* reconstructed here, because inventing them would make -the importer's output disagree with the parse it claims to reproduce: +preferred label; the extended link roles as groups. The definition linkbase is +written as cubes, and :func:`_definition_networks` walks them back into arcs, +reading the three things a cube does not name (the hypercube element, its +primary items, an axis's default member) from the report's own structure. Four +things have no home in it and are therefore *not* reconstructed here, because +inventing them would make the importer's output disagree with the parse it +claims to reproduce: - the derived period semantics (duration bucket, calendar placement) — the one exception, recomputed by :mod:`xbrlkit.periods` from the dates themselves, which is where the parse gets them too; -- ``is_hypercube_item`` and the abstractness of axes, domains and members: the - emitter turns those elements into dimensional objects, and TAVI has no flag - for either; +- the abstractness of axes, domains and members: the emitter turns those + elements into dimensional objects, and TAVI has no flag for it (a hypercube + is marked ``is_hypercube_item`` when its role presents the table); - reference linkbase entries, ``Network.role_id``, a fact's source hash and raw lexical value, and a dimension's segment/scenario axis; - a fact's own entity when it differs from the report's — the emitter writes @@ -59,8 +63,14 @@ from ..serialize._values import CIK_SCHEME from ..serialize.tavi import ( CALCULATION_RELATIONSHIP, + DIM_ALL, + DIM_DIMENSION_DEFAULT, + DIM_DIMENSION_DOMAIN, + DIM_DOMAIN_MEMBER, + DIM_HYPERCUBE_DIMENSION, ITEM_TYPE_DATATYPES, LABEL_ROLE_TYPES, + OPTIONAL_CORE_DIMENSIONS, PRESENTATION_RELATIONSHIP, ROOT_SOURCE, ) @@ -190,8 +200,9 @@ def _read(document: Mapping[str, Any]) -> tuple[XbrlModel, ImportGaps]: } gaps = ImportGaps( missing=[ - "is_hypercube_item", "abstract flag on axes, domains and members", + "a hypercube's name and primary items where its role presents no table", + "an axis's default member, read as its domain", "concept references", "network role_id", "fact source_hash and raw_value", @@ -204,6 +215,7 @@ def _read(document: Mapping[str, Any]) -> tuple[XbrlModel, ImportGaps]: concepts = _concepts(xbrl_model, namespaces, gaps) _apply_labels(xbrl_model, concepts, entity, _entity_sqname(xbrl_model), gaps) networks = _networks(xbrl_model) + networks.extend(_definition_networks(xbrl_model, networks, concepts)) facts, periods, units = _facts(xbrl_model, concepts, entity, namespaces, gaps) _mark_text_facts(concepts, facts) filing = _filing(document, xbrl_model, namespaces, entity, facts, concepts) @@ -492,8 +504,17 @@ def _apply_labels( # -- networks ------------------------------------------------------------------- -def _networks(xbrl_model: Mapping[str, Any]) -> list[Network]: - """Networks, rejoined to the extended link roles their groups stand for.""" +@dataclass +class _Roles: + """What a group stands for: its extended link role, and that role's texts.""" + + by_object: dict[str, str] + definitions: dict[str, str] + documentations: dict[str, str] + + +def _roles(xbrl_model: Mapping[str, Any]) -> _Roles: + """Each network or cube → the extended link role its group stands for.""" group_roles: dict[str, str] = {} for entry in _sequence(xbrl_model.get("groups")): group = _mapping(entry) @@ -521,12 +542,17 @@ def _networks(xbrl_model: Mapping[str, Any]) -> list[Network]: target = content.get("forObject") if role_uri and isinstance(target, str): network_roles[target] = role_uri + return _Roles(network_roles, definitions, documentations) + +def _networks(xbrl_model: Mapping[str, Any]) -> list[Network]: + """Networks, rejoined to the extended link roles their groups stand for.""" + roles = _roles(xbrl_model) networks: list[Network] = [] for entry in _sequence(xbrl_model.get("networks")): obj = _mapping(entry) name = str(obj.get("name", "")) - role_uri = network_roles.get(name) + role_uri = roles.by_object.get(name) if role_uri is None: continue kind: NetworkKind = ( @@ -543,8 +569,8 @@ def _networks(xbrl_model: Mapping[str, Any]) -> list[Network]: networks.append( Network( role_uri=role_uri, - definition=definitions.get(role_uri), - documentation=documentations.get(role_uri), + definition=roles.definitions.get(role_uri), + documentation=roles.documentations.get(role_uri), kind=kind, arcs=_arcs(_sequence(obj.get("relationships")), kind), ) @@ -552,6 +578,144 @@ def _networks(xbrl_model: Mapping[str, Any]) -> list[Network]: return networks +# The dimensions every reconstructed cube declares that are not taxonomy axes. +_CORE_DIMENSIONS = frozenset(("xbrl:concept", *OPTIONAL_CORE_DIMENSIONS)) + + +def _definition_networks( + xbrl_model: Mapping[str, Any], + networks: Sequence[Network], + concepts: dict[str, Concept], +) -> list[Network]: + """The definition linkbase, rebuilt from the cubes that replaced it. + + The emitter turns each (role, hypercube) into a cube whose taxonomy + dimensions carry a domain network and an ``optional`` flag, and drops the + definition arcs. This walks the other way, one definition network per role, + so :func:`xbrlkit.information_block.build_hypercubes` reads a TAVI-loaded + filing's breakdowns as it reads a parsed one. Three things the cube does not + record are read back from the report's own structure: + + - the hypercube element is the presentation parent of the cube's axes in its + role, which is where EDGAR filings place their tables; + - the primary items are that table's other presentation children, the line + items the ``all`` arc hangs the cube on; + - an optional axis defaults to its domain, the EDGAR convention. + + Where the presentation tree does not supply a table, the cube's own name + stands in, so the axes, domains and members still read back. + """ + roles = _roles(xbrl_model) + domain_networks = { + str(_mapping(entry).get("name", "")): _mapping(entry) + for entry in _sequence(xbrl_model.get("domainNetworks")) + } + parents: dict[str, dict[str, list[str]]] = {} + children: dict[str, dict[str, list[str]]] = {} + for network in networks: + if network.kind != "presentation": + continue + role_parents = parents.setdefault(network.role_uri, {}) + role_children = children.setdefault(network.role_uri, {}) + for arc in network.arcs: + role_parents.setdefault(arc.to_qname, []).append(arc.from_qname) + role_children.setdefault(arc.from_qname, []).append(arc.to_qname) + claimed_tables: set[tuple[str, str]] = set() + + arcs_by_role: dict[str, list[Arc]] = {} + for entry in _sequence(xbrl_model.get("cubes")): + cube = _mapping(entry) + name = str(cube.get("name", "")) + role_uri = roles.by_object.get(name) + if role_uri is None: + continue + axes = [ + _mapping(d) + for d in _sequence(cube.get("cubeDimensions")) + if str(_mapping(d).get("dimension", "")) not in _CORE_DIMENSIONS + ] + if not axes: + continue + axis_names = [str(axis["dimension"]) for axis in axes] + # Only the cube's own role is searched: a table another role presents is a + # guess, and a wrong name hides the wrong row. Two cubes that would claim + # one table keep their own names instead. + hypercube = _table_of(axis_names, parents.get(role_uri, {})) + if hypercube is None or (role_uri, hypercube) in claimed_tables: + hypercube = name + claimed_tables.add((role_uri, hypercube)) + primary_items = sorted( + { + child + for child in children.get(role_uri, {}).get(hypercube, []) + if child not in axis_names + } + ) or [hypercube] + if hypercube in concepts: + concepts[hypercube].is_hypercube_item = True + + arcs = arcs_by_role.setdefault(role_uri, []) + for primary in primary_items: + arcs.append(Arc(from_qname=primary, to_qname=hypercube, arcrole=DIM_ALL)) + for order, axis in enumerate(axes, start=1): + axis_name = str(axis["dimension"]) + arcs.append( + Arc( + from_qname=hypercube, + to_qname=axis_name, + arcrole=DIM_HYPERCUBE_DIMENSION, + order=float(order), + ) + ) + domain_network = domain_networks.get(str(axis.get("domainNetwork", ""))) + root = domain_network.get("root") if domain_network else None + if not isinstance(root, str): + continue # a typed axis: no domain element to walk + arcs.append( + Arc(from_qname=axis_name, to_qname=root, arcrole=DIM_DIMENSION_DOMAIN) + ) + for position, entry in enumerate( + _sequence(domain_network.get("relationships")), 1 + ): + relationship = _mapping(entry) + source, target = relationship.get("source"), relationship.get("target") + if isinstance(source, str) and isinstance(target, str): + arcs.append( + Arc( + from_qname=source, + to_qname=target, + arcrole=DIM_DOMAIN_MEMBER, + order=float(position), + ) + ) + if axis.get("optional") is True: + arcs.append( + Arc(from_qname=axis_name, to_qname=root, arcrole=DIM_DIMENSION_DEFAULT) + ) + + return [ + Network( + role_uri=role_uri, + definition=roles.definitions.get(role_uri), + documentation=roles.documentations.get(role_uri), + kind="definition", + arcs=arcs, + ) + for role_uri, arcs in arcs_by_role.items() + ] + + +def _table_of(axes: Sequence[str], parents: Mapping[str, list[str]]) -> str | None: + """The one presentation parent every axis of a cube shares, if there is one.""" + common: set[str] | None = None + for axis in axes: + found = set(parents.get(axis, [])) + common = found if common is None else common & found + if not common: + return None + return sorted(common)[0] + + def _arcs(relationships: Sequence[Any], kind: NetworkKind) -> list[Arc]: """Relationships as arcs, with the roots the virtual root source declares.""" roots: set[str] = set()