diff --git a/example_notebooks/colab_example_notebook.ipynb b/example_notebooks/colab_example_notebook.ipynb index 70d5c98c9..1ced3c124 100644 --- a/example_notebooks/colab_example_notebook.ipynb +++ b/example_notebooks/colab_example_notebook.ipynb @@ -466,6 +466,7 @@ " ],\n", ")\n", "graph.validate()\n", + "display(graph.visualize())\n", "print('Graph ID:', graph.id)\n", "print('Tables:', list(graph._named_tables))\n", "print('Edges:', len(graph.edges))\n", diff --git a/example_notebooks/databricks_example_notebook.ipynb b/example_notebooks/databricks_example_notebook.ipynb index 727527969..0ade7e371 100644 --- a/example_notebooks/databricks_example_notebook.ipynb +++ b/example_notebooks/databricks_example_notebook.ipynb @@ -617,6 +617,7 @@ ")\n", "\n", "graph.validate()\n", + "display(graph.visualize())\n", "print(\"graph id:\", graph.id)" ] }, diff --git a/example_notebooks/snowflake_example_notebook.ipynb b/example_notebooks/snowflake_example_notebook.ipynb index 64679ccf7..bb676e88a 100644 --- a/example_notebooks/snowflake_example_notebook.ipynb +++ b/example_notebooks/snowflake_example_notebook.ipynb @@ -187,7 +187,7 @@ "language": "python" }, "outputs": [], - "source": "graph = kumo.Graph(\n tables=[customer, part, orders, lineitem],\n edges=[\n Edge(src_table=\"ORDERS\", fkey=\"O_CUSTKEY\", dst_table=\"CUSTOMER\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_ORDERKEY\", dst_table=\"ORDERS\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_PARTKEY\", dst_table=\"PART\"),\n ],\n)\n\ngraph.validate()\n\nsnapshot = graph.snapshot()\n\nprint(\"Snapshot:\", snapshot.id)" + "source": "from IPython.display import display\n\ngraph = kumo.Graph(\n tables=[customer, part, orders, lineitem],\n edges=[\n Edge(src_table=\"ORDERS\", fkey=\"O_CUSTKEY\", dst_table=\"CUSTOMER\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_ORDERKEY\", dst_table=\"ORDERS\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_PARTKEY\", dst_table=\"PART\"),\n ],\n)\n\ngraph.validate()\ndisplay(graph.visualize())\n\nsnapshot = graph.snapshot()\n\nprint(\"Snapshot:\", snapshot.id)" }, { "cell_type": "markdown", diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index b2a21bdf7..e425c3bbf 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -101,6 +101,126 @@ To use your own data, point `FileConnector` at a directory of CSV or Parquet fil `graph.snapshot()` ingests the tables into an on-disk graph index under the scratch directory, which also holds training runs, model bundles, and predictions. It defaults to `~/.cache/kumo-engine`; to put it on a large local disk, call `kumo.engine.configure(scratch_dir=...)` first. +## Visualize table relationships + +Display declared tables and relationships in a notebook, or export SVG, PDF, or Markdown: + +```python +graph.visualize() +graph.visualize("relationships.svg", show_cols=True) +graph.visualize("relationships.pdf", show_cols=True, layout="compact") +graph.visualize("relationships.md") +graph.visualize(layout="compact") +``` + +Visualization reads no source data by default and uses Kumo Engine's existing dependencies. +The return value is SVG text with notebook display support. +The default view includes primary keys, relationship foreign keys, and declared time (`TS`) and end-time (`TE`) columns. +`show_cols=True` adds other columns up to 10 rows per table, or 5 above 10 tables; key and time columns always remain visible. +Graphs are limited to 30 tables and require valid declared schemas without self-relations. + +`layout="auto"` chooses Standard or Compact from graph structure; either is also selectable in the notebook. +Drag to pan, use the controls or Ctrl/Cmd+wheel to zoom, and choose Fit to reset the view. +Hover or focus a header or column for details without moving the graph. +The default notebook view follows the browser’s light or dark appearance; SVG and PDF exports use the light palette. +Notebook display automatically uses the widget protocol in detected Snowflake Workspace notebooks; Databricks and other hosts use HTML. +Installing the Snowflake connector extra does not select the display protocol. +HTML interactivity requires trusted output; fullscreen also needs permission from the notebook host. +Saved HTML output includes the complete, sample-free SVG image in the requested initial layout and “Rerun this cell to activate animation.” +Rerun the visualization cell once after upgrading and save the notebook to store this fallback; VS Code Restricted Mode can refuse HTML/SVG output entirely. +The Snowflake Workspace view uses the environment's `anywidget>=0.9,<1` and `ipywidgets` packages, imported only when displaying the result. +Snowflake Workspace Runtime 2.6 and later supplies these packages, including the documented GPU Runtime 2.9 target; Runtime 2.5 and earlier requires installation according to your environment's package policy. +See [Snowflake's widget installation guidance](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-in-workspaces/notebooks-in-workspaces-interactive-widgets#installation). +Kumo Engine does not install these packages as base dependencies or connector extras. +If the widget packages are unavailable or incompatible, the Workspace view shows a static graph and a diagnostic; SVG, PDF, and Markdown exports remain available. +The optional `graph.visualize().as_widget()` helper explicitly selects the same widget view on compatible hosts. +The widget view uses the same controls, and widget restoration still depends on the notebook host. +It inherits a host color scheme declared when mounted, with browser appearance as the fallback. +Each displayed view has its own camera; redisplay after restarting the kernel. +SVG exports contain the initial arrangement and native tooltips. +The Export commands button reveals a selectable PDF save command followed by a Markdown save command. +Run the desired command with your graph variable to save the file in the notebook filesystem. +Opening the panel does not generate a PDF, start a download, or write a file. +The PDF includes a paginated index of all declared tables, columns, data types, key/time roles, and connections, including columns omitted from the diagram. +Passing a `.pdf` path to `visualize` saves the same document on the filesystem where Kumo Engine runs, using the requested initial layout; it overwrites an existing file. +PDF export uses Matplotlib and its embedded DejaVu fonts; unsupported characters are shown as explicit Unicode escapes. +For example, `_ = graph.visualize("graph.pdf", layout="standard")` saves a PDF in the notebook's working directory; use `layout="compact"` for Compact. +Use `_ = graph.visualize("graph.md")` to save Markdown. +The assignment suppresses automatic display of the returned SVG. +In Snowflake Workspaces, retrieve the resulting file through the workspace file browser; saving requires write access to the directory. +This writes a notebook-side file and does not start a browser download. + +Passing a `.md` or `.markdown` path writes a Mermaid relationship diagram and Markdown tables containing every declared column, data type, key/time role, and outgoing connection. +The file is saved on the machine running Kumo Engine and overwrites an existing file. +Mermaid diagrams render on GitHub and in compatible Markdown viewers, using the viewer's layout rather than the notebook arrangement. +The schema tables are readable without Mermaid support. + +`show_samples=True` shows up to six non-null examples per visible column in two columns and up to three rows, sampled independently from up to 30 preview rows per table. +The default `show_samples=False` hides samples and reads no source data. +Previews can run warehouse queries and incur costs; the row limit does not bound bytes scanned. +Unavailable or empty previews are labeled. +Sample values are saved in notebook HTML or widget state, but never included in SVG, PDF, or Markdown exports. + +### Developing another notebook widget + +Use the private `_widget` helpers to define one feature specification for both notebook transports. +The shared shell supplies theme tokens, controls, view identity, and disposal; keep feature behavior inside its initializer. +This chart uses the same specification with trusted HTML and with real widget MIME: + +```python +from IPython.display import HTML, display +from kumo.engine._widget import ( + CSS_THEME, button, render_anywidget, render_widget, +) + +chart = dict( + class_name="kumo-contribution", + body=( + '
' + + button("chart-toggle", "Toggle contribution", text="↔") + + '
' + ), + style=f'.kumo-contribution rect{{fill:{CSS_THEME["accent"]}}}', + data={"fraction": 0.75}, + script=""" +const svg=host.querySelector('svg'),bar=svg.querySelector('rect'); +const toggle=host.querySelector('.chart-toggle'); +let fraction=data.fraction; +const update=()=>{bar.setAttribute('width',300*fraction); +svg.setAttribute('aria-label','Contribution: '+Math.round(100*fraction)+'%')}; +toggle.addEventListener('click',()=>{fraction=1-fraction;update()},{signal}); +update();toggle.disabled=false; +""", +) +display(HTML(render_widget(**chart))) +display(render_anywidget(**chart)) # uses environment-provided widget packages +``` + +`render_anywidget` lazily imports the optional dependency and provides an AFM `render({model, el})` entry point. +It transports markup and structured `data` in model state; executable feature code stays in the module. +Each view initializes its own closure, so repeated displays of one model have independent controls and camera state. +The specification is a snapshot; updating Python dictionaries does not update existing views. + +- Escape user text in markup and use `data` for values; never interpret model state as code or include executable scripts in `body`. + Keep frontend code self-contained, without CDN imports, `eval`, or `new Function`. +- Initializers receive `host`, `widgetId`, `data`, `signal`, and `onDispose(callback)`. + Scope queries to `host`, attach listeners with `signal`, and enable controls after initialization. + Register timer/animation cancellation, observer disconnection, pointer release, and object-URL revocation with `onDispose`; the shell invokes cleanup on HTML removal or AFM disposal. +- Scope CSS under the feature's root class and use `CSS_THEME` for interactive colors. + Both shells retain `nodarkmode`; the widget transport inherits a declared host color scheme available at mount and otherwise follows browser appearance. + Redisplay to inherit a scheme that becomes available only after mounting. +- Use `widgetId` for IDs created by the initializer; for IDs and references already in SVG markup, pass their shared unique prefix as `namespace` so each view rewrites it independently. +- Keep static export serialization in the feature, using the resolved `THEME` palette for light exports. + Notebook-only sample values may enter saved HTML or widget state, but must stay out of static exports. +- HTML needs a trusted execution surface; anywidget needs a compatible active notebook host. + Request fullscreen directly from a user action and handle rejection; use `host.matches(":fullscreen")` to recognize the actual fullscreen element inside Shadow DOM. + Fullscreen remains subject to host permissions, and widget support alone does not prove that capability. + Show explicit notebook-file export commands with accurate destinations; keep file generation outside display initialization. + +AFM initializes once attached; temporary host detachment preserves the view until the host calls its disposer. +Disposed views must be rendered again. + ## Documentation - [ADLS connector guide](https://github.com/NVIDIA/kumo-engine/blob/main/packages/kumo-adls/README.md): setup, authentication, supported reads and exports. diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py new file mode 100644 index 000000000..9e75198ab --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -0,0 +1,568 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""SVG rendering of declared table relationships.""" + +import json +import re +from datetime import date +from html import escape +from reprlib import Repr +from typing import Mapping, Sequence +from unicodedata import combining +from uuid import uuid4 +from xml.etree import ElementTree + +from kumoapi.graph import Edge, validate_edge_endpoints +from kumoapi.source_table import CustomSQLSourceTable +from kumoapi.table import TableDefinition + +from kumo.engine import _widget +from kumo.engine._widget import THEME, render_anywidget, render_widget +from kumo.engine.table import Table + +_COLUMN_LIMIT = 10 +_LARGE_GRAPH_COLUMN_LIMIT = 5 + + +class SVG(str): + """SVG text with optional notebook-only layout controls and previews.""" + + def __new__(cls, value: str, samples: dict[str, str] | None = None, + namespace: str = '', layouts: dict | None = None, + initial_layout: str = 'standard', index: list | None = None): + obj = super().__new__(cls, value) + obj._samples = samples or {} + obj._namespace = namespace + obj._layouts = layouts or {} + obj._initial_layout = initial_layout + obj._index = index or [] + obj._pdfs = {} + obj._widget = None + return obj + + def _repr_svg_(self) -> str: + return str(self) + + def _pdf(self, layout: str) -> bytes: + from kumo.engine._visualization_pdf import render_pdf + + if layout not in self._pdfs: + self._pdfs[layout] = render_pdf(self, self._layouts[layout], + self._index) + return self._pdfs[layout] + + def _markdown(self) -> str: + def label(value, mermaid=False): + value = value.replace('\r\n', '\n').replace('\r', '\n') + if mermaid: + return ''.join( + '
' if char == '\n' else + f'#{ord(char)};' if char in '#&"<>\\`|' else char + for char in value) + return re.sub(r'([\\`*_\[\]|#.@:])', r'\\\1', + escape(value, quote=False)).replace('\n', '
') + + lines = ['# Table relationships', ''] + if not self._index: + return '\n'.join(lines + ['No tables.', '']) + ids = {table['name']: f'table_{i}' + for i, table in enumerate(self._index)} + lines += ['Arrows point from foreign keys to referenced primary keys.', + 'Sample values are excluded.', '', '```mermaid', + 'flowchart LR'] + lines.extend(f' {ids[name]}["{label(name, True)}"]' for name in ids) + root = ElementTree.fromstring(self) + for edge in root.findall( + './/{http://www.w3.org/2000/svg}path[@class="kumo-edge"]'): + source = ids[edge.get('data-src-table')] + target = ids[edge.get('data-dst-table')] + columns = (f'{edge.get("data-src-column")} → ' + f'{edge.get("data-dst-column")}') + lines.append(f' {source} -->|"{label(columns, True)}"| {target}') + lines += ['```', '', '## Schema', '', + 'All declared columns are listed. PK: primary key; ' + 'FK: foreign key; TS: time column; TE: end-time column.', ''] + for table in self._index: + lines += [f'### {label(table["name"])}', '', + '| Column | Data type | Role |', '| --- | --- | --- |'] + for column in table['columns']: + cells = [column['name'], column['type'], + column['role'] or 'Column'] + lines.append('| ' + ' | '.join(map(label, cells)) + ' |') + if not table['columns']: + lines += ['', 'No columns.'] + lines += ['', '**Outgoing connections**', ''] + lines.extend(f'- {label(connection)}' for connection in + table['connections'] or ['No outgoing connections']) + lines.append('') + return '\n'.join(lines) + + def as_widget(self): + """Return a cached widget model with independent frontend views.""" + if self._widget is None or self._widget.comm is None: + self._widget = render_anywidget(**self._viewer()) + return self._widget + + def _ipython_display_(self): + if not _widget.is_snowflake_workspace(): + raise NotImplementedError + from IPython.display import display + + try: + widget = self.as_widget() + except ImportError as error: + display({'text/html': _widget.render_static_svg( + self, str(error), 'Table relationships'), + 'text/plain': str(error)}, raw=True) + else: + display(widget) + + def _repr_html_(self) -> str: + return render_widget(**self._viewer()) + + def _viewer(self) -> dict: + from kumo.engine._visualization_viewer import viewer + + svg = str(self) + for metadata, sampled in self._samples.items(): + svg = svg.replace(metadata, sampled) + return dict(viewer(svg, self._layouts, self._initial_layout, str(self)), + namespace=self._namespace) + + +def _wrap(text: str, width: int) -> list[str]: + lines, line, size = [], '', 0 + split, split_size = 0, 0 + for char in text: + code = ord(char) + if not (code in (9, 10, 13) or 32 <= code <= 0xD7FF + or 0xE000 <= code <= 0xFFFD or 0x10000 <= code <= 0x10FFFF): + raise ValueError('Table and column names must contain valid XML ' + 'characters to visualize them.') + char_size = 0 if combining(char) else ( + 2 if code > 127 or char.isupper() or char in 'mw@%&#' else 1) + while line and size + char_size > width: + if split: + lines.append(line[:split]) + line, size = line[split:], size - split_size + else: + lines.append(line) + line, size = '', 0 + split, split_size = 0, 0 + line += char + size += char_size + if char.isspace() or char in '_-': + split, split_size = len(line), size + return lines + [line] + + +def _squircle(width: float, height: float, radius: float = 18, + *, square_bottom: bool = False) -> str: + # Cubic corners flatten at the sides, unlike circular rounded rectangles. + r = min(radius, width / 2, height / 2) + bottom = (f'V {height} H 0 ' if square_bottom else + f'V {height - r} C {width} {height} {width} {height} ' + f'{width - r} {height} H {r} C 0 {height} 0 {height} ' + f'0 {height - r} ') + return (f'M {r} 0 H {width - r} C {width} 0 {width} 0 {width} {r} ' + f'{bottom}V {r} C 0 0 0 0 {r} 0 Z') + + +def _attribute(value: str) -> str: + return (escape(value).replace('\t', ' ').replace('\n', ' ') + .replace('\r', ' ')) + + +def _tooltip(text: str, row: str = '', table: str = '', + column: str = '', + connected_tables: Sequence[str] = ()) -> str: + connections = escape(json.dumps(list(connected_tables))) + attr = _attribute(text) + text = escape(text) + table = _attribute(table) + column = _attribute(column) + return (f'{text}') + + +def _preview(table: Table, columns: Sequence[str]) -> dict[str, str]: + import pandas as pd + from kumo_core.sources.preview import sample_dataframe + + fallback = dict.fromkeys(columns, '\nSample values: unavailable') + try: + assert table.definition is not None + source = table.definition.source_table + sql = source.sql if isinstance(source, CustomSQLSourceTable) else None + response = table.connector.source_backend.get_sample( + table.connector, table.table_name, 30, sql=sql, timeout=10) + frame = sample_dataframe(response.sample_rows) + if frame is None: + return fallback + short = Repr() + short.maxstring, short.maxlist, short.maxtuple = 120, 5, 5 + for column in columns: + if column not in frame: + continue + values = frame[column].iloc[:30].tolist() + if not values: + fallback[column] = '\nSample values: no rows returned' + continue + labels = [] + for value in values: + null = (value is None or value is pd.NA or value is pd.NaT + or isinstance(value, float) and value != value) + if null: + continue + label = (str(value) if isinstance(value, date) else + short.repr(value)) + labels.append(label if len(label) <= 160 else label[:157] + + '...') + if len(labels) == 6: + break + fallback[column] = ( + '\nSample values:\n' + '\n'.join(labels) if labels else + '\nSample values: No non-null values in preview') + except Exception: + # Preview failures must not hide the schema or expose source secrets. + pass + return fallback + + +def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, + show_cols: bool, show_samples: bool = False, layout: str = 'auto') -> SVG: + from kumo.engine._visualization_layout import build_layouts, choose_layout + + if not isinstance(layout, str) or layout not in ( + 'auto', 'standard', 'compact'): + raise ValueError('layout must be auto, standard, or compact.') + if type(show_samples) is not bool: + raise ValueError('show_samples must be True or False.') + if len(tables) > 30: + raise ValueError('Graph visualization supports at most 30 tables; ' + 'select a smaller graph to visualize.') + validate_edge_endpoints(tables, edges) + names = sorted(tables) + edges = sorted(edges, key=lambda e: (e.src_table, e.fkey, e.dst_table)) + definitions: dict[str, TableDefinition] = {} + keys = {name: {} for name in names} + neighbors: dict[str, set[str]] = {name: set() for name in names} + for name in names: + definition = tables[name].definition + if definition is None: + raise ValueError( + f'Table {name!r} has no definition; infer metadata ' + 'or supply its definition before visualizing.') + definitions[name] = definition + columns = [col.name for col in definition.cols] + if len(columns) != len(set(columns)): + raise ValueError(f'Table {name!r} has duplicate column names; ' + 'give each column a unique name.') + if definition.pkey is not None: + if definition.pkey not in columns: + raise ValueError(f'Primary key {name}.{definition.pkey} is ' + 'missing from the table columns.') + keys[name][definition.pkey] = 'PK' + for edge in edges: + src, dst = edge.src_table, edge.dst_table + if src == dst: + raise ValueError(f'Self-relation {src}.{edge.fkey} -> {dst} cannot ' + 'be visualized; self-relations are not supported ' + 'by graph snapshots. Remove this edge first.') + if edge.fkey not in {col.name for col in definitions[src].cols}: + raise ValueError(f'Foreign key {src}.{edge.fkey} is missing ' + 'from the table columns.') + if definitions[dst].pkey is None: + raise ValueError(f'Destination table {dst!r} has no primary key; ' + 'declare one before visualizing this edge.') + keys[src][edge.fkey] = ('PK/FK' + if definitions[src].pkey == edge.fkey else 'FK') + neighbors[src].add(dst) + neighbors[dst].add(src) + + for name, definition in definitions.items(): + for column, marker, label in ( + (definition.time_col, 'TS', 'Time'), + (definition.end_time_col, 'TE', 'End-time')): + if column is not None: + if column not in {col.name for col in definition.cols}: + raise ValueError(f'{label} column {name}.{column} is ' + 'missing from the table columns.') + role = keys[name].get(column, '') + keys[name][column] = f'{role}/{marker}' if role else marker + + cards = {} + for name in names: + definition = definitions[name] + pkey = definition.pkey + columns = sorted((col.name for col in definition.cols), key=lambda col: + (col != pkey, 'FK' not in keys[name].get(col, ''), + col not in keys[name], col)) + limit = (_COLUMN_LIMIT + if len(names) <= 10 else _LARGE_GRAPH_COLUMN_LIMIT) + columns = columns[:max(len(keys[name]), limit if show_cols else 0)] + title = _wrap(name, 27) + header = 20 + 22 * len(title) + rows = [] + ports = {} + y = header + 8 + for column in columns: + # Reserve extra room when a field carries multiple role markers. + lines = _wrap(column, 23 if len(keys[name].get(column, '')) > 5 + else 25) + height = 12 + 20 * len(lines) + ports[column] = y + height / 2 + rows.append((column, lines, y, height)) + y += height + hidden = len(definition.cols) - len(columns) + footer = (f'+ {hidden} other column' + + ('s' if hidden != 1 else '') if hidden else + ('No columns' if not columns else '')) + height = y + (26 if footer else 4) + cards[name] = (title, header, rows, ports, footer, height) + + # Rank primary-key tables before their dependents. + parents = {name: set() for name in names} + for edge in edges: + parents[edge.src_table].add(edge.dst_table) + ranks: dict[str, int] = {} + pending = set(names) + while pending: + ready = [name for name in pending if not parents[name] & pending] + name = min(ready or pending, key=lambda n: (-len(neighbors[n]), n)) + ranks[name] = max( + (ranks[parent] + 1 for parent in parents[name] if parent in ranks), + default=0) + pending.remove(name) + count = max(ranks.values(), default=0) + 1 + layers = [[] for _ in range(count)] + heights = [cards[name][-1] for name in names] + for node, name in enumerate(names): + layers[ranks[name]].append(node) + node_ids = {name: node for node, name in enumerate(names)} + routes = [] + adjacent: dict[int, list[int]] = {node: [] for node in range(len(names))} + for edge in edges: + source, target = ranks[edge.dst_table], ranks[edge.src_table] + route = [node_ids[edge.dst_table]] + # Empty slots carry long edges through a rank without crossing a card. + for rank in range(source + 1, target): + node = len(heights) + heights.append(16) + layers[rank].append(node) + adjacent[node] = [] + route.append(node) + route.append(node_ids[edge.src_table]) + routes.append(route) + if source < target: + for a, b in zip(route, route[1:]): + adjacent[a].append(b) + adjacent[b].append(a) + centers = { + node: index + for layer in layers + for index, node in enumerate(layer) + } + for sweep in (list(range(count)), list(reversed(range(count)))) * 2: + for rank in sweep: + layers[rank].sort( + key=lambda node: (sum(centers[n] for n in adjacent[node]) / len( + adjacent[node]) if adjacent[node] else centers[node], node)) + offset = 0 + for node in layers[rank]: + centers[node] = offset + heights[node] / 2 + offset += heights[node] + 50 + layer_heights = [ + sum(heights[node] for node in layer) + max(0, + len(layer) - 1) * 50 + for layer in layers + ] + feedback = sum(ranks[e.dst_table] >= ranks[e.src_table] for e in edges) + card_width, gap = 304, max(120, 60 + 8 * feedback) + margin = max(80, 32 + 6 * feedback) + top = 40 + 24 * feedback + canvas_height = max(layer_heights, default=90) + xy = {} + for rank, layer in enumerate(layers): + y = top + (canvas_height - layer_heights[rank]) / 2 + for node in layer: + xy[node] = (margin + rank * (card_width + gap), y) + y += heights[node] + 50 + width = count * card_width + (count - 1) * gap + 2 * margin + height = top + canvas_height + 40 + if not names: + width, height = 304, 112 + standard_paths = [] + lane = 0 + handles = set() + for index, edge in enumerate(edges): + route = routes[index] + sx, sy = xy[route[0]] + dx, dy = xy[route[-1]] + sx += card_width + sy += cards[edge.dst_table][3][definitions[edge.dst_table].pkey] + dy += cards[edge.src_table][3][edge.fkey] + path = f'M {sx} {sy}' + if ranks[edge.dst_table] < ranks[edge.src_table]: + x, y = sx, sy + for node in route[1:]: + nx, ny = (dx, dy) if node == route[-1] else (xy[node][0], + xy[node][1] + + heights[node] / 2) + # Use horizontal Bézier controls at half the column gap. + half = (nx - x) / 2 + path += f' C {x + half} {y} {nx - half} {ny} {nx} {ny}' + x, y = nx, ny + if node != route[-1]: + path += (f' C {x + card_width / 3} {y} ' + f'{x + 2 * card_width / 3} {y} ' + f'{x + card_width} {y}') + x += card_width + else: + overhead = 20 + lane * 24 + bow = 42 + lane * 8 + lane += 1 + path += (f' C {sx + bow} {sy} {sx + bow} {overhead} ' + f'{sx} {overhead} C {sx - gap} {overhead} ' + f'{dx + gap} {overhead} {dx} {overhead} ' + f'C {dx - bow} {overhead} {dx - bow} {dy} {dx} {dy}') + standard_paths.append(path) + handles.update(((sx, sy), (dx, dy))) + + layout_cards = [dict(name=name, width=card_width, + height=cards[name][-1], rank=ranks[name], + x=xy[node_ids[name]][0], y=xy[node_ids[name]][1]) + for name in names] + layout_edges = [dict(source=edge.dst_table, target=edge.src_table, + source_port=cards[edge.dst_table][3][ + definitions[edge.dst_table].pkey], + target_port=cards[edge.src_table][3][edge.fkey]) + for edge in edges] + standard = dict(width=width, height=height, + positions={name: list(xy[node_ids[name]]) for name in names}, + paths=standard_paths, handles=sorted(handles)) + layouts = build_layouts(layout_cards, layout_edges, standard) + selected = (choose_layout(layout_cards, layout_edges, layouts) + if layout == 'auto' else layout) + geometry = layouts[selected] + width, height = geometry['width'], geometry['height'] + samples = {} + uid = 'kumo-' + uuid4().hex + svg = [ + f'', + 'Table relationships', + 'Lines connect foreign keys (FK) to primary keys (PK).' + ' TS marks the declared time column; TE marks the end-time column.' + '', f'', + f'', + f'' + ] + if not names: + svg.append('' + 'No tables') + for index, edge in enumerate(edges): + path = geometry['paths'][index] + label = _attribute( + f'{edge.src_table}.{edge.fkey} → ' + f'{edge.dst_table}.{definitions[edge.dst_table].pkey}') + endpoints = ' '.join( + f'data-{key}="{_attribute(value)}"' + for key, value in [('src-table', + edge.src_table), ( + 'src-column', + edge.fkey), ('dst-table', edge.dst_table), + ('dst-column', + definitions[edge.dst_table].pkey)]) + svg.append( + f'{label}' + f'') + for name in names: + x, y = geometry['positions'][name] + title, header, rows, ports, footer, card_height = cards[name] + definition = definitions[name] + preview = (_preview(tables[name], [row[0] for row in rows]) + if show_samples and rows else {}) + outgoing = [e for e in edges if e.src_table == name] + connected = sorted(neighbors[name]) + summary = (f'{name}\n{len(definition.cols)} ' + f'column{"s" if len(definition.cols) != 1 else ""}') + if connected: + summary += '\nConnected tables:\n' + '\n'.join(connected) + else: + summary += '\nNo connected tables' + shape = _squircle(card_width, card_height) + clip = f'{uid}-card-{node_ids[name]}' + svg.append(f'' + f'' + '' + f'') + svg.append(_tooltip(summary, table=name, connected_tables=connected)) + svg.append(f'') + for i, line in enumerate(title): + svg.append('{escape(line)}' + '') + svg.append('') + for row_index, (column, lines, ry, rh) in enumerate(rows): + role = keys[name].get(column, '') + col = next(c for c in definition.cols if c.name == column) + detail = (f'{name}.{column}\nType: {col.dtype.value}\n' + f'Semantic type: {col.stype.value}\n' + f'Role: {role or "Column"}') + detail += ''.join(f'\nReferences: {e.dst_table}.' + f'{definitions[e.dst_table].pkey}' + for e in outgoing if e.fkey == column) + row = f'{node_ids[name]}-{row_index}' + metadata = _tooltip(detail, row, name, column) + if column in preview: + samples[metadata] = _tooltip(detail + preview[column], row, + name, column) + svg.append(metadata) + fill = THEME['role_background'] if role else 'transparent' + cls = 'kumo-surface kumo-key-surface' if role else 'kumo-surface' + svg.append(f'') + for i, line in enumerate(lines): + svg.append('{escape(line)}' + '') + if role: + svg.append(f'{role}') + svg.append('') + if footer: + svg.append(f'{footer}') + svg.append('') + for x, y in geometry['handles']: + svg.append(f'') + index = [dict( + name=name, + columns=[dict(name=col.name, type=col.dtype.value, + role=keys[name].get(col.name, '')) + for col in definitions[name].cols], + connections=[f'{edge.src_table}.{edge.fkey} → ' + f'{edge.dst_table}.{definitions[edge.dst_table].pkey}' + for edge in edges if edge.src_table == name], + ) for name in names] + return SVG(''.join(svg) + '', samples, uid, layouts=layouts, + initial_layout=selected, index=index) diff --git a/packages/kumo-engine/kumo/engine/_visualization_layout.py b/packages/kumo-engine/kumo/engine/_visualization_layout.py new file mode 100644 index 000000000..2ebb7b4ee --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization_layout.py @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Deterministic graph geometry, independent of rendering and viewport state. + +All coordinates use SVG units. Edges run from a primary-key table's right +port to a foreign-key table's left port. Layouts never read source data. +""" +from __future__ import annotations + +import heapq +import math +from collections import defaultdict +from typing import TypedDict + + +class Card(TypedDict): + name: str + width: float + height: float + rank: int + x: float + y: float + + +class Edge(TypedDict): + source: str + target: str + source_port: float + target_port: float + + +class Geometry(TypedDict): + width: float + height: float + positions: dict[str, list[float]] + paths: list[str] + handles: list[list[float]] + + +Point = tuple[float, float] +Rect = tuple[float, float, float, float] +_GAP = 72.0 +_COLUMN_GAP = 140.0 +_MARGIN = 80.0 + + +def _components(cards: list[Card], edges: list[Edge]) -> list[list[Card]]: + lookup = {card['name']: card for card in cards} + neighbors: dict[str, set[str]] = {name: set() for name in lookup} + for edge in edges: + neighbors[edge['source']].add(edge['target']) + neighbors[edge['target']].add(edge['source']) + pending = set(lookup) + result = [] + while pending: + stack = [min(pending)] + found = set() + while stack: + name = stack.pop() + if name in found: + continue + found.add(name) + stack.extend(sorted(neighbors[name] - found, reverse=True)) + pending -= found + result.append([lookup[name] for name in sorted(found)]) + return sorted(result, key=lambda group: (-len(group), group[0]['name'])) + + +def _overlap(a: Rect, b: Rect, gap: float = 0) -> bool: + return (a[0] < b[2] + gap and a[2] + gap > b[0] + and a[1] < b[3] + gap and a[3] + gap > b[1]) + + +def _bounds(cards: list[Card], positions: dict[str, list[float]]) -> Rect: + return (min(positions[c['name']][0] for c in cards), + min(positions[c['name']][1] for c in cards), + max(positions[c['name']][0] + c['width'] for c in cards), + max(positions[c['name']][1] + c['height'] for c in cards)) + + +def _compact_component(cards: list[Card], edges: list[Edge] + ) -> dict[str, list[float]]: + ranks: dict[int, list[Card]] = defaultdict(list) + for card in cards: + ranks[card['rank']].append(card) + sparse_hub = len(ranks) <= 2 and len(edges) <= 2 * len(cards) + for group in ranks.values(): + group.sort(key=lambda c: (c['y'], c['name'])) + # Dense cyclic ranks are artifacts of breaking cycles, not useful layers. + # A requested compact view may also wrap a long, single-table chain. + if len(edges) > 2 * len(cards) or len(ranks) > max(4, len(cards) // 2): + ordered = sorted(cards, key=lambda c: (c['rank'], c['y'], c['name'])) + ranks = {0: ordered} + tallest = max(c['height'] for c in cards) + total = sum(c['height'] + _GAP for c in cards) + candidates = [] + for column_count in range(1, min(len(cards), 8) + 1): + limit = max(tallest, total / column_count) + columns: list[list[Card]] = [] + for rank in sorted(ranks): + group = ranks[rank] + parts = min(len(group), max(1, math.ceil( + sum(c['height'] + _GAP for c in group) / (limit + _GAP)))) + if sparse_hub: + # Splitting one crowded stack into two is enough for sparse + # hubs; avoid spreading a simple graph across many columns. + parts = min(parts, 2) + remaining = list(group) + for part in range(parts): + needed = parts - part + target = sum(c['height'] + _GAP for c in remaining) / needed + running, best, stop = 0.0, math.inf, 1 + for at, card in enumerate(remaining[:len(remaining) - needed + 1]): + running += card['height'] + _GAP + if abs(running - target) < best: + best, stop = abs(running - target), at + 1 + columns.append(remaining[:stop]) + remaining = remaining[stop:] + heights = [sum(c['height'] for c in col) + _GAP * (len(col) - 1) + for col in columns] + positions: dict[str, list[float]] = {} + x = 0.0 + for column, height in zip(columns, heights): + y = (max(heights) - height) / 2 + for card in column: + positions[card['name']] = [x, y] + y += card['height'] + _GAP + x += max(c['width'] for c in column) + _COLUMN_GAP + width, height = x - _COLUMN_GAP, max(heights) + # Score against a fixed 1.45:1 aspect ratio. + score = max(width / 1.45, height) + .06 * (width + height) + span = sum(math.hypot( + positions[e['source']][0] - positions[e['target']][0], + positions[e['source']][1] + e['source_port'] + - positions[e['target']][1] - e['target_port']) for e in edges) + score += .06 * span / max(1, len(edges)) + candidates.append((score, column_count, positions)) + return min(candidates, key=lambda item: item[:2])[2] + + +def _pack_components(cards: list[Card], edges: list[Edge] + ) -> dict[str, list[float]]: + blocks = [] + for group in _components(cards, edges): + names = {c['name'] for c in group} + relations = [e for e in edges if e['source'] in names] + positions = _compact_component(group, relations) + left, top, right, bottom = _bounds(group, positions) + positions = {name: [x - left, y - top] + for name, (x, y) in positions.items()} + blocks.append((right - left, bottom - top, positions)) + # Place separate domains as rectangles. Small isolated tables can fill + # gaps beside an earlier component instead of forcing another full row. + gap = 96.0 + minimum = max(w for w, _, _ in blocks) + total = sum(w + gap for w, _, _ in blocks) - gap + candidates = [] + for steps in range(9): + limit = minimum + (total - minimum) * steps / 8 + placed: list[Rect] = [] + result: dict[str, list[float]] = {} + for width, height, positions in blocks: + xs = sorted({0.0, *(r[2] + gap for r in placed)}) + ys = sorted({0.0, *(r[3] + gap for r in placed)}) + slots = [(y, x) for y in ys for x in xs + if x + width <= limit + 1e-6 + and not any(_overlap((x, y, x + width, y + height), + r, gap) for r in placed)] + y, x = min(slots) + placed.append((x, y, x + width, y + height)) + for name, (cx, cy) in positions.items(): + result[name] = [cx + x + _MARGIN, cy + y + _MARGIN] + width = max(r[2] for r in placed) + height = max(r[3] for r in placed) + score = max(width / 1.45, height) + .06 * (width + height) + candidates.append((score, steps, result)) + return min(candidates, key=lambda item: item[:2])[2] + + +def _line_hits(a: Point, b: Point, rect: Rect) -> bool: + """Whether an open segment enters the interior of a rectangle.""" + if (max(a[0], b[0]) <= rect[0] or min(a[0], b[0]) >= rect[2] + or max(a[1], b[1]) <= rect[1] or min(a[1], b[1]) >= rect[3]): + return False + low, high = 0.0, 1.0 + for axis in (0, 1): + delta = b[axis] - a[axis] + if abs(delta) < 1e-9: + if a[axis] <= rect[axis] or a[axis] >= rect[axis + 2]: + return False + else: + start = (rect[axis] - a[axis]) / delta + end = (rect[axis + 2] - a[axis]) / delta + low, high = max(low, min(start, end)), min(high, max(start, end)) + if low >= high - 1e-9: + return False + return high > 0 and low < 1 + + +def _curve_clear(curve: tuple[Point, ...], rect: Rect, depth: int = 7) -> bool: + box = (min(p[0] for p in curve), min(p[1] for p in curve), + max(p[0] for p in curve), max(p[1] for p in curve)) + if not _overlap(box, rect): + return True + if depth == 0: + return False + a, b, c, d = curve + ab = ((a[0] + b[0]) / 2, (a[1] + b[1]) / 2) + bc = ((b[0] + c[0]) / 2, (b[1] + c[1]) / 2) + cd = ((c[0] + d[0]) / 2, (c[1] + d[1]) / 2) + abc = ((ab[0] + bc[0]) / 2, (ab[1] + bc[1]) / 2) + bcd = ((bc[0] + cd[0]) / 2, (bc[1] + cd[1]) / 2) + mid = ((abc[0] + bcd[0]) / 2, (abc[1] + bcd[1]) / 2) + return (_curve_clear((a, ab, abc, mid), rect, depth - 1) + and _curve_clear((mid, bcd, cd, d), rect, depth - 1)) + + +def _point(point: Point) -> str: + return f'{point[0]:.3f} {point[1]:.3f}' + + +def _smooth(points: list[Point], rectangles: list[Rect]) -> str: + # Grow corner rounding only as far as the surrounding tables allow. + path = 'M ' + _point(points[0]) + for a, b, c in zip(points, points[1:], points[2:]): + before, after = math.dist(a, b), math.dist(b, c) + if min(before, after) < 1e-9: + continue + radius = min(64.0, before * .45, after * .45) + while True: + p = (b[0] + (a[0] - b[0]) * radius / before, + b[1] + (a[1] - b[1]) * radius / before) + q = (b[0] + (c[0] - b[0]) * radius / after, + b[1] + (c[1] - b[1]) * radius / after) + controls = (p, ((p[0] + 2 * b[0]) / 3, (p[1] + 2 * b[1]) / 3), + ((q[0] + 2 * b[0]) / 3, (q[1] + 2 * b[1]) / 3), q) + if radius <= 1 or all(_curve_clear(controls, rect) + for rect in rectangles): + break + radius /= 2 + path += f' L {_point(p)} Q {_point(b)} {_point(q)}' + return path + ' L ' + _point(points[-1]) + + +def _segments(points: list[Point]): + """Index collinear route segments by their supporting line.""" + for a, b in zip(points, points[1:]): + length = math.dist(a, b) + if length < 1e-9: + continue + dx, dy = (b[0] - a[0]) / length, (b[1] - a[1]) / length + if dx < 0 or (dx == 0 and dy < 0): + dx, dy = -dx, -dy + key = (round(dx, 8), round(dy, 8), round(a[1] * dx - a[0] * dy, 3)) + low, high = sorted((a[0] * dx + a[1] * dy, + b[0] * dx + b[1] * dy)) + yield key, low, high + + +class _Router: + """A reusable visibility graph for one arrangement and clearance lane.""" + + def __init__(self, rectangles: list[Rect], clearance: float): + self.rectangles = [(x - clearance, y - clearance, + right + clearance, bottom + clearance) + for x, y, right, bottom in rectangles] + self.points = [(x, y) for left, top, right, bottom in self.rectangles + for x, y in ((left, top), (right, top), + (right, bottom), (left, bottom))] + self.links: list[list[tuple[int, float]]] = [ + [] for _ in self.points] + for i, a in enumerate(self.points): + for j in range(i): + b = self.points[j] + if self.clear(a, b): + distance = math.dist(a, b) + self.links[i].append((j, distance)) + self.links[j].append((i, distance)) + self.connectors: dict[Point, list[tuple[int, float]]] = {} + + def clear(self, a: Point, b: Point) -> bool: + return not any(_line_hits(a, b, r) for r in self.rectangles) + + def connections(self, point: Point) -> list[tuple[int, float]]: + if point not in self.connectors: + self.connectors[point] = [ + (i, math.dist(point, p)) for i, p in enumerate(self.points) + if self.clear(point, p)] + return self.connectors[point] + + def route(self, start: Point, end: Point) -> list[Point]: + if self.clear(start, end): + return [start, end] + finish = dict(self.connections(end)) + distance = [math.inf] * len(self.points) + previous: dict[int, int] = {} + queue = [] + for node, cost in self.connections(start): + distance[node] = cost + heapq.heappush(queue, (cost, node)) + best, last = math.inf, None + while queue: + cost, node = heapq.heappop(queue) + if cost != distance[node]: + continue + if cost >= best: + break + if node in finish and cost + finish[node] < best: + best, last = cost + finish[node], node + for neighbor, step in self.links[node]: + candidate = cost + step + if candidate < distance[neighbor]: + distance[neighbor] = candidate + previous[neighbor] = node + heapq.heappush(queue, (candidate, neighbor)) + if last is None: + raise ValueError('Unable to route graph relationship around tables.') + points = [end] + while last is not None: + points.append(self.points[last]) + last = previous.get(last) + points.append(start) + return points[::-1] + + +def _geometry(cards: list[Card], edges: list[Edge]) -> Geometry: + positions = _pack_components(cards, edges) + lookup = {c['name']: c for c in cards} + rectangles = [(positions[c['name']][0], positions[c['name']][1], + positions[c['name']][0] + c['width'], + positions[c['name']][1] + c['height']) for c in cards] + routers: dict[int, _Router] = {} + handles = set() + occupied: dict[tuple, list[tuple[float, float]]] = defaultdict(list) + keys = [(e['source'], e['target'], e['source_port'], e['target_port']) + for e in edges] + routed = {} + for key in sorted(set(keys)): + source, target, source_port, target_port = key + sx, sy = positions[source] + tx, ty = positions[target] + start = (sx + lookup[source]['width'], sy + source_port) + end = (tx, ty + target_port) + handles.update((start, end)) + half = (end[0] - start[0]) / 2 + curve = (start, (start[0] + half, start[1]), + (end[0] - half, end[1]), end) + if half >= 24 and all( + _curve_clear(curve, (rect[0] - 10, rect[1] - 10, + rect[2] + 10, rect[3] + 10)) + for c, rect in zip(cards, rectangles) + if c['name'] not in (source, target)): + routed[key] = ('M ' + _point(start) + ' C ' + + ' '.join(_point(p) for p in curve[1:])) + else: + candidates = [] + for lane in range(7): + if lane not in routers: + routers[lane] = _Router(rectangles, 14 + lane * 3.5) + offset = 40 + lane * 3.5 + route = routers[lane].route( + (start[0] + offset, start[1]), + (end[0] - offset, end[1])) + segments = list(_segments(route)) + overlap = sum(max(0, min(high, old_high) - max(low, old_low)) + for line, low, high in segments + for old_low, old_high in occupied[line]) + length = sum(math.dist(a, b) for a, b in zip(route, route[1:])) + candidates.append((overlap * 4 + length, lane, route, segments)) + _, _, route, segments = min(candidates) + for line, low, high in segments: + occupied[line].append((low, high)) + routed[key] = _smooth([start, *route, end], rectangles) + paths = [routed[key] for key in keys] + _, _, right, bottom = _bounds(cards, positions) + return {'width': right + _MARGIN, 'height': bottom + _MARGIN, + 'positions': positions, 'paths': paths, + 'handles': [list(p) for p in sorted(handles)]} + + +def build_layouts(cards: list[Card], edges: list[Edge], + standard: Geometry) -> dict[str, Geometry]: + """Return Standard and Compact geometry without modifying inputs.""" + if not cards: + return dict.fromkeys(('standard', 'compact'), standard) + ordered = sorted(cards, key=lambda c: c['name']) + return {'standard': standard, 'compact': _geometry(ordered, edges)} + + +def choose_layout(cards: list[Card], edges: list[Edge], + layouts: dict[str, Geometry]) -> str: + """Select the initial layout from graph structure.""" + if not cards: + return 'standard' + standard, compact = layouts['standard'], layouts['compact'] + components = _components(cards, edges) + ranks: dict[int, int] = defaultdict(int) + for card in cards: + ranks[card['rank']] += 1 + crowded = max(ranks.values()) >= 7 and standard['height'] > 1800 + rank_by_name = {c['name']: c['rank'] for c in cards} + feedback = any(rank_by_name[e['source']] >= rank_by_name[e['target']] + for e in edges) + dense = len(edges) > 2 * len(cards) and feedback + # Keep layered many-to-many dimensions in their familiar two-column + # overview; compact remains available as an explicit alternative. + if len(ranks) == 2 and min(ranks.values()) >= 7: + return 'standard' + disconnected = len(components) > 1 + old_extent = max(standard['width'] / 1.45, standard['height']) + new_extent = max(compact['width'] / 1.45, compact['height']) + improvement = 1.0 if disconnected else .85 + if ((crowded or dense or disconnected) + and new_extent < old_extent * improvement): + return 'compact' + return 'standard' diff --git a/packages/kumo-engine/kumo/engine/_visualization_pdf.py b/packages/kumo-engine/kumo/engine/_visualization_pdf.py new file mode 100644 index 000000000..c6aea44da --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization_pdf.py @@ -0,0 +1,284 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Vector graph diagrams with a paginated schema and relationship index.""" + +import re +from functools import lru_cache +from io import BytesIO +from xml.etree import ElementTree as ET + +from matplotlib import get_data_path, rc_context +from matplotlib.backends.backend_pdf import PdfPages +from matplotlib.figure import Figure +from matplotlib.font_manager import FontProperties, get_font +from matplotlib.patches import Circle, PathPatch, Rectangle +from matplotlib.path import Path +from matplotlib.textpath import TextToPath +from matplotlib.transforms import Affine2D + +from kumo.engine._widget import THEME + + +@lru_cache(maxsize=1) +def _fonts(): + directory = get_data_path() + '/fonts/ttf/DejaVuSans' + paths = [directory + suffix + '.ttf' for suffix in ('', '-Bold')] + fonts = [FontProperties(fname=path) for path in paths] + supported = set.intersection(*(set(get_font(path).get_charmap()) + for path in paths)) + return fonts, supported + + +def _path(value): + """Read only the absolute path commands emitted by our graph layouts.""" + tokens = iter( + re.findall(r'[MLHVQCZ]|[-+]?(?:\d*\.\d+|\d+)(?:[eE][-+]?\d+)?', value)) + vertices, codes = [], [] + x = y = 0 + for command in tokens: + if command == 'Z': + vertices.append((0, 0)) + codes.append(Path.CLOSEPOLY) + continue + count = {'M': 2, 'L': 2, 'H': 1, 'V': 1, 'Q': 4, 'C': 6}[command] + points = [float(next(tokens)) for _ in range(count)] + if command == 'H': + points = [points[0], y] + elif command == 'V': + points = [x, points[0]] + vertices.extend(zip(points[::2], points[1::2])) + codes.extend([{ + 'M': Path.MOVETO, + 'Q': Path.CURVE3, + 'C': Path.CURVE4 + }.get(command, Path.LINETO)] * (len(points) // 2)) + x, y = points[-2:] + return Path(vertices, codes) + + +def render_pdf(svg: str, geometry: dict, index: list[dict]) -> bytes: + """Export our internally generated SVG subset, without sample metadata.""" + fonts, supported = _fonts() + metrics = TextToPath() + escaped = False + + def safe(value): + nonlocal escaped + result = ''.join( + char if ord(char) in supported or char in '\n\t' else char. + encode('unicode_escape').decode('ascii') for char in value) + escaped |= result != value + return result + + def width(value, size: float = 9, bold=False): + font = fonts[bold].copy() + font.set_size(size) + return metrics.get_text_width_height_descent(value, font, False)[0] + + def text(axes, x, y, value, size: float = 9, bold=False, color=None, + align='left'): + axes.text(x, y, value, fontproperties=fonts[bold], fontsize=size, + color=color or THEME['foreground'], ha=align, va='baseline', + parse_math=False, usetex=False, clip_on=False, zorder=3) + + def page(w, h): + figure = Figure(figsize=(w / 72, h / 72)) + axes = figure.add_axes((0, 0, 1, 1), xlim=(0, w), ylim=(h, 0)) + axes.set_axis_off() + return figure, axes + + output = BytesIO() + with rc_context({ + 'pdf.fonttype': 42, 'figure.facecolor': THEME['background'], + 'savefig.facecolor': THEME['background'], + 'savefig.edgecolor': THEME['background'], + 'savefig.transparent': False, + }), PdfPages( + output, metadata={ + 'Title': 'Kumo graph and schema index', + 'Author': 'Kumo Engine' + }) as pdf: + w, h = geometry['width'], geometry['height'] + pw, ph = min(14400, max(842, w + 64)), min(14400, max(595, h + 96)) + figure, axes = page(pw, ph) + text(axes, 32, 32, 'Table relationships', 16, True, THEME['accent']) + text(axes, 32, ph - 20, + 'PK: primary key FK: foreign key TS: time column ' + 'TE: end-time column') + scale = min(1, (pw - 64) / w, (ph - 96) / h) + origin = ((pw - w * scale) / 2, (ph - h * scale) / 2 + 8) + root = ET.fromstring(svg) + clips = { + node.get('id'): _path(next(iter(node)).get('d')) + for node in root.iter() if node.tag.endswith('}clipPath') + } + edges = iter(geometry['paths']) + + def draw(node, x: float = 0, y: float = 0, column_limit=None): + tag = node.tag.rsplit('}', 1)[-1] + if tag in ('defs', 'title', 'desc', 'circle', 'rect'): + return + if 'data-kumo-card' in node.attrib: + x, y = geometry['positions'][node.get('data-kumo-card')] + elif node.get('transform'): + dx, dy = map(float, + re.findall(r'[-+]?[\d.]+', node.get('transform'))) + x, y = x + dx, y + dy + for child in node: + if child.get('class') == 'kumo-role': + # Match the PDF font, leaving a gap before this row's badge. + column_limit = (float(child.get('x')) - 18 - 8 + - width(child.text or '', 10, True)) + transform = Affine2D().translate( + x, y).scale(scale).translate(*origin) + axes.transData + if tag == 'path': + value = next(edges) if node.get( + 'class') == 'kumo-edge' else node.get('d') + fill = node.get('fill', 'none') + patch = PathPatch( + _path(value), transform=transform, + facecolor='none' if fill == 'transparent' else fill, + edgecolor=node.get('stroke', 'none'), + linewidth=float(node.get('stroke-width', 1)) * scale) + if node.get('clip-path'): + patch.set_clip_path(clips[node.get('clip-path')[5:-1]], + transform) + axes.add_patch(patch) + elif tag == 'text': + original = node.text or '' + value = safe(original).expandtabs(4) + size = float(node.get('font-size', 16)) + bold = node.get('font-weight') == '700' + if value != original or (column_limit is not None and + node.get('class') == 'kumo-column-name'): + limit = (column_limit or 210) if size == 16 else 272 + size *= min(1, limit / max(1, width(value, size, bold))) + text(axes, origin[0] + (x + float(node.get('x', 0))) * scale, + origin[1] + (y + float(node.get('y', 0))) * scale, value, + size * scale, bold, node.get('fill'), + 'right' if node.get('text-anchor') == 'end' else 'left') + for child in node: + draw(child, x, y, column_limit) + + draw(root) + for x, y in geometry['handles']: + axes.add_patch( + Circle((origin[0] + x * scale, origin[1] + y * scale), + 4 * scale, color=THEME['muted'], linewidth=0)) + pdf.savefig(figure) + figure.clear() + # Index pages use point coordinates so line lengths and page breaks are + # independent of graph size. Split even a single oversized column row. + pw, ph = 595.28, 841.89 + figure, axes = page(pw, ph) + y = 40 + page_number = 2 + columns = (36, 321, 461, pw - 36) + + def next_page(): + nonlocal figure, axes, y, page_number + text(axes, pw - 36, ph - 20, f'Page {page_number}', align='right', + color=THEME['muted']) + pdf.savefig(figure) + figure.clear() + figure, axes = page(pw, ph) + y = 40 + page_number += 1 + + def wrap(value, limit, size=9, bold=False): + lines = [] + for part in safe(value).expandtabs(4).split('\n'): + if not part: + lines.append('') + while part: + if width(part, size, bold) <= limit: + lines.append(part) + break + lo, hi = 1, len(part) + while lo < hi: + mid = (lo + hi + 1) // 2 + if width(part[:mid], size, bold) <= limit: + lo = mid + else: + hi = mid - 1 + cut = lo + if cut < len(part) and ' ' in part[:cut]: + cut = part.rfind(' ', 0, cut) + 1 + lines.append(part[:cut]) + part = part[cut:] + return lines + + def paragraph(value, size=9, bold=False): + nonlocal y + for line in wrap(value, pw - 72, size, bold): + if y + size + 5 > ph - 40: + next_page() + text(axes, 36, y + size, line, size, bold) + y += size + 5 + + def row(values, header=False): + nonlocal y + lines = [ + wrap(value, columns[i + 1] - columns[i] - 12, bold=header) + for i, value in enumerate(values) + ] + offset, count = 0, max(map(len, lines)) + while offset < count: + if y + 25 > ph - 40: + next_page() + if not header: + row(('Column', 'Data type', 'Role'), True) + take = min(count - offset, max(1, int( + (ph - 40 - y - 12) // 13))) + height = take * 13 + 12 + axes.add_patch( + Rectangle((36, y), pw - 72, height, + facecolor=THEME['header_background'] + if header else 'none', edgecolor='none')) + for i, cell in enumerate(lines): + for n, line in enumerate(cell[offset:offset + take]): + text(axes, columns[i] + 6, y + 15 + n * 13, line, + bold=header) + y += height + axes.plot((36, pw - 36), (y, y), color=THEME['card_border'], + linewidth=.3) + offset += take + + paragraph('Tables and connections', 14, True) + paragraph( + 'All declared columns are listed, including columns omitted from the diagram. Sample values are excluded.' + ) + # Check the complete index before deciding whether its fallback note is needed. + for table in index: + safe(table['name'] + ''.join(table['connections'])) + for col in table['columns']: + safe(col['name'] + col['type'] + col['role']) + if escaped: + paragraph( + 'Characters unavailable in the PDF fonts use Unicode escapes (\\uXXXX or \\UXXXXXXXX).' + ) + for table in index: + y += 12 + if y + 70 > ph - 40: + next_page() + paragraph(table['name'], 14, True) + y += 6 + if table['columns']: + row(('Column', 'Data type', 'Role'), True) + for col in table['columns']: + row((col['name'], col['type'], col['role'] or 'Column')) + else: + paragraph('No columns') + y += 6 + for connection in table['connections'] or [ + 'No outgoing connections' + ]: + paragraph(connection) + if not index: + paragraph('No tables') + text(axes, pw - 36, ph - 20, f'Page {page_number}', align='right', + color=THEME['muted']) + pdf.savefig(figure) + figure.clear() + return output.getvalue() diff --git a/packages/kumo-engine/kumo/engine/_visualization_viewer.py b/packages/kumo-engine/kumo/engine/_visualization_viewer.py new file mode 100644 index 000000000..7c726b638 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization_viewer.py @@ -0,0 +1,575 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Self-contained notebook viewer; layout geometry is computed in Python.""" + +from string import Template + +from kumo.engine._widget import CSS_THEME, button, render_static_svg + +_STYLE = Template(r""" +.kumo-graph{container:kumo-graph/inline-size;padding:16px;overflow:clip} +.kumo-graph,.kumo-graph *{user-select:none;-webkit-user-select:none} +.kumo-graph:fullscreen{width:100%;height:100%;max-height:none} +.kumo-graph-toolbar{display:grid; +grid-template-columns:minmax(0,1fr) auto minmax(0,1fr)} +.kumo-graph-layout{display:flex;grid-column:2;grid-row:1;justify-self:center; +border-radius:18px;background:$background;padding:2px;color:$chrome_foreground; +font:13px $font} +.kumo-graph-layout label{position:relative;cursor:pointer} +.kumo-graph-layout input{position:absolute;opacity:0;width:1px;height:1px} +.kumo-graph-layout span{display:flex;align-items:center;height:28px; +padding:0 12px;border-radius:16px} +.kumo-graph-layout input:checked+span{background:$chrome_hover;color:$foreground} +.kumo-graph-actions{display:flex;align-items:center;gap:6px; +grid-column:1;grid-row:1;justify-self:start} +.kumo-graph-zoom{display:flex;align-items:center;gap:0;flex:none; +background:$background;border-radius:18px} +.kumo-graph-layout,.kumo-graph-zoom,.kumo-graph-actions>button{ +box-shadow:$chrome_shadow} +@container kumo-graph (max-width:900px){ +.kumo-graph-toolbar{grid-template-columns:minmax(0,1fr);row-gap:12px} +.kumo-graph-layout{grid-column:1;grid-row:2}} +@container kumo-graph (max-width:340px){.kumo-graph-actions{gap:2px}} +.kumo-graph-layout input:focus-visible+span{ +outline:2px solid $focus;outline-offset:2px} +.kumo-graph-scale{min-width:44px;text-align:center;color:$muted; +font:12px $font;font-variant-numeric:tabular-nums} +.kumo-graph .kumo-graph-exit{display:none} +.kumo-graph:fullscreen .kumo-graph-enter{display:none} +.kumo-graph:fullscreen .kumo-graph-exit{display:inline-flex} +.kumo-graph-view{position:relative;overflow:clip;min-height:0;flex:none; +max-width:100%;cursor:grab; +background:radial-gradient(circle at 1px 1px,$grid .7px, +transparent .8px) 0 0/16px 16px} +.kumo-graph:fullscreen .kumo-graph-view{flex:1;height:auto!important} +.kumo-graph-scene{position:absolute;left:0;top:0;transform-origin:0 0} +.kumo-graph .kumo-graph-view [data-kumo-tip]{cursor:inherit} +.kumo-graph-view[data-dragging],.kumo-graph-view[data-dragging] *{cursor:grabbing} +.kumo-graph-view svg{display:block;flex:none;max-width:none!important; +max-height:none!important} +.kumo-graph-status{font-size:13px;color:$muted;padding:6px 0; +white-space:pre-line;user-select:text;-webkit-user-select:text} +.kumo-graph-export-options{display:grid;gap:8px; +padding:10px 12px;margin-bottom:12px;background:$surface;border-radius:10px} +.kumo-graph-export-options,.kumo-graph-export-options *{ +user-select:text;-webkit-user-select:text} +.kumo-graph-export-options code{display:block;margin:8px 0;white-space:pre-wrap; +overflow-wrap:anywhere} +.kumo-graph-view svg>rect{fill:transparent} +.kumo-graph-tooltip{position:absolute;top:0;left:0;z-index:10; +box-sizing:border-box; +width:320px;max-width:calc(100% - 16px);border:1px solid $tooltip_border; +border-radius:18px;corner-shape:squircle;background:$tooltip_background;color:$foreground; +box-shadow:$tooltip_shadow; +font:16px/1.4 $font;overflow-wrap:anywhere; +overflow:hidden;pointer-events:none;opacity:0;visibility:hidden; +transition:opacity 140ms ease,visibility 140ms ease} +.kumo-graph-tooltip[data-open]{opacity:1;visibility:visible} +.kumo-graph .kumo-tip-head{background:$header_background;border-bottom:1px solid $tooltip_header_border; +padding:7px 10px} +.kumo-graph .kumo-tip-context{color:$muted;font-size:16px;margin-bottom:2px} +.kumo-graph .kumo-tip-name{font-size:17px;font-weight:700;color:$heading} +.kumo-graph .kumo-tip-body{padding:7px 10px} +.kumo-graph .kumo-tip-badges{display:flex;flex-wrap:wrap;gap:5px} +.kumo-graph .kumo-tip-samples{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:5px} +.kumo-graph .kumo-tip-badge{background:$surface;border:1px solid $surface_border; +border-radius:7px;corner-shape:squircle;padding:2px 6px;color:$muted} +.kumo-graph .kumo-tip-key{background:$row_highlight;border-color:transparent;color:$accent} +.kumo-graph .kumo-tip-type{color:$muted} +.kumo-graph .kumo-tip-ref{padding-top:5px;color:$muted} +.kumo-graph .kumo-tip-connections{list-style:none;padding:0;margin:0;display:grid;gap:4px 10px; +grid-auto-flow:column;grid-template-columns:minmax(0,1fr)} +.kumo-graph .kumo-tip-connections[data-columns="2"]{grid-template-columns:repeat(2,minmax(0,1fr))} +.kumo-graph .kumo-tip-connection{color:$foreground;font-weight:500;min-width:0; +white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.kumo-graph .kumo-tip-label{color:$muted;margin:5px 0 3px} +.kumo-graph .kumo-tip-sample{min-width:0;max-width:100%;white-space:nowrap; +overflow:hidden;text-overflow:ellipsis;background:$surface;border-radius:7px;corner-shape:squircle; +padding:2px 6px;color:$foreground; +font:16px/1.4 ui-monospace,SFMono-Regular,Consolas,monospace} +.kumo-graph .kumo-tip-sample:only-child{grid-column:1/-1} +.kumo-graph [data-kumo-tip]{outline:none;cursor:default} +.kumo-graph .kumo-row-light{fill:$row_highlight;stroke:none; +opacity:0;pointer-events:none;transition:opacity 140ms ease} +.kumo-graph .kumo-active .kumo-row-light{opacity:1} +.kumo-graph [data-kumo-tip]:focus-visible .kumo-row-light{ +stroke:$focus;stroke-width:2} +.kumo-graph [data-kumo-row]:not([data-kumo-row=""]) .kumo-row-light{ +stroke:none;filter:drop-shadow(0 1px 2px #00000026)} +.kumo-graph [data-kumo-row]:not([data-kumo-row=""]):focus-visible .kumo-row-light{ +filter:drop-shadow(0 0 2px $focus)} +.kumo-graph .kumo-edge{transition:stroke 140ms ease,stroke-width 140ms ease, +opacity 140ms ease} +.kumo-graph.kumo-links-focused .kumo-edge:not(.kumo-edge-active){opacity:.35} +.kumo-graph .kumo-edge-active{stroke-width:1.8} +@media(prefers-reduced-motion:reduce){.kumo-graph-tooltip, +.kumo-graph .kumo-row-light, +.kumo-graph .kumo-edge{transition:none}} + +.kumo-graph .kumo-edge{vector-effect:non-scaling-stroke;stroke:$edge} +.kumo-graph .kumo-edge-active{stroke:$edge_active} +.kumo-graph .kumo-card{fill:$card_background;stroke:$card_border} +.kumo-graph [data-kumo-row=""]>.kumo-surface{fill:$header_background;stroke:none} +.kumo-graph .kumo-key-surface{fill:$role_background} +.kumo-graph .kumo-card-name{fill:$heading} +.kumo-graph .kumo-column-name{fill:$foreground} +.kumo-graph .kumo-role{fill:$accent} +.kumo-graph .kumo-muted,.kumo-graph-handles circle{fill:$muted} +.kumo-graph .kumo-active>.kumo-row-light{fill:$row_highlight} +""").substitute(CSS_THEME) + +_HOVER = r""" +const tip=document.createElement('div'); +tip.className='kumo-graph-tooltip';tip.role='tooltip'; +tip.id='kumo-tip-'+Math.random().toString(36).slice(2);host.append(tip); +let active=null, card=null, timer, placement=null; +const firstCard=svg.querySelector(':scope>[data-kumo-card]'); +const element=(cls,text)=>{const n=document.createElement('div'); +n.className='kumo-tip-'+cls;n.textContent=text;return n}; +const cancelHide=()=>{clearTimeout(timer);timer=null}; +const hide=()=>{cancelHide();tip.removeAttribute('data-open'); +placement=null; +if(host.classList.contains('kumo-links-focused')) +edges.forEach(edge=>svg.insertBefore(edge.parentElement,firstCard)); +host.classList.remove('kumo-links-focused'); +edges.forEach(edge=>edge.classList.remove('kumo-edge-active')); +if(active){active.removeAttribute('aria-describedby'); +active.classList.remove('kumo-active')} +active=card=null}; +const content=el=>{ +const row=!!el.dataset.kumoRow, table=el.dataset.kumoTable; +const name=el.dataset.kumoColumn, full=table+(row?'.'+name:''); +const lines=el.dataset.kumoTip.slice(full.length+1).split('\n'); +const head=element('head',''), body=element('body',''); +head.append(element('context',row?table:'Table'),element('name',row?name:table)); +const badges=element('badges',''); +if(row){ +const role=lines.find(x=>x.startsWith('Role: '))?.slice(6); +if(role&&role!=='Column')badges.append(element('badge kumo-tip-key',role)); +if(badges.childElementCount)body.append(badges); +for(const [prefix,label] of [['Type: ','Data type: '], +['Semantic type: ','Semantic type: ']]){ +const value=lines.find(x=>x.startsWith(prefix))?.slice(prefix.length); +if(value)body.append(element('type',label+value))} +for(const edge of edges){const d=edge.dataset; +if(d.srcTable===table&&d.srcColumn===name) +body.append(element('ref','↗ '+d.dstTable+'.'+d.dstColumn))} +const index=lines.findIndex(x=>x.startsWith('Sample values:')); +if(index>=0){body.append(element('label','Sample values')); +const status=lines[index].slice(14).trim(), samples=element('samples',''); +for(let value of status?[status]:lines.slice(index+1,index+7)){ +value=value.replace(/^'|'$/g,''); +samples.append(element('sample', +value.length>90?value.slice(0,87)+'…':value))}body.append(samples)} +}else{ +badges.append(element('badge',lines[0])); +body.append(badges); +const connections=JSON.parse(el.dataset.kumoConnections); +body.append(element('label',connections.length?'Connected tables': +'No connected tables')); +if(connections.length){const list=document.createElement('ul'); +list.className='kumo-tip-connections'; +for(const name of connections){const item=document.createElement('li'); +item.className='kumo-tip-connection';item.textContent=name;item.title=name;list.append(item)} +body.append(list)}} +tip.replaceChildren(head,body)}; +const layoutConnections=sx=>{ +const list=tip.querySelector('.kumo-tip-connections');if(!list)return; +const items=Array.from(list.children),short=[],long=[],range=document.createRange(); +const half=(list.getBoundingClientRect().width-parseFloat(getComputedStyle(list).columnGap)*sx)/2; +for(const item of items){range.selectNodeContents(item); +(range.getBoundingClientRect().width<=half?short:long).push(item)} +const fallback=long.length>items.length/2||Math.ceil(short.length/2)+long.length>15; +const paired=items.length===1?[]:fallback?items:short; +const full=items.length===1?items:fallback?[]:long; +const rows=Math.min(15,Math.ceil(paired.length/2)); +list.dataset.columns=paired.length?'2':'1'; +list.style.gridTemplateRows=`repeat(${rows+full.length},auto)`; +paired.forEach((item,i)=>{ +item.style.gridColumn=String(Math.floor(i/rows)+1); +item.style.gridRow=String(i%rows+1)}); +full.forEach((item,i)=>{item.style.gridColumn='1 / -1'; +item.style.gridRow=String(rows+i+1)})}; +const viewport=()=>{ +const h=host.getBoundingClientRect(), v=view.getBoundingClientRect(); +const sx=h.width/host.offsetWidth||1, sy=h.height/host.offsetHeight||1; +let left=Math.max(h.left,v.left,0), right=Math.min(h.right,v.right,innerWidth); +let top=Math.max(h.top,v.top,0), bottom=Math.min(h.bottom,v.bottom,innerHeight); +for(let el=host.matches(':fullscreen')?null:host.parentElement; +el;el=el.parentElement){ +if(el===document.body||el===document.documentElement)break; +const style=getComputedStyle(el), r=el.getBoundingClientRect(); +if(/auto|scroll|hidden|clip/.test(style.overflowX)){ +left=Math.max(left,r.left);right=Math.min(right,r.right)} +if(/auto|scroll|hidden|clip/.test(style.overflowY)){ +top=Math.max(top,r.top);bottom=Math.min(bottom,r.bottom)}} +return{h,sx,sy,left:left+8*sx,right:right-8*sx, +top:top+8*sy,bottom:bottom-8*sy}}; +const intersects=(r,b)=>r.right>b.left&&r.leftb.top&&r.top{ +const r=edge.getBoundingClientRect();if(!intersects(r,b))return null; +if(!spans&&r.left>=b.left&&r.right<=b.right&&r.top>=b.top&&r.bottom<=b.bottom) +return[r.left,r.right]; +const d=edge.getAttribute('d');let cached=edgeCurves.get(edge); +if(cached?.d!==d){ +// Generated edges contain absolute M/L/Q/C segments; retain their control points. +const tokens=d.match(/[MLQC]|[-+]?(?:\d*\.)?\d+(?:e[-+]?\d+)?/gi); +const curves=[];let point; +for(let i=0;i{ +const xs=points.map(p=>p.x),ys=points.map(p=>p.y); +if(Math.max(...xs)b.right|| +Math.max(...ys)b.bottom)return; +const before=points[0],after=points.at(-1),dx=after.x-before.x,dy=after.y-before.y; +// The Bezier convex hull bounds deviation from the parameterized chord. +if(points.some((p,i)=>Math.abs(p.x-before.x-dx*i/(points.length-1))>epsilon|| +Math.abs(p.y-before.y-dy*i/(points.length-1))>epsilon)){ +const a=[before],z=[after];let level=points; +while(level.length>1){level=level.slice(1).map((p,i)=>({ +x:(p.x+level[i].x)/2,y:(p.y+level[i].y)/2})); +a.push(level[0]);z.unshift(level.at(-1))} +visit(a);visit(z);return} +let lo=0,hi=1; +for(const [p,q] of [[-dx,before.x-b.left+epsilon],[dx,b.right-before.x+epsilon], +[-dy,before.y-b.top+epsilon],[dy,b.bottom-before.y+epsilon]]){ +if(!p){if(q<0){hi=-1;break}}else if(p<0)lo=Math.max(lo,q/p);else hi=Math.min(hi,q/p)} +if(lo<=hi){const a=before.x+lo*dx,z=before.x+hi*dx; +left=Math.min(left,a,z);right=Math.max(right,a,z); +if(spans)spans.push([Math.min(a,z)-2,Math.max(a,z)+2])}}; +for(const curve of cached.curves)visit(curve.map(([x,y])=>({ +x:matrix.a*x+matrix.c*y+matrix.e,y:matrix.b*x+matrix.d*y+matrix.f}))); +return left===Infinity?null:[left-2,right+2]}; +const classify=()=>{ +const b=viewport(),c=card.getBoundingClientRect(),table=active.dataset.kumoTable; +const visible=Array.from(svg.querySelectorAll(':scope>[data-kumo-card]'), +x=>x.getBoundingClientRect()).filter(r=>intersects(r,b)); +const leftmost=c.left<=Math.min(...visible.map(r=>r.left))+1; +const rightmost=c.right>=Math.max(...visible.map(r=>r.right))-1; +const role=tip.querySelector('.kumo-tip-key')?.textContent||''; +const pk=/\bPK\b/.test(role),fk=/\bFK\b/.test(role); +const incident=edges.filter(e=>e.dataset.srcTable===table||e.dataset.dstTable===table); +const selected=incident.filter(e=>e.classList.contains('kumo-edge-active')); +const relevant=active.dataset.kumoRow?selected:incident; +const extents=new Map(),extent=e=>{if(!extents.has(e))extents.set(e,edgeExtent(e,b));return extents.get(e)}; +let side; +if(fk&&!pk)side='right'; +else if(leftmost!==rightmost)side=leftmost?'left':'right'; +else if(pk!==fk)side=pk?'left':'right'; +else{ +let left=0,right=0;for(const e of relevant){const r=extent(e);if(!r)continue; +if(r[0]c.right+1)right++} +side=left===right?(c.left-b.left>=b.right-c.right?'left':'right'): +(left{ +if(!active||!placement)return; +const b=viewport(),a=active.getBoundingClientRect(),c=card.getBoundingClientRect(); +if(b.right<=b.left||b.bottom<=b.top){hide();return} +tip.style.maxWidth=Math.max(0,Math.min(320,(b.right-b.left)/b.sx))+'px'; +tip.style.maxHeight=Math.max(0,(b.bottom-b.top)/b.sy)+'px'; +layoutConnections(b.sx); +const t=tip.getBoundingClientRect(),left=placement.side==='left'; +let x=(left?c.left-t.width-12*b.sx:c.right+12*b.sx)+placement.offset*b.sx; +const y=Math.max(b.top,Math.min(a.top,b.bottom-t.height)); +if(clearRoutes&&active.dataset.kumoRow){ +// Keep the popup clear of the visible highlighted routes. +const band={...b,top:Math.max(b.top,y-2),bottom:Math.min(b.bottom,y+t.height+2)}; +const spans=[]; +if(band.topleft?z[1]-a[1]:a[0]-z[0]); +const adjacent=x; +for(const span of spans)if(xspan[0]) +x=left?span[0]-t.width-12*b.sx:span[1]+12*b.sx; +placement.offset+=(x-adjacent)/b.sx} +if(!intersects(a,b)){hide();return} +tip.dataset.side=placement.side;tip.dataset.clamped=String(y!==a.top); +tip.style.left=(x-b.h.left)/b.sx+'px';tip.style.top=(y-b.h.top)/b.sy+'px'}; +const show=el=>{if(drag)return; +cancelHide();if(active===el)return; +hide();active=el;card=el.parentElement; +el.classList.add('kumo-active'); +content(el);el.setAttribute('aria-describedby',tip.id); +if(el.dataset.kumoRow){ +const table=el.dataset.kumoTable, column=el.dataset.kumoColumn; +edges.forEach(edge=>{const d=edge.dataset; +const src=d.srcTable===table?d.srcColumn===column:false; +const dst=d.dstTable===table?d.dstColumn===column:false; +edge.classList.toggle('kumo-edge-active',src||dst); +if(src||dst){svg.insertBefore(edge.parentElement,firstCard); +host.classList.add('kumo-links-focused')}})} +placement=classify();place(true); +if(active)tip.setAttribute('data-open','')}; +const scheduleHide=()=>{if(timer)return;timer=setTimeout(()=>{timer=null; +if(active!==document.activeElement)hide()},150)}; +view.querySelectorAll('title').forEach(el=>el.remove()); +view.querySelectorAll('[data-kumo-tip]').forEach(el=>{ +const light=el.querySelector('.kumo-surface').cloneNode(); +light.classList.add('kumo-row-light');light.removeAttribute('fill'); +light.removeAttribute('stroke');el.insertBefore(light,el.querySelector('text')); +el.addEventListener('pointerenter',e=>{if(e.pointerType!=='touch')show(el)},{signal}); +el.addEventListener('pointerleave',scheduleHide,{signal}); +el.addEventListener('focus',()=>show(el),{signal}); +el.addEventListener('blur',scheduleHide,{signal})}); +host.addEventListener('pointerleave',scheduleHide,{signal}); +document.addEventListener('pointerout',e=>{ +if(!e.relatedTarget)scheduleHide() +},{signal}); + +""" + +_CAMERA = r""" +const selector=host.querySelector('.kumo-graph-layout'); +const label=host.querySelector('.kumo-graph-scale'); +const status=host.querySelector('[role=status]'); +const enter=host.querySelector('.kumo-graph-enter'); +const exit=host.querySelector('.kumo-graph-exit'); +const payload=data; +const layouts=payload.layouts; +let selected=payload.initial, scale=1, fitted=true, full=false, inlineCamera=null; +let layoutWidth=Number(svg.getAttribute('width'))||304; +let layoutHeight=Number(svg.getAttribute('height'))||112; +let resizeFrame=0, panX=0, panY=0, isDragging=false, drag=null; +let viewWidth=0, viewHeight=0; +const cards=new Map(Array.from(svg.querySelectorAll(':scope>[data-kumo-card]'), +el=>[el.dataset.kumoCard,el])); +const controls=host.querySelectorAll('button,input'); +selector.querySelectorAll('input').forEach(el=>{ +el.name=tip.id+'-layout';el.checked=el.value===selected}); +if(Object.keys(layouts).length<2)selector.hidden=true; + +// Only applyLayout changes graph geometry. Camera actions never call it. +const applyLayout=name=>{ +const layout=layouts[name];if(!layout)return; +hide(); +for(const [name,position] of Object.entries(layout.positions)){ +const card=cards.get(name); +if(card)card.setAttribute('transform',`translate(${position[0]} ${position[1]})`)} +edges.forEach((edge,index)=>edge.setAttribute('d',layout.paths[index])); +svg.querySelectorAll(':scope>circle,:scope>.kumo-graph-handles').forEach(el=>el.remove()); +const handles=document.createElementNS(svg.namespaceURI,'g'); +handles.setAttribute('class','kumo-graph-handles'); +for(const [x,y] of layout.handles){ +const circle=document.createElementNS(svg.namespaceURI,'circle'); +for(const [key,value] of Object.entries({cx:x,cy:y,r:4,fill:'var(--kumo-muted)'})) +circle.setAttribute(key,value);handles.append(circle)} +svg.append(handles);layoutWidth=layout.width;layoutHeight=layout.height; +svg.setAttribute('viewBox',`0 0 ${layoutWidth} ${layoutHeight}`); +svg.setAttribute('width',layoutWidth);svg.setAttribute('height',layoutHeight); +svg.querySelectorAll(':scope>rect').forEach(rect=>{ +rect.setAttribute('width',layoutWidth);rect.setAttribute('height',layoutHeight)}); +selected=name;host.dataset.layout=name; +}; +const minHeight=svg.querySelector('[data-kumo-tip*="Sample values:"]')?320:240; +const inlineHeight=()=>Math.min(720,Math.max(minHeight,layoutHeight+32)); +const fitScale=()=>{ +const side=348; +const padding=view.clientWidth>=side*2+320?side:16; +return Math.min(1,Math.max(1,view.clientWidth-padding*2)/layoutWidth, +Math.max(1,view.clientHeight-32)/layoutHeight)}; +const resizeScene=()=>{ +viewWidth=view.clientWidth;viewHeight=view.clientHeight; +svg.style.width=layoutWidth*scale+'px';svg.style.height=layoutHeight*scale+'px'; +scene.style.transform=`translate(${panX}px,${panY}px)`; +view.style.backgroundSize=`${16*scale}px ${16*scale}px`; +view.style.backgroundPosition=`${panX}px ${panY}px`; +view.style.backgroundImage=`radial-gradient(circle at ${scale}px ${scale}px, +var(--kumo-grid) ${.7*scale}px,transparent ${.8*scale}px)`; +label.textContent=Math.round(scale*100)+'%';host.dataset.scale=String(scale); +}; +const panBy=(dx,dy)=>{fitted=false; +panX+=dx;panY+=dy;resizeScene()}; +const fit=()=>{ +hide();fitted=true;scale=fitScale(); +panX=(view.clientWidth-layoutWidth*scale)/2; +panY=(view.clientHeight-layoutHeight*scale)/2;resizeScene(); +}; +const zoom=(next,x=view.clientWidth/2,y=view.clientHeight/2)=>{ +hide();fitted=false; +const graphX=(x-panX)/scale, graphY=(y-panY)/scale; +scale=Math.max(Math.min(.1,fitScale()),Math.min(4,next)); +panX=x-graphX*scale;panY=y-graphY*scale;resizeScene(); +}; +view.addEventListener('pointerdown',e=>{ +if(e.button!==0||e.pointerType==='touch')return; +hide(); +const rect=view.getBoundingClientRect(); +drag={id:e.pointerId,x:e.clientX,y:e.clientY,panX,panY, +sx:rect.width/view.clientWidth,sy:rect.height/view.clientHeight}; +view.setPointerCapture(e.pointerId); +},{capture:true,signal}); +view.addEventListener('pointermove',e=>{ +if(!drag||e.pointerId!==drag.id)return; +const dx=e.clientX-drag.x, dy=e.clientY-drag.y; +if(!isDragging&&Math.hypot(dx,dy)<4)return; +if(!isDragging){isDragging=true;hide();view.dataset.dragging='true'} +e.preventDefault();panBy(drag.panX+dx/drag.sx-panX,drag.panY+dy/drag.sy-panY); +},{capture:true,signal}); +const stopDrag=e=>{ +if(e&&drag&&e.pointerId!==drag.id)return; +const id=drag?.id;drag=null;isDragging=false;delete view.dataset.dragging; +if(id!==undefined&&view.hasPointerCapture(id))view.releasePointerCapture(id); +}; +for(const event of ['pointerup','pointercancel','lostpointercapture']) +view.addEventListener(event,stopDrag,{capture:true,signal}); +view.addEventListener('keydown',e=>{ +if(e.target!==view)return; +const delta={ArrowLeft:[40,0],ArrowRight:[-40,0], +ArrowUp:[0,40],ArrowDown:[0,-40]}[e.key]; +if(delta){e.preventDefault();hide();panBy(...delta)} +},{signal}); +selector.addEventListener('change',event=>{ +status.hidden=true;applyLayout(event.target.value);if(!full)view.style.height=inlineHeight()+'px';fit(); +},{signal}); +host.querySelector('.kumo-graph-fit').addEventListener('click',fit,{signal}); +host.querySelector('.kumo-graph-zoom-in').addEventListener('click',()=>zoom(scale*1.25),{signal}); +host.querySelector('.kumo-graph-zoom-out').addEventListener('click',()=>zoom(scale/1.25),{signal}); +view.addEventListener('wheel',e=>{ +if(!e.ctrlKey&&!e.metaKey)return; +e.preventDefault();const box=view.getBoundingClientRect(); +const delta=e.deltaY*(e.deltaMode===1?16:e.deltaMode===2?view.clientHeight:1); +zoom(scale*Math.exp(-delta*.004), +(e.clientX-box.left)*view.clientWidth/box.width, +(e.clientY-box.top)*view.clientHeight/box.height); +},{passive:false,signal}); +enter.addEventListener('click',()=>{ +status.hidden=true; +// Capture the inline camera before fullscreen changes viewport dimensions. +inlineCamera={selected,scale,fitted,panX,panY}; +const denied=()=>{status.textContent='Browser full screen is unavailable in this notebook. '+ +'Use the graph controls here.';status.hidden=false;inlineCamera=null}; +try{Promise.resolve(host.requestFullscreen()).catch(denied)} +catch(error){denied()} +},{signal}); +exit.addEventListener('click',()=>{ +if(host.matches(':fullscreen'))document.exitFullscreen(); +},{signal}); +host.addEventListener('fullscreenchange',()=>{ +const entering=host.matches(':fullscreen'); +if(entering===full)return; +hide();full=entering; +if(full){ +if(!inlineCamera)inlineCamera={selected,scale,fitted,panX,panY}; +fit();exit.focus({preventScroll:true}); +}else{ +view.style.height=inlineHeight()+'px'; +if(inlineCamera&&inlineCamera.selected===selected){ +scale=inlineCamera.scale;fitted=inlineCamera.fitted; +panX=inlineCamera.panX;panY=inlineCamera.panY;resizeScene(); +}else fit(); +inlineCamera=null;enter.focus({preventScroll:true}); +} +},{signal}); +document.addEventListener('keydown',e=>{if(e.key==='Escape')hide()},{signal}); +document.addEventListener('scroll',()=>place(),{capture:true,signal}); +window.addEventListener('resize',hide,{signal}); +const resizeObserver=new ResizeObserver(()=>{ +if(signal.aborted)return; +cancelAnimationFrame(resizeFrame); +resizeFrame=requestAnimationFrame(()=>{ +if(view.clientWidth===viewWidth&&view.clientHeight===viewHeight)return; +if(fitted)fit();else{hide();resizeScene()} +}); +}); +onDispose(()=>{ +stopDrag();hide();cancelAnimationFrame(resizeFrame);resizeObserver.disconnect(); +}); +view.style.height=inlineHeight()+'px';host.dataset.layout=selected; +fit();resizeObserver.observe(view);controls.forEach(el=>el.disabled=false); +""" + + +_EXPORT = r""" +const exportButton=host.querySelector('.kumo-graph-export'); +const options=host.querySelector('.kumo-graph-export-options'); +const command=options.querySelector('code'); +options.id=widgetId+'-exports';exportButton.setAttribute('aria-controls',options.id); +exportButton.setAttribute('aria-expanded','false'); +const closeExport=()=>{options.hidden=true;exportButton.setAttribute('aria-expanded','false')}; +exportButton.addEventListener('click',()=>{ +if(!options.hidden){closeExport();return} +status.hidden=true; +command.textContent=`_ = graph.visualize('graph.pdf', layout='${selected}')`; +options.hidden=false;exportButton.setAttribute('aria-expanded','true'); +command.focus({preventScroll:true}); +},{signal}); +selector.addEventListener('change',closeExport,{signal}); +host.addEventListener('keydown',event=>{ +if(event.key==='Escape'&&!options.hidden){closeExport();exportButton.focus()} +},{signal}); +""" + + +def viewer(svg: str, layouts: dict, initial_layout: str, static_svg: str) -> dict: + """Prepare shared controls, behavior, and precomputed geometries.""" + options = ''.join( + '' + for name, label in [('standard', 'Standard'), ('compact', 'Compact')] + if name in layouts) + if not options: + options = ('') + fit_icon = ('M 3 8 V 3 H 8 M 16 3 H 21 V 8 M 21 16 V 21 H 16 ' + 'M 8 21 H 3 V 16 M 7 7 H 17 V 17 H 7 Z') + enter_icon = ('M 9 3 H 3 V 9 M 15 3 H 21 V 9 M 3 15 V 21 H 9 ' + 'M 21 15 V 21 H 15') + exit_icon = ('M 3 9 H 9 V 3 M 15 3 V 9 H 21 M 9 21 V 15 H 3 ' + 'M 21 15 H 15 V 21') + export_icon = ('M 12 3 V 15 M 7 10 L 12 15 L 17 10 ' + 'M 4 16 V 21 H 20 V 16') + + body = ( + '' + '
' + + render_static_svg(static_svg, + 'Rerun this cell to activate animation.', + 'Table relationships') + '
') + script = ( + 'const view=host.querySelector(".kumo-graph-view");' + 'const svg=view.querySelector("svg"),scene=view.firstElementChild;' + 'const live=host.querySelector(".kumo-graph-live");' + 'const fallback=host.querySelector(".kumo-graph-static");' + 'onDispose(()=>{live.style.display="none";fallback.hidden=false;' + 'host.querySelector(".kumo-graph-toolbar").hidden=true;' + 'host.querySelector(".kumo-graph-export-options").hidden=true});' + 'live.style.display="contents";view.hidden=false;' + 'const edges=Array.from(view.querySelectorAll(".kumo-edge"));' + + _HOVER + _CAMERA + _EXPORT + + 'host.querySelector(".kumo-graph-toolbar").hidden=false;' + 'fallback.hidden=true;' + 'view.tabIndex=0;view.setAttribute("aria-label","Pannable graph visualization");') + return dict(body=body, style=_STYLE, script=script, + data={'layouts': layouts, 'initial': initial_layout}, + class_name='kumo-graph') diff --git a/packages/kumo-engine/kumo/engine/_widget.py b/packages/kumo-engine/kumo/engine/_widget.py new file mode 100644 index 000000000..cc31329f3 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_widget.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared notebook theme, controls, and output cleanup.""" + +import json +import os +from base64 import b64encode +from hashlib import sha256 +from html import escape +from string import Template +from types import MappingProxyType +from uuid import uuid4 + +THEME = MappingProxyType({ + 'font': 'Arial, Helvetica, sans-serif', + 'foreground': '#202020', 'muted': '#687078', 'accent': '#386000', + 'heading': '#386000', 'header_background': '#f4f8ec', + 'tooltip_header_border': '#e3e7df', 'grid': '#d9ddd5', + 'edge': '#a1a1a1', 'edge_active': '#6e9947', 'focus': '#598d13', + 'card_border': '#c5c9cd', 'card_background': 'white', + 'role_background': '#f7f7f7', 'surface': '#f5f6f4', + 'surface_border': '#e1e4df', 'row_highlight': '#edf6df', + 'chrome_hover': '#e8e8e8', 'chrome_foreground': '#555', + 'chrome_shadow': '0 1px 4px #0000001a', 'background': 'white', + 'tooltip_background': 'white', 'tooltip_border': '#d1d6cb', + 'tooltip_shadow': '0 8px 28px #00000018,0 1px 4px #0000000a', +}) +_DARK = { + 'foreground': '#f2f5f7', 'muted': '#c1cad2', 'accent': '#b2e36b', + 'heading': '#ffffff', 'header_background': '#3b464f', + 'tooltip_header_border': '#56636f', 'grid': '#262e35', + 'edge': '#94a2af', 'edge_active': '#b2e36b', 'focus': '#d6e0e8', + 'card_border': '#56636f', 'card_background': '#272e35', + 'role_background': '#333d46', 'surface': '#35404a', + 'surface_border': '#53616d', 'row_highlight': '#485661', + 'chrome_hover': '#3b464f', 'chrome_foreground': '#d7dee5', + 'background': '#111416', 'tooltip_background': '#293139', + 'tooltip_border': '#71808d', +} +CSS_THEME = {key: f'var(--kumo-{key.replace("_", "-")})' + for key in THEME if key != 'font'} | {'font': THEME['font']} + + +def is_snowflake_workspace() -> bool: + """Match the active Workspace notebook runtime, not installed connectors.""" + if not os.environ.get('SNOWFLAKE_WORKSPACE_ID'): + return False + try: + from IPython import get_ipython + except ImportError: + return False + extensions = getattr(getattr(get_ipython(), 'extension_manager', None), + 'loaded', ()) + return ('snowflake_notebook_utils.ipython_execution_context_monitor_extension' + in extensions) + + +def render_static_svg(svg: str, message: str, label: str) -> str: + """Render a script-free SVG image and escaped explanatory text.""" + return (f'{escape(label)}' + f'

{escape(message)}

') + + +_STYLE = Template(r""" +:where(.kumo-widget){position:relative;display:flex;flex-direction:column; +max-width:100%;padding:8px;box-sizing:border-box;background:$background; +color:$foreground;font:14px $font} +.kumo-widget [hidden]{display:none!important} +.kumo-widget:fullscreen{filter:none!important} +:where(.kumo-widget-toolbar){display:flex;align-items:center;gap:6px; +flex:none;margin-bottom:16px} +:where(.kumo-widget) .kumo-widget-button{box-sizing:border-box;display:inline-flex; +align-items:center;justify-content:center;flex:0 0 32px; +width:32px!important;min-width:32px!important;max-width:32px!important; +height:32px!important;min-height:32px!important;max-height:32px!important; +padding:6px;margin:0;background:$background;color:$chrome_foreground; +border:0;border-radius:16px;cursor:pointer;font:18px/18px $font} +.kumo-widget-toolbar>.kumo-widget-button{box-shadow:$chrome_shadow} +:where(.kumo-widget) .kumo-widget-button:hover{background:$chrome_hover} +:where(.kumo-widget) .kumo-widget-button:focus-visible{ +outline:2px solid $focus;outline-offset:2px} +:where(.kumo-widget) .kumo-widget-button>svg{display:block;flex:none; +width:18px!important;height:18px!important} +""").substitute(CSS_THEME) +_STYLE = ( + ':where(.kumo-widget){color-scheme:light dark;' + ''.join( + f'--kumo-{key.replace("_", "-")}:light-dark({THEME[key]},{dark});' + for key, dark in _DARK.items()) + + '--kumo-chrome-shadow:0 1px 4px light-dark(#0000001a,#00000040);' + '--kumo-tooltip-shadow:0 8px 28px light-dark(#00000018,#00000050),' + '0 1px 4px light-dark(#0000000a,#00000030);}' + '.kumo-widget[data-kumo-theme="light"]{color-scheme:light}' + '.kumo-widget[data-kumo-theme="dark"]{color-scheme:dark}' + + _STYLE) + + +def button(cls: str, label: str, icon: str = '', text: str = '') -> str: + """Build a labeled button, disabled until initialization.""" + graphic = (f'' + if icon else escape(text)) + label = escape(label) + return (f'') + + +_INITIALIZE = r""" +function initialize(host,data,feature,observeRemoval=true){ +const widgetId=host.id,controller=new AbortController(),signal=controller.signal; +const cleanups=[],onDispose=callback=>cleanups.push(callback); +let disposed=false; +const dispose=()=>{if(disposed)return;disposed=true; +controller.abort();removalObserver.disconnect(); +host.dataset.disposed="true"; +for(const cleanup of cleanups.splice(0)){try{cleanup()}catch(error){console.error(error)}}}; +const removalObserver=new MutationObserver(records=>{ +if(records.some(record=>record.removedNodes.length)&&!host.isConnected)dispose();}); +if(observeRemoval)removalObserver.observe(host.getRootNode(),{childList:true,subtree:true}); +try{feature({host,widgetId,data,signal,onDispose})}catch(error){dispose();throw error} +return dispose; +} +""" + + +def render_anywidget(body: str, *, style: str = '', script: str = '', + data: dict | None = None, class_name: str = '', + namespace: str = ''): + """Create an optional AFM widget with independent state for each view.""" + try: + import anywidget + import traitlets + from packaging.specifiers import SpecifierSet + + if str(getattr(anywidget, '__version__', '0')) not in SpecifierSet( + '>=0.9,<1'): + raise ImportError('Incompatible anywidget version.') + except (ImportError, ValueError) as error: + raise ImportError( + 'Interactive widget display requires anywidget>=0.9,<1 and ' + 'ipywidgets. Snowflake Workspace Runtime 2.6+ supplies these ' + 'packages; restore the runtime packages or install compatible ' + 'versions according to your environment policy. Static SVG, PDF, ' + 'and Markdown exports remain available through ' + 'graph.visualize("graph.svg"), graph.visualize("graph.pdf"), ' + 'or graph.visualize("graph.md").') from error + + class NotebookWidget(anywidget.AnyWidget): + _css = (_STYLE + style + + ':where(.kumo-widget-inherit-theme){color-scheme:inherit}') + _esm = (_INITIALIZE + 'export default {render({model,el}){' + 'const host=document.createElement("div");' + 'host.id="kumo-"+crypto.randomUUID();' + 'host.className=' + json.dumps( + 'kumo-widget nodarkmode ' + class_name) + ';' + 'const namespace=model.get("namespace"),body=model.get("body");' + 'host.innerHTML=namespace?body.replaceAll(namespace,host.id):body;' + 'el.append(host);' + 'let initialized=false,dispose;const mount=()=>{' + 'if(initialized||!host.isConnected)return;initialized=true;' + 'mounted.disconnect();' + 'const scheme=getComputedStyle(el).colorScheme;' + 'if(scheme&&scheme!=="normal")' + 'host.classList.add("kumo-widget-inherit-theme");' + 'try{dispose=initialize(host,model.get("data"),' + '({host,widgetId,data,signal,onDispose})=>{' + script + '},false);}' + 'catch(error){host.remove();throw error}};' + 'const mounted=new ResizeObserver(mount);mounted.observe(host);mount();' + 'return()=>{initialized=true;mounted.disconnect();' + 'dispose?.();host.remove()};}};') + body = traitlets.Unicode().tag(sync=True) + data = traitlets.Dict().tag(sync=True) + namespace = traitlets.Unicode().tag(sync=True) + + NotebookWidget.__name__ += '_' + sha256( + (NotebookWidget._esm + NotebookWidget._css).encode()).hexdigest() + return NotebookWidget(body=body, data=data or {}, namespace=namespace) + + +def render_widget(body: str, *, style: str = '', script: str = '', + data: dict | None = None, class_name: str = '', + attributes: dict[str, str] | None = None, + namespace: str = '') -> str: + """Compose trusted feature markup/code with escaped data and local cleanup. + + Feature scripts use host, widgetId, data, signal and onDispose(callback). + Root attributes are data-* or aria-* names; controls are enabled by features. + """ + widget_id = 'kumo-' + uuid4().hex + if namespace: + body = body.replace(namespace, widget_id) + attributes = attributes or {} + if any(not name.startswith(('data-', 'aria-')) or not all( + c.isascii() and (c.isalnum() or c == '-') for c in name) + for name in attributes): + raise ValueError('Widget attributes must be data-* or aria-* names.') + attrs = ''.join(f' {key}="{escape(value)}"' + for key, value in attributes.items()) + payload = json.dumps(data or {}, ensure_ascii=True, separators=(',', ':')) + for char, encoded in [('<', r'\u003c'), ('>', r'\u003e'), ('&', r'\u0026')]: + payload = payload.replace(char, encoded) + return ( + f'
' + f'{body}' + f'
') diff --git a/packages/kumo-engine/kumo/engine/graph.py b/packages/kumo-engine/kumo/engine/graph.py index b5ecf3299..379ceacee 100644 --- a/packages/kumo-engine/kumo/engine/graph.py +++ b/packages/kumo-engine/kumo/engine/graph.py @@ -75,6 +75,7 @@ snapshot_scratch_dir, ) from kumo.engine._storage import network_storage +from kumo.engine._visualization import SVG from kumo.engine.table import Table _warned_scratch: set[Path] = set() @@ -345,6 +346,63 @@ def validate(self) -> None: """ self.native_graph + def visualize(self, path: str | Path | None = None, *, + show_cols: bool = False, show_samples: bool = False, + layout: str = 'auto') -> SVG: + """Display pannable table relationships in notebooks and return SVG. + + Args: + path: Optional local file, overwritten if it exists. A .pdf suffix + writes a vector graph and full schema/connection index. + A .md or .markdown suffix writes a Mermaid relationship + diagram and complete Markdown schema tables; + other paths receive UTF-8 SVG. Paths refer to the filesystem + where Kumo Engine runs. + show_cols: Add columns up to 10 rows per table (5 above 10 tables). + Primary keys, foreign keys, and declared time (TS) and end-time + (TE) columns are always included, even above this limit. + False shows only these columns. + show_samples: Show up to six non-null examples per column in notebook + tooltips, in two columns and up to three rows, selected from + up to 30 preview rows per table. False (default) reads no + source data. Previews may + run warehouse queries and incur costs; returned row limits + do not bound bytes scanned. Unavailable previews are labeled. + Samples become part of saved notebook output, never file exports. + layout: Initial arrangement: auto, standard, or compact. + Auto chooses from graph structure, independently of viewport + size. Standard preserves the layered relationship view; + compact balances crowded groups. + Notebook views offer a selector to compare arrangements. + + Uses declared schemas. More than 30 tables, self-relations, + and missing schema/key definitions raise ValueError. + Display automatically uses environment-provided widget packages in + detected Snowflake Workspace notebooks and HTML in other hosts. + The optional as_widget() helper explicitly selects the widget view. + Full screen is available when the notebook host permits it. + Dragging, full screen, Fit, and zoom preserve the selected arrangement. + SVG exports use the initial layout and contain no interactive controls. + The export control shows Python commands to save PDF and Markdown + files in the notebook filesystem. PDF exports include all declared + columns in the index, even when the diagram omits them. + Markdown diagrams use the Markdown viewer's Mermaid layout rather than + the notebook arrangement; schema tables remain readable without Mermaid. + """ + from kumo.engine._visualization import render + + svg = render(self._named_tables, self.edges, show_cols=show_cols, + show_samples=show_samples, layout=layout) + if path is not None: + destination = Path(path) + if destination.suffix.lower() == '.pdf': + destination.write_bytes(svg._pdf(svg._initial_layout)) + elif destination.suffix.lower() in ('.md', '.markdown'): + destination.write_text(svg._markdown(), encoding='utf-8') + else: + destination.write_text(svg, encoding='utf-8') + return svg + def snapshot(self) -> GraphSnapshot: r"""Materialize this graph: read each table's connector, run the result through `kumo-core` to build column stats and the on-disk diff --git a/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb b/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb index 003b9428f..b0222c0c5 100644 --- a/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb +++ b/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb @@ -199,6 +199,8 @@ "metadata": {}, "outputs": [], "source": [ + "from IPython.display import display\n", + "\n", "from kumo.engine import Edge\n", "\n", "graph = kumo.Graph(\n", @@ -209,6 +211,7 @@ " ],\n", ")\n", "graph.validate()\n", + "display(graph.visualize())\n", "print('graph valid. snapshot_id:', graph.id)" ] }, diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 959cef60c..5d9437ed7 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -2,15 +2,28 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json +import re +from base64 import b64decode from dataclasses import replace from pathlib import Path -from unittest.mock import patch +from unittest.mock import Mock, PropertyMock, patch +from xml.etree import ElementTree import pytest from kumoapi.graph import Edge -from kumoapi.source_table import FileType, S3RelativeSource, S3SourceTable +from kumoapi.source_table import ( + CustomSQLSourceTable, + FileType, + S3RelativeSource, + S3SourceTable, +) from kumoapi.table import Column, TableDefinition from kumoapi.typing import Dtype, Stype +from matplotlib import rc_context, rcParams, style +from matplotlib.backends.backend_pdf import PdfPages, RendererPdf +from matplotlib.colors import to_rgba +from matplotlib.text import Text from kumo.engine import FileConnector, Graph, S3Connector, Table from kumo.engine._portable import PortableGraphError @@ -62,6 +75,35 @@ def tables(request, tmp_path) -> list[Table]: return _s3_tables() +@pytest.fixture +def pdf_pages(monkeypatch): + pages = [] + savefig = PdfPages.savefig + + def record(document, figure, *args, **kwargs): + assert figure.get_facecolor() == (1, 1, 1, 1) + facecolor = kwargs.get('facecolor', rcParams['savefig.facecolor']) + assert to_rgba(figure.get_facecolor() if facecolor == 'auto' + else facecolor) == (1, 1, 1, 1) + assert not kwargs.get('transparent', rcParams['savefig.transparent']) + if not pages: + figure.set_dpi(72) # PDF coordinates and font sizes are in points. + renderer = RendererPdf(None, 72, figure.get_figheight(), + figure.get_figwidth()) + texts = figure.axes[0].texts + for role in texts: + if re.fullmatch(r'(PK|FK|TS|TE)(/(PK|FK|TS|TE))*', role.get_text()): + bounds = role.get_window_extent(renderer) + assert all(not bounds.overlaps(other.get_window_extent(renderer)) + for other in texts if other is not role) + pages.append('\n'.join(text.get_text() for text in + figure.findobj(match=Text))) + return savefig(document, figure, *args, **kwargs) + + monkeypatch.setattr(PdfPages, 'savefig', record) + return pages + + @pytest.mark.parametrize( 'edge, missing', [ @@ -162,3 +204,702 @@ def test_omitted_self_relation_preserves_column_type(tables, original_stype): if col['name'] == 'user_id')['stype'] == original_stype assert next(col for col in source.definition.cols if col.name == 'user_id').stype == original_stype + + +@pytest.mark.parametrize('alias, key, collision', [ + (' & customers', 'customer<&"账户', False), + ('PURCHASES_AND_TRANSACTIONS', 'O' * 22, False), + ('Ж' * 27, 'Ш' * 22, False), + ('sales data', 'order id', False), + ('sales\tdata', 'order\tid', False), + ('sales\rdata', 'order\rid', False), + ('sales\ndata', 'order\nid', False), + ('sales\tdata', 'order\tid', True), +]) +@pytest.mark.parametrize('suffix', ['svg', 'pdf']) +def test_visualize_exports_escaped_svg_without_source_access( + tmp_path, alias, key, collision, suffix, pdf_pages): + users, events = _s3_tables() + events = replace( + events, definition=replace( + events.definition, cols=[ + events.definition.cols[0], + Column(key, Stype.ID, Dtype.int64) + ])) + tables = {'purchases': events, alias: users} + if collision: + tables[alias.replace('\t', ' ').replace('\r', ' ')] = users + graph = Graph(tables, [Edge('purchases', key, alias)]) + destination = tmp_path / f'relationships.{suffix}' + with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, + side_effect=AssertionError('source accessed')), \ + patch.object(Table, 'infer_metadata', + side_effect=AssertionError('inference started')), \ + patch.object(Graph, 'snapshot', + side_effect=AssertionError('snapshot started')): + svg = graph.visualize(destination, show_samples=False) + assert isinstance(svg, str) + assert svg._repr_svg_() == svg + if suffix == 'pdf': + data = destination.read_bytes() + assert data.startswith(b'%PDF-') and data.rstrip().endswith(b'%%EOF') + assert len(re.findall(rb'/Type /Page\b', data)) == len(pdf_pages) >= 2 + assert '\t' not in pdf_pages[0] + index = '\n'.join(pdf_pages[1:]) + pdf_alias = alias.replace('账户', r'\u8d26\u6237').replace( + '\r', r'\r').expandtabs(4).replace('\n', '') + pdf_key = key.replace('账户', r'\u8d26\u6237').replace( + '\r', r'\r').expandtabs(4).replace('\n', '') + assert pdf_alias in index.replace('\n', '') + assert pdf_key in index.replace('\n', '') + connection = f'purchases.{key} → {alias}.id'.replace( + '账户', r'\u8d26\u6237').replace('\r', r'\r').expandtabs(4) + assert connection.replace('\n', '') in index.replace('\n', '') + assert 's3://schema-only-fixture' not in index + else: + assert destination.read_bytes().decode('utf-8') == svg + root = ElementTree.fromstring(svg) + assert root.tag == '{http://www.w3.org/2000/svg}svg' + html = ElementTree.fromstring(svg._repr_html_()) + embedded = html.find('.//{http://www.w3.org/2000/svg}svg') + assert re.sub(rb'kumo-[0-9a-f]{32}', b'kumo-ID', + ElementTree.tostring(embedded)) == re.sub( + rb'kumo-[0-9a-f]{32}', b'kumo-ID', + ElementTree.tostring(root)) + assert html.find('.//div[@role="region"]').get('tabindex') is None + assert html.find('./div[@class="kumo-graph-live"]').get('style') == ( + 'display:none') + assert 'hidden' in html.find( + './/div[@class="kumo-widget-toolbar kumo-graph-toolbar"]').attrib + assert 'hidden' in html.find('.//div[@role="region"]').attrib + assert html.find('./div[@class="kumo-graph-static"]/p').text == ( + 'Rerun this cell to activate animation.') + text = ''.join(element.text or '' + for element in root.iter('{http://www.w3.org/2000/svg}text') + if element.get('{http://www.w3.org/XML/1998/namespace}space') + == 'preserve') + assert alias.replace('\r', '\n') in text + assert key.replace('\r', '\n') in text + if alias.isupper(): + visible = [ + element.text + for element in root.iter('{http://www.w3.org/2000/svg}text') + ] + assert alias not in visible and key not in visible + assert not root.findall('.//{http://www.w3.org/2000/svg}script') + assert all(not name.lower().startswith('on') for element in root.iter() + for name in element.attrib) + tips = root.findall('.//{http://www.w3.org/2000/svg}g[@data-kumo-tip]') + assert len(tips) == 5 + 2 * collision # Each extra table has a PK row. + assert all(tip.get('tabindex') == '0' for tip in tips) + details = [tip.get('data-kumo-tip') for tip in tips] + assert any(f'References: {alias}.id' in tip for tip in details) + assert any('Type: int64\nSemantic type: ID\nRole: FK' in tip + for tip in details) + assert any(f'Connected tables:\n{alias}' in tip for tip in details) + edge = root.find('.//{http://www.w3.org/2000/svg}path[@class="kumo-edge"]') + assert edge.get('data-dst-table') == alias + assert edge.get('data-src-column') == key + ids = {table['name']: f'table_{i}' + for i, table in enumerate(svg._index)} + relationship = next(line for line in svg._markdown().splitlines() + if ' -->|' in line) + assert relationship.startswith(f' {ids["purchases"]} -->|') + assert relationship.endswith(f'| {ids[alias]}') + assert 's3://schema-only-fixture' not in svg + + +@pytest.mark.parametrize('count, limit', [(10, 10), (11, 5)]) +@pytest.mark.parametrize('show_cols', [False, True]) +def test_visualize_bounds_column_detail(count, limit, show_cols): + users, _ = _s3_tables() + extras = [Column(f'feature_{i:02}', Stype.numerical, Dtype.int64) + for i in range(17)] + table = replace(users, definition=replace( + users.definition, time_col='event_time', end_time_col='zz_end_time', + cols=[*users.definition.cols, *extras, + Column('event_time', Stype.timestamp, Dtype.date), + Column('zz_end_time', Stype.timestamp, Dtype.date)])) + graph = Graph({f'table_{i:02}': table for i in range(count)}) + root = ElementTree.fromstring(graph.visualize(show_cols=show_cols)) + visible = limit - 3 if show_cols else 0 + expected = {'id', 'event_time', 'zz_end_time'} | { + col.name for col in extras[:visible]} + for card in root.findall('./{http://www.w3.org/2000/svg}g[@data-kumo-card]'): + rows = card.findall('.//{http://www.w3.org/2000/svg}g[@data-kumo-column]') + assert {row.get('data-kumo-column') for row in rows + if row.get('data-kumo-column')} == expected + time = next(row for row in rows if row.get('data-kumo-column') == 'event_time') + assert 'Role: TS' in time.get('data-kumo-tip') + end = next(row for row in rows if row.get('data-kumo-column') == 'zz_end_time') + assert 'Role: TE' in end.get('data-kumo-tip') + assert re.search(rf'\+\s*{len(extras) - visible} (other )?columns', + ''.join(card.itertext())) + + +@pytest.mark.parametrize('time_column, end_time_column, show_cols, time_roles', [ + (None, None, False, {}), (None, None, True, {}), + ('occurred_at', None, False, {'occurred_at': 'TS'}), + ('event_id', None, False, {'event_id': 'PK/FK/TS'}), + ('foreign_00', None, True, {'foreign_00': 'FK/TS'}), + (None, 'ended_at', False, {'ended_at': 'TE'}), + ('occurred_at', 'ended_at', True, {'occurred_at': 'TS', 'ended_at': 'TE'}), + ('event_id', 'event_id', False, {'event_id': 'PK/FK/TS/TE'}), + ('occurred_at', 'event_id', False, {'occurred_at': 'TS', 'event_id': 'PK/FK/TE'}), + ('occurred_at', 'foreign_00', True, {'occurred_at': 'TS', 'foreign_00': 'FK/TE'}), + ('n' * 21, 'n' * 21, False, {'n' * 21: 'PK/FK/TS/TE'}), + ('occurred_at', 'n' * 23, True, {'occurred_at': 'TS', 'n' * 23: 'PK/FK/TE'}), + ('账户' * 12, '账户' * 12, True, {'账户' * 12: 'PK/FK/TS/TE'}), + ('occurred_at', '账户' * 12, False, {'occurred_at': 'TS', '账户' * 12: 'PK/FK/TE'}), +]) +def test_visualize_retains_keys_beyond_column_limit( + time_column, end_time_column, show_cols, time_roles, tmp_path, pdf_pages): + users, events = _s3_tables() + pkey = next((name for name, role in time_roles.items() if 'PK' in role), + 'event_id') + keys = [Column(f'foreign_{i:02}', Stype.ID, Dtype.int64) for i in range(12)] + events = replace( + events, definition=replace(events.definition, + pkey=pkey, + time_col=time_column, + end_time_col=end_time_column, + cols=[Column(pkey, Stype.ID, Dtype.int64), *keys, + Column('occurred_at', Stype.timestamp, + Dtype.date), + Column('ended_at', Stype.timestamp, + Dtype.date)])) + edges = [Edge('events', column.name, 'users') for column in keys] + if pkey in (time_column, end_time_column): + edges.append(Edge('events', pkey, 'users')) + graph = Graph([users, events], edges) + root = ElementTree.fromstring( + graph.visualize(tmp_path / 'roles.pdf', show_cols=show_cols)) + card = root.find('./{http://www.w3.org/2000/svg}g[@data-kumo-card="events"]') + rows = card.findall('.//{http://www.w3.org/2000/svg}g[@data-kumo-column]') + expected = {pkey} | {column.name for column in keys} | time_roles.keys() + assert {row.get('data-kumo-column') for row in rows + if row.get('data-kumo-column')} == expected + for column, role in time_roles.items(): + row = root.find('.//{http://www.w3.org/2000/svg}g' + f'[@data-kumo-column="{column}"]') + assert f'Role: {role}' in row.get('data-kumo-tip') + + +@pytest.mark.parametrize('layout', ['standard', 'compact']) +def test_visualize_layout_is_deterministic_with_cycles_and_isolates(layout): + _, table = _s3_tables() + tables = { + name: replace(table, definition=replace( + table.definition, pkey=key, + cols=[*table.definition.cols, Column(key, Stype.ID, Dtype.int64)])) + for name, key in [('a', 'b.c'), ('a.b', 'c')] + } | {'c': table, 'isolated': table} + edges = [ + Edge('c', 'user_id', 'a.b'), Edge('c', 'event_id', 'a'), + Edge('a.b', 'user_id', 'c'), Edge('a', 'event_id', 'c'), + ] + svg = Graph(tables, edges).visualize(layout=layout) + other = Graph(dict(reversed(list(tables.items()))), + list(reversed(edges))).visualize(layout=layout) + assert re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', + svg) == re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', other) + ids = lambda value: set(re.findall(r' id="([^"]+)"', value)) + assert ids(svg).isdisjoint(ids(other)) + root = ElementTree.fromstring(svg) + text = ''.join(root.itertext()) + assert all(name in text for name in tables) + assert all(float(value) > 0 for value in root.attrib['viewBox'].split()[2:]) + titles = { + element.text + for element in root.iter('{http://www.w3.org/2000/svg}title') + } + assert all(f'{edge.src_table}.{edge.fkey} → {edge.dst_table}.' + f'{tables[edge.dst_table].definition.pkey}' in titles + for edge in edges) + paths = root.findall('.//{http://www.w3.org/2000/svg}path[@fill="none"]') + assert len(paths) == len(edges) + assert { + tuple( + path.get(f'data-{part}') + for part in ('src-table', 'src-column', 'dst-table', 'dst-column')) + for path in paths + } == {(edge.src_table, edge.fkey, edge.dst_table, + tables[edge.dst_table].definition.pkey) + for edge in edges} + assert len({path.attrib['d'] for path in paths}) == len(edges) + + +@pytest.mark.parametrize('case, message', [ + ('unresolved', 'definition|infer_metadata'), + ('endpoint', 'missing'), + ('no_primary_key', 'primary key'), + ('missing_primary_key', 'missing|primary key'), + ('missing_foreign_key', 'missing|foreign key'), + ('missing_time_column', 'Time column users.gone is missing'), + ('missing_end_time_column', 'End-time column users.gone is missing'), + ('self_relation', '[Ss]elf'), + ('duplicate_columns', 'duplicate column'), + ('xml_control', 'valid XML'), +]) +def test_visualize_rejects_invalid_metadata(case, message): + users, events = _s3_tables() + edge = Edge('events', 'user_id', 'users') + if case == 'unresolved': + users = replace(users, definition=None) + elif case in ('no_primary_key', 'missing_primary_key'): + users = replace( + users, definition=replace( + users.definition, + pkey=None if case == 'no_primary_key' else 'gone')) + elif case == 'endpoint': + edge = Edge('missing', 'user_id', 'users') + elif case == 'missing_foreign_key': + edge = Edge('events', 'gone', 'users') + elif case == 'missing_time_column': + users = replace(users, definition=replace(users.definition, time_col='gone')) + elif case == 'missing_end_time_column': + users = replace(users, definition=replace(users.definition, end_time_col='gone')) + elif case == 'self_relation': + edge = Edge('events', 'user_id', 'events') + elif case == 'duplicate_columns': + users = replace( + users, definition=replace(users.definition, + cols=users.definition.cols * 2)) + elif case == 'xml_control': + users = replace(users, table_name='users\x00') + edge = Edge('events', 'user_id', users.table_name) + with pytest.raises(ValueError, match=message): + Graph([users, events], [edge]).visualize() + + +@pytest.mark.parametrize('layout', ['auto', 'standard', 'compact']) +def test_visualize_empty_graph(layout): + svg = Graph([]).visualize(layout=layout) + root = ElementTree.fromstring(svg) + assert 'No tables' in ''.join(root.itertext()) + assert all(float(value) > 0 for value in root.get('viewBox').split()[2:]) + payload = ElementTree.fromstring(svg._repr_html_()).find( + './script[@type="application/json"]') + assert json.loads(payload.text)['initial'] == ( + 'standard' if layout == 'auto' else layout) + + +def test_widget_shell_composes_non_graph_content_and_escapes_data(): + from kumo.engine._widget import button, render_widget + + label = ' & values' + body = ('
' + + button('chart-toggle', label, text='') + '
' + '' + '') + html, repeated = [ + render_widget(body, data={'labels': [label], 'values': [0.75]}, + attributes={'aria-label': label}) for _ in range(2) + ] + root = ElementTree.fromstring(html) + assert 'nodarkmode' in root.get('class').split() + assert root.get('id') != ElementTree.fromstring(repeated).get('id') + with pytest.raises(ValueError, match='data-.*aria-'): + render_widget(body, attributes={'onclick': 'alert(1)'}) + assert root.get('aria-label') == label + control = root.find('.//button') + assert control.get('aria-label') == label + assert control.text == '' + assert 'disabled' in control.attrib + payload = root.find('.//script[@type="application/json"]') + assert json.loads(payload.text) == {'labels': [label], 'values': [0.75]} + assert root.find('.//{http://www.w3.org/2000/svg}rect').get('width') == '75' + assert '|账户 www.example.com a@b.com' + key = 'user_id|[link](https://example.com)' + events = replace(events, definition=replace(events.definition, cols=[ + events.definition.cols[0], Column(key, Stype.ID, Dtype.int64), + Column('hidden_**value**', Stype.numerical, Dtype.float64)])) + graph = Graph({alias: users, 'events': events}, + [Edge('events', key, alias)]) + destination = tmp_path / f'graph.{suffix}' + destination.write_text('stale contents') + with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, + side_effect=AssertionError('source accessed')): + svg = graph.visualize(destination, show_cols=False) + markdown = destination.read_text(encoding='utf-8') + assert markdown.startswith('# Table relationships\n') + assert markdown.count('```') == 2 + assert 'flowchart LR\n' in markdown + assert 'table_1 -->|"' in markdown and '| table_0' in markdown + assert 'end#34;]
#96;#96;#96;#60;script#62;' in markdown + schema = markdown.split('```')[-1] + assert '' + definition = tables[0].definition + assert definition is not None + tables[0] = replace(tables[0], definition=replace( + definition, cols=[ + *definition.cols, + Column('created_at', Stype.timestamp, Dtype.date)])) + frame = pd.DataFrame({ + 'id': [secret, None, ['alpha', 'beta'], 'later non-null', + 'fourth', 'fifth', 'sixth', 'outside cap'], + 'event_id': ['x' * 1000, None, 'third', 'later non-null', + 'fourth', 'fifth', 'sixth', 'outside cap'], + 'user_id': list(range(1, 9)), + 'created_at': pd.date_range('2026-10-01 09:00:00.123', periods=8, + tz='UTC'), + }) + if outcome == 'nulls': + frame = pd.DataFrame({ + 'id': [None, pd.NA, pd.NaT], + 'event_id': [float('nan')] * 3, + 'user_id': [None, 7, None], + }) + if outcome == 'empty': + frame = frame.iloc[:0] + backend = Mock() + backend.get_sample.return_value.sample_rows = ( + None if outcome == 'unavailable' else frame.to_json(orient='table')) + if outcome == 'error': + backend.get_sample.side_effect = RuntimeError('credential secret') + if outcome == 'collision': + tables = [ + replace( + table, table_name=name, + definition=replace(table.definition, + cols=[Column(key, Stype.ID, + Dtype.int64)], pkey=key)) + for table, name, key in zip(tables, ['a', 'a.b'], ['b.c', 'c']) + ] + backend.get_sample.side_effect = lambda connector, name, *args, **kw: Mock( + sample_rows=pd.DataFrame({ + 'b.c' if name == 'a' else 'c': [name] + }).to_json(orient='table')) + destination = tmp_path / 'graph.svg' + with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, + return_value=backend), patch.object( + Graph, 'snapshot', side_effect=AssertionError): + svg = Graph(tables).visualize(destination, show_samples=True, + show_cols=True) + html = svg._repr_html_() + if outcome == 'values': + repeat = svg._repr_html_() + assert re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', + html) == re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', repeat) + assert set(re.findall(r' id="([^"]+)"', html)).isdisjoint( + re.findall(r' id="([^"]+)"', repeat)) + assert backend.get_sample.call_count == 2 + for call, table in zip(backend.get_sample.call_args_list, + sorted(tables, key=lambda table: table.table_name)): + assert call.args == (table.connector, table.table_name, 30) + assert call.kwargs['sql'] == (sql if table.table_name == 'users' else None) + assert call.kwargs['timeout'] > 0 + assert 'Sample values' not in svg + image = ElementTree.fromstring(html).find( + './div[@class="kumo-graph-static"]/img') + assert b64decode(image.get('src').split(',', 1)[1]).decode() == svg + markdown = svg._markdown() + assert 'private sample' not in markdown and 'credential secret' not in markdown + assert svg._repr_svg_() == destination.read_text(encoding='utf-8') == str(svg) + assert 'credential secret' not in html + assert '