From 69dfabb2d8f7beabb93b2c88327988a31f413cce Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Fri, 9 Oct 2026 12:05:20 +0300 Subject: [PATCH 1/4] Add existing named Excel table read and write connector --- .gitignore | 1 + README.md | 3 + docs/excel-tables.md | 99 +++++++++++++++++++ tests/connectors/test_excel.py | 174 +++++++++++++++++++++++++++++++++ wrangles/connectors/excel.py | 95 ++++++++++++++++++ 5 files changed, 372 insertions(+) create mode 100644 docs/excel-tables.md diff --git a/.gitignore b/.gitignore index e3ad1d91..95d6fd1b 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,7 @@ venv.bak/ docs/* !docs/ai_configuration.md !docs/spreadsheet_recipes.md +!docs/excel-tables.md !docs/ai_answers.md !docs/extract_ai_user_guide.md !docs/github-app-deployment.md diff --git a/README.md b/README.md index 064abd58..eac6739e 100644 --- a/README.md +++ b/README.md @@ -224,3 +224,6 @@ write: - file: name: file.xlsx ``` + +Named workbook table recipes: see [excel.table](docs/excel-tables.md) for the +WranglesXL companion contract, supported actions and integration requirements. diff --git a/docs/excel-tables.md b/docs/excel-tables.md new file mode 100644 index 00000000..aabaa13c --- /dev/null +++ b/docs/excel-tables.md @@ -0,0 +1,99 @@ +# Named Excel tables in recipes + +`excel.table` reads and writes an existing structured table in the calling +WranglesXL workbook. This is spreadsheet write mode `table`, alongside `columns` +and `sheet`. A table name identifies a workbook-wide object, not a worksheet or +selected range; names are matched case-insensitively. This connector does not +open an `.xlsx` file or use the current selection. For files, use `file`. + +```yaml +read: + - excel.table: + name: Products +wrangles: + - math: + input: Price * 2 + output: Price +write: + - excel.table: + name: Results + action: replace +``` + +Both tables must already exist. Creating missing tables is outside the first +version of [#1220](https://github.com/wrangleworks/WranglesPY/issues/1220). +Missing names and unknown tables fail explicitly; there is no fallback to the +active sheet or current selection. Selection input retains its separate contract +(`grid.selected_data` / `excel.selected_data` when those companion connectors are +available). This feature does not expand #943 or implement `excel.sql`. + +## Reads and writes + +- Read headers and every body row, including filtered/hidden rows, excluding + the totals row. A table with no body rows yields an empty dataframe with its + headers. Ordinary recipe read options (`columns`, `where`, etc.) still apply. +- `replace` (default) clears/replaces the body, including empty results, and + adjusts the table's row count. Existing table name, headers and table identity + remain. It does not replace or create a worksheet. +- `append` adds rows; empty results add nothing. Existing and output column + names must match exactly, but their order may differ: XL aligns values by name. + Header names must be non-empty strings and unique ignoring case. New/missing + columns fail; no columns are silently dropped or invented. +- Each write is an ordered operation. Repeated writes in a single request stay + separate. For external selection batches, a `replace` becomes `append` after + the first batch to retain every batch's rows. +- In this initial implementation, write targets with active filters or body + formulas are rejected before mutation. Clear target filters first. Calculated + columns are not replaced or silently converted to values. Incoming strings + beginning with `=` are also rejected instead of being interpreted as formulas. + +Read snapshots are captured before execution and stay unchanged for that run, +including when the same table is also a destination. A recipe may compose table +reads through `union`, `join` or `concatenate`. A table-only recipe runs once, +independently of the user's unrelated selection. If combined with selected-data +reads, each selected-data batch uses the same full table snapshot. + +## Workbook transport + +WranglesXL injects the reserved recipe variable `__excel_tables`: + +```json +{"Products": {"columns": ["ID", "Price"], "data": [[1, 10], [2, 20]]}} +``` + +Python reads this mapping through its existing recipe-variable injection. Input +snapshots are never added to `memory.dataframes`, since the Lambda returns that +collection as write results. Python writes a split dataframe payload with +`connector: excel.table.write`, `name`, and `action`. The existing Lambda variable +and output transport can carry this contract without a new top-level API field. +Do not set `__excel_tables` as a user runtime variable in XL; XL owns it. + +XL routes table payloads separately from sheet/columns output and stages them +until recipe execution completes. It validates all staged table destinations +before starting table mutations. Excel writes are not a transaction: a later +Office API failure may still leave earlier changes applied. This staging does +not make unrelated sheet writes atomic or provide a rollback mechanism. + +## Integration and verification + +Python-only installation cannot read an open workbook. Deploy the matching +WranglesXL companion and publish the Python version and generated recipe schema +before enabling this feature. Saved child recipes whose reads are not visible to +XL planning cannot request additional workbook snapshots dynamically; keep table +reads in the effective top-level recipe (including supported compositions). +Workbook snapshots must fit the existing request-size limits; oversized requests +fail rather than silently truncating table data. This first version does not +introduce streaming table input. + +Offline validation: + +```sh +python -m pytest -c pytest-local.ini tests/connectors/test_excel.py -q +``` + +Before closing #1220, verify in live Excel with synthetic data: all-row reading +under filters, totals exclusion, empty tables, replacement growing/shrinking and +clearing rows, append/reordered columns, missing names, formula/filter rejection, +multiple writes, selection batches, cancellation/failure before table writes, +and preservation of table identity and surrounding cells. Record Python, XL, +JS and Lambda versions. Offline mocks do not establish deployed Office behavior. diff --git a/tests/connectors/test_excel.py b/tests/connectors/test_excel.py index 2a8c599d..e677980a 100644 --- a/tests/connectors/test_excel.py +++ b/tests/connectors/test_excel.py @@ -438,3 +438,177 @@ def test_excel_sheet_overwrite_uses_append_after_first_external_batch(): assert first_batch["action"] == "overwrite" assert second_batch["action"] == "append" + + +@pytest.fixture +def table_memory(): + memory.clear() + yield + memory.clear() + + +def test_table_recipe_reads_named_snapshot_and_emits_only_output(table_memory): + snapshots = {"Products": {"columns": ["ID", "Price"], "data": [[1, 10], [2, 20]]}} + result = wrangles.recipe.run( + {"read": [{"excel.table": {"name": "products"}}], + "wrangles": [{"math": {"output": "Price", "input": "Price * 2"}}], + "write": [{"excel.table": {"name": "Results"}}]}, + variables={"__excel_tables": snapshots}, + dataframe=pd.DataFrame({"UnrelatedSelection": [999]}), + ) + assert result.to_dict("list") == {"ID": [1, 2], "Price": [20, 40]} + assert list(memory.dataframes.values()) == [{ + "index": [0, 1], "columns": ["ID", "Price"], "data": [[1, 20], [2, 40]], + "connector": "excel.table.write", "name": "Results", "action": "replace", + }] + assert snapshots == {"Products": {"columns": ["ID", "Price"], "data": [[1, 10], [2, 20]]}} + + +def test_table_composed_read_uses_multiple_named_tables(table_memory): + result = wrangles.recipe.run( + {"read": [{"union": {"sources": [ + {"excel.table": {"name": "A"}}, {"excel.table": {"name": "B"}}, + ]}}]}, + variables={"__excel_tables": { + "A": {"columns": ["ID"], "data": [[1]]}, + "B": {"columns": ["ID"], "data": [[2], [3]]}, + }}, + ) + assert result.to_dict("list") == {"ID": [1, 2, 3]} + assert memory.dataframes == {} + + +def test_table_empty_read_and_replace_keep_headers(table_memory): + df = wrangles.connectors.excel.table.read("Empty", { + "__excel_tables": {"Empty": {"columns": ["ID"], "data": []}}, + }) + assert df.shape == (0, 1) + assert df.columns.tolist() == ["ID"] + wrangles.connectors.excel.table.write(df, "Results") + assert list(memory.dataframes.values())[0] == { + "columns": ["ID"], "index": [], "data": [], + "connector": "excel.table.write", "name": "Results", "action": "replace", + } + + +@pytest.mark.parametrize("variables, message", [ + ({}, "supplied by WranglesXL"), + ({"__excel_tables": {}}, "not found"), + ({"__excel_tables": {"T": {}, "t": {}}}, "ambiguous"), + ({"__excel_tables": {"T": []}}, "invalid data payload"), + ({"__excel_tables": {"T": {"columns": ["ID"], "data": [[1, 2]]}}}, "match its headers"), + ({"__excel_tables": {"T": {"columns": ["ID"], "data": None}}}, "match its headers"), +]) +def test_table_read_rejects_missing_or_malformed_workbook_data(variables, message, table_memory): + with pytest.raises((ValueError, RuntimeError), match=message): + wrangles.connectors.excel.table.read("T", variables) + assert memory.dataframes == {} + + +@pytest.mark.parametrize("name", [None, "", " ", " T", 5]) +def test_table_rejects_invalid_names(name, table_memory): + with pytest.raises(ValueError, match="name must"): + wrangles.connectors.excel.table.read(name) + with pytest.raises(ValueError, match="name must"): + wrangles.connectors.excel.table.write(pd.DataFrame({"ID": [1]}), name) + assert memory.dataframes == {} + + +@pytest.mark.parametrize("columns", [[], [""], [1], ["ID", "id"]]) +def test_table_rejects_invalid_headers(columns, table_memory): + with pytest.raises(ValueError, match="headers must"): + wrangles.connectors.excel.table.read("T", {"__excel_tables": { + "T": {"columns": columns, "data": []}, + }}) + with pytest.raises(ValueError, match="headers must"): + wrangles.connectors.excel.table.write(pd.DataFrame(columns=columns), "T") + assert memory.dataframes == {} + + +@pytest.mark.parametrize("action, number, total, expected", [ + ("replace", 1, 1, "replace"), ("append", 1, 1, "append"), + ("replace", 1, 3, "replace"), ("replace", 2, 3, "append"), + ("replace", 3, 3, "append"), ("append", 2, 3, "append"), +]) +def test_table_write_normalizes_external_batch_action(action, number, total, expected, table_memory): + wrangles.connectors.excel.table.write( + pd.DataFrame({"ID": [number]}), "T", action, + {"batch_number": number, "batch_total": total}, + ) + assert list(memory.dataframes.values()) == [{ + "columns": ["ID"], "index": [0], "data": [[number]], + "name": "T", "action": expected, "connector": "excel.table.write", + }] + + +@pytest.mark.parametrize("action, variables, message", [ + ("create", {}, "action must"), ("overwrite", {}, "action must"), + ("replace", {"batch_number": 0}, "positive integers"), + ("replace", {"batch_total": "2"}, "positive integers"), + ("replace", {"batch_number": True}, "positive integers"), + ("replace", {"batch_number": 2, "batch_total": 1}, "exceeds"), +]) +def test_table_write_rejects_invalid_action_or_batch_metadata(action, variables, message, table_memory): + with pytest.raises(ValueError, match=message): + wrangles.connectors.excel.table.write(pd.DataFrame({"ID": [1]}), "T", action, variables) + assert memory.dataframes == {} + + +def test_table_multiple_writes_preserve_order_and_target_metadata(table_memory): + wrangles.recipe.run( + {"read": [{"excel.table": {"name": "${source}"}}], + "write": [{"excel.table": {"name": "T", "action": "replace"}}, + {"excel.table": {"name": "T", "action": "append"}}, + {"excel.sheet": {"name": "Sheet"}}]}, + variables={"source": "A", "__excel_tables": {"A": {"columns": ["ID"], "data": [[1]]}}}, + ) + payloads = list(memory.dataframes.values()) + assert [(p["connector"], p["name"], p.get("action")) for p in payloads] == [ + ("excel.table.write", "T", "replace"), ("excel.table.write", "T", "append"), + ("excel.sheet.write", "Sheet", None), + ] + assert [p["data"] for p in payloads] == [[[1]], [[1]], [[1]]] + + +def test_table_schema_requires_name_and_only_supported_actions(): + import yaml + import jsonschema + read = yaml.safe_load(wrangles.connectors.excel.table._schema["read"]) + write = yaml.safe_load(wrangles.connectors.excel.table._schema["write"]) + jsonschema.validate({"name": "Products"}, read) + jsonschema.validate({"name": "Results", "action": "append"}, write) + for value, schema in [({}, read), ({"name": ""}, read), + ({"name": "T", "action": "create"}, write), + ({"name": "T", "sheet": "Sheet1"}, write)]: + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(value, schema) + + + +def test_table_is_discovered_in_generated_recipe_schema(monkeypatch, tmp_path): + import json + import runpy + import shutil + import requests + import jsonschema + from pathlib import Path + from types import SimpleNamespace + root = Path(__file__).resolve().parents[2] + shutil.copy(root / "schema/recipe_base_schema.json", tmp_path / "recipe_base_schema.json") + monkeypatch.chdir(tmp_path) + def get_meta_schema(url): + assert url == "http://json-schema.org/draft-07/schema#" + return SimpleNamespace(json=lambda: jsonschema.Draft7Validator.META_SCHEMA) + monkeypatch.setattr(requests, "get", get_meta_schema) + runpy.run_path(str(root / "schema/generate_recipe_schema.py")) + schema = json.loads((tmp_path / "schema.json").read_text()) + jsonschema.validate({ + "read": [{"excel.table": {"name": "Products", "columns": ["ID"]}}], + "write": [{"excel.table": {"name": "Results", "action": "append", "columns": ["ID"]}}], + }, schema) + for recipe in [ + {"read": [{"excel.table": {}}]}, + {"write": [{"excel.table": {"name": "T", "action": "create"}}]}, + ]: + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(recipe, schema) diff --git a/wrangles/connectors/excel.py b/wrangles/connectors/excel.py index b79d1a63..e650d83b 100644 --- a/wrangles/connectors/excel.py +++ b/wrangles/connectors/excel.py @@ -287,3 +287,98 @@ def write( type: boolean description: Whether the column values should wrap text within the cell. """ + + +class table(): + """Existing workbook tables, transported by WranglesXL through recipe variables.""" + _schema = {} + + def read(name: str, variables: dict = None): + _validate_table_name(name) + snapshots = (variables or {}).get("__excel_tables") + if not isinstance(snapshots, dict): + raise RuntimeError("excel.table requires workbook table data supplied by WranglesXL") + matches = [value for key, value in snapshots.items() + if isinstance(key, str) and key.casefold() == name.casefold()] + if len(matches) != 1: + raise ValueError(f"Excel table '{name}' was not found or is ambiguous") + payload = matches[0] + if not isinstance(payload, dict): + raise ValueError(f"Excel table '{name}' has an invalid data payload") + columns, rows = payload.get("columns"), payload.get("data") + _validate_table_columns(columns) + if not isinstance(rows, list) or any( + not isinstance(row, list) or len(row) != len(columns) for row in rows + ): + raise ValueError(f"Excel table '{name}' rows must match its headers") + # The workbook snapshot is input only: never place it in memory outputs. + return _pd.DataFrame(rows, columns=columns) + + def write(df: _pd.DataFrame, name: str, action: str = "replace", variables: dict = None): + _validate_table_name(name) + _validate_table_columns(df.columns.tolist()) + if action not in ("replace", "append"): + raise ValueError("excel.table action must be replace or append") + variables = variables or {} + batch_number = variables.get("batch_number", 1) + batch_total = variables.get("batch_total", 1) + for value in (batch_number, batch_total): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError("excel.table batch metadata must be positive integers") + if batch_number > batch_total: + raise ValueError("excel.table batch_number exceeds batch_total") + if action == "replace" and batch_number > 1: + action = "append" + _memory.write(df, connector="excel.table.write", orient="split", name=name, action=action) + + _schema["read"] = """ + type: object + description: >- + Read all body rows and headers from an existing named table in the + calling Excel workbook. Requires WranglesXL; independent of selection. + Includes filtered rows and excludes the totals row. + additionalProperties: false + required: [name] + properties: + name: + type: string + minLength: 1 + description: Workbook-wide table name, matched case-insensitively. + """ + _schema["write"] = """ + type: object + description: >- + Write to an existing named table in the calling Excel workbook. + Spreadsheet write mode table. Does not create tables. Headers must + match the existing column names; column order is aligned by name. + Calculated/formula columns are not supported in this initial version. + additionalProperties: false + required: [name] + properties: + name: + type: string + minLength: 1 + description: Workbook-wide table name, matched case-insensitively. + action: + type: string + enum: [replace, append] + default: replace + description: >- + replace clears and replaces body rows, including empty results; + append adds rows. The table resizes to fit the result. + Later external batches append after the first replacement. + """ + + +def _validate_table_name(name): + if not isinstance(name, str) or not name.strip() or name != name.strip(): + raise ValueError("excel.table name must be a non-empty string without surrounding whitespace") + + +def _validate_table_columns(columns): + if not isinstance(columns, list) or not columns or any( + not isinstance(column, str) or not column.strip() for column in columns + ): + raise ValueError("excel.table headers must be non-empty strings") + if len({column.casefold() for column in columns}) != len(columns): + raise ValueError("excel.table headers must be unique (case-insensitive)") From fab48049ee642a26348dffef6c15452712b6adb7 Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Fri, 9 Oct 2026 12:08:20 +0300 Subject: [PATCH 2/4] Verify empty table replacement through recipe execution --- tests/connectors/test_excel.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/connectors/test_excel.py b/tests/connectors/test_excel.py index e677980a..d56f839b 100644 --- a/tests/connectors/test_excel.py +++ b/tests/connectors/test_excel.py @@ -479,12 +479,13 @@ def test_table_composed_read_uses_multiple_named_tables(table_memory): def test_table_empty_read_and_replace_keep_headers(table_memory): - df = wrangles.connectors.excel.table.read("Empty", { - "__excel_tables": {"Empty": {"columns": ["ID"], "data": []}}, - }) + df = wrangles.recipe.run( + {"read": [{"excel.table": {"name": "Empty"}}], + "write": [{"excel.table": {"name": "Results"}}]}, + variables={"__excel_tables": {"Empty": {"columns": ["ID"], "data": []}}}, + ) assert df.shape == (0, 1) assert df.columns.tolist() == ["ID"] - wrangles.connectors.excel.table.write(df, "Results") assert list(memory.dataframes.values())[0] == { "columns": ["ID"], "index": [], "data": [], "connector": "excel.table.write", "name": "Results", "action": "replace", From 3563298e75c3740565edc9c045e98cd93f4057b5 Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Fri, 9 Oct 2026 17:41:53 +0300 Subject: [PATCH 3/4] Add sheet and cell defaults for missing Excel table outputs --- docs/excel-tables.md | 40 +++++++++++-- tests/connectors/test_excel.py | 105 ++++++++++++++++++++++++++++++++- wrangles/connectors/excel.py | 60 +++++++++++++++++-- 3 files changed, 193 insertions(+), 12 deletions(-) diff --git a/docs/excel-tables.md b/docs/excel-tables.md index aabaa13c..6686d36d 100644 --- a/docs/excel-tables.md +++ b/docs/excel-tables.md @@ -20,21 +20,46 @@ write: action: replace ``` -Both tables must already exist. Creating missing tables is outside the first -version of [#1220](https://github.com/wrangleworks/WranglesPY/issues/1220). -Missing names and unknown tables fail explicitly; there is no fallback to the -active sheet or current selection. Selection input retains its separate contract +Read tables must already exist. A missing write table is created by WranglesXL +with the output headers on its destination worksheet. Missing read tables and +missing names fail explicitly; there is no fallback to the active selection. +Selection input retains its separate contract (`grid.selected_data` / `excel.selected_data` when those companion connectors are available). This feature does not expand #943 or implement `excel.sql`. ## Reads and writes +For a missing output table, specify its worksheet and top-left cell: + +```yaml +write: + - excel.table: + name: Results + sheet: Output + cell: C3 + action: replace +``` + +`cell` defaults to `A1`. Omitted `sheet` uses the first 10 characters of +`-`, using the recipe variable `recipe_name` supplied by +XL (`Recipe` when unavailable). For example, `Clean` + `Results` gives +`Clean-Resu`. Invalid generated worksheet characters are replaced by `_`. +Explicit worksheet names are not truncated. Absolute cell references such as +`$C$3` are accepted and normalized to `C3`. + +Existing workbook-wide tables retain their worksheet and location; `sheet` and +`cell` only select the location for a missing table. XL creates a missing worksheet, +but refuses occupied cells, table overlaps, invalid table names and out-of-bounds +results before any table mutation. Empty output creates a header-only table. +The 10-character prefix can collide for multiple output tables: provide distinct +`sheet`/`cell` destinations when needed. Batched outputs reuse the new table. + - Read headers and every body row, including filtered/hidden rows, excluding the totals row. A table with no body rows yields an empty dataframe with its headers. Ordinary recipe read options (`columns`, `where`, etc.) still apply. - `replace` (default) clears/replaces the body, including empty results, and adjusts the table's row count. Existing table name, headers and table identity - remain. It does not replace or create a worksheet. + remain. It does not replace a worksheet. - `append` adds rows; empty results add nothing. Existing and output column names must match exactly, but their order may differ: XL aligns values by name. Header names must be non-empty strings and unique ignoring case. New/missing @@ -64,7 +89,10 @@ WranglesXL injects the reserved recipe variable `__excel_tables`: Python reads this mapping through its existing recipe-variable injection. Input snapshots are never added to `memory.dataframes`, since the Lambda returns that collection as write results. Python writes a split dataframe payload with -`connector: excel.table.write`, `name`, and `action`. The existing Lambda variable +`connector: excel.table.write`, `name`, `action`, `sheet`, and `cell`. Missing +dataframe values introduced by `union`, join, or other reads are emitted as JSON +`null` in table output without changing the logical dataframe or input snapshots. +The existing Lambda variable and output transport can carry this contract without a new top-level API field. Do not set `__excel_tables` as a user runtime variable in XL; XL owns it. diff --git a/tests/connectors/test_excel.py b/tests/connectors/test_excel.py index d56f839b..9d9235e0 100644 --- a/tests/connectors/test_excel.py +++ b/tests/connectors/test_excel.py @@ -460,6 +460,7 @@ def test_table_recipe_reads_named_snapshot_and_emits_only_output(table_memory): assert list(memory.dataframes.values()) == [{ "index": [0, 1], "columns": ["ID", "Price"], "data": [[1, 20], [2, 40]], "connector": "excel.table.write", "name": "Results", "action": "replace", + "sheet": "Recipe-Res", "cell": "A1", }] assert snapshots == {"Products": {"columns": ["ID", "Price"], "data": [[1, 10], [2, 20]]}} @@ -489,6 +490,7 @@ def test_table_empty_read_and_replace_keep_headers(table_memory): assert list(memory.dataframes.values())[0] == { "columns": ["ID"], "index": [], "data": [], "connector": "excel.table.write", "name": "Results", "action": "replace", + "sheet": "Recipe-Res", "cell": "A1", } @@ -539,6 +541,7 @@ def test_table_write_normalizes_external_batch_action(action, number, total, exp assert list(memory.dataframes.values()) == [{ "columns": ["ID"], "index": [0], "data": [[number]], "name": "T", "action": expected, "connector": "excel.table.write", + "sheet": "Recipe-T", "cell": "A1", }] @@ -578,9 +581,10 @@ def test_table_schema_requires_name_and_only_supported_actions(): write = yaml.safe_load(wrangles.connectors.excel.table._schema["write"]) jsonschema.validate({"name": "Products"}, read) jsonschema.validate({"name": "Results", "action": "append"}, write) + jsonschema.validate({"name": "Results", "sheet": "Sheet1", "cell": "C3"}, write) for value, schema in [({}, read), ({"name": ""}, read), ({"name": "T", "action": "create"}, write), - ({"name": "T", "sheet": "Sheet1"}, write)]: + ({"name": "T", "sheet": ""}, write)]: with pytest.raises(jsonschema.ValidationError): jsonschema.validate(value, schema) @@ -613,3 +617,102 @@ def get_meta_schema(url): ]: with pytest.raises(jsonschema.ValidationError): jsonschema.validate(recipe, schema) + + +@pytest.mark.parametrize("recipe_name, expected", [ + ("Clean", "Clean-Resu"), (None, "Recipe-Res"), + ("A", "A-Results"), ("1234567890X", "1234567890"), + ("A/B:C", "A_B_C-Resu"), +]) +def test_table_write_default_sheet_and_cell(recipe_name, expected, table_memory): + variables = {"recipe_name": recipe_name} + wrangles.recipe.run( + {"read": [{"test": {"rows": 1, "values": {"ID": 7}}}], + "write": [{"excel.table": {"name": "Results"}}]}, + variables=variables, + ) + payload, = memory.dataframes.values() + assert payload == {"columns": ["ID"], "index": [0], "data": [[7]], + "connector": "excel.table.write", "name": "Results", + "action": "replace", "sheet": expected, "cell": "A1"} + + +def test_table_write_preserves_explicit_sheet_and_normalizes_cell(table_memory): + wrangles.recipe.run( + {"read": [{"test": {"rows": 1, "values": {"ID": 7}}}], + "write": [{"excel.table": {"name": "Results", "sheet": "Manual output", "cell": "$c$3"}}]}, + variables={"recipe_name": "Other"}, + ) + payload, = memory.dataframes.values() + assert (payload["sheet"], payload["cell"], payload["data"]) == ("Manual output", "C3", [[7]]) + + +@pytest.mark.parametrize("options", [ + {"sheet": ""}, {"sheet": "a" * 32}, {"sheet": "bad/name"}, + {"sheet": "'quoted"}, {"sheet": 7}, {"cell": "A0"}, + {"cell": "A1:B2"}, {"cell": "Sheet1!A1"}, {"cell": "XFE1"}, + {"cell": "A1048577"}, {"cell": ""}, {"cell": 7}, +]) +def test_table_write_rejects_invalid_locations_before_output(options, table_memory): + with pytest.raises(ValueError, match="excel.table (sheet|cell)"): + wrangles.connectors.excel.table.write(pd.DataFrame({"ID": [1]}), "Results", **options) + assert memory.dataframes == {} + + +def test_table_write_accepts_last_excel_cell(table_memory): + wrangles.connectors.excel.table.write(pd.DataFrame(columns=["ID"]), "Results", cell="XFD1048576") + payload, = memory.dataframes.values() + assert payload["cell"] == "XFD1048576" and payload["data"] == [] + + +@pytest.mark.parametrize("composition, columns, rows", [ + ({"union": {"sources": [{"excel.table": {"name": "Products"}}, {"excel.table": {"name": "Extra"}}]}}, + ["ID", "Value", "Category"], [[1, "alpha", None], [2, "bravo", None], [3, None, "external"]]), + ({"join": {"how": "left", "left_on": "ID", "right_on": "ID", "sources": [{"excel.table": {"name": "Products"}}, {"excel.table": {"name": "Lookup"}}]}}, + ["ID", "Value", "Category"], [[1, "alpha", None], [2, "bravo", "tools"]]), + ({"concatenate": {"sources": [{"excel.table": {"name": "Products"}}, {"excel.table": {"name": "Marker"}}]}}, + ["ID", "Value", "Marker"], [[1, "alpha", "a"], [2, "bravo", "b"]]), + ([{"excel.table": {"name": "Products"}}, {"excel.table": {"name": "Extra"}}], + ["ID", "Value", "Category"], [[1, "alpha", None], [2, "bravo", None], [3, None, "external"]]), +]) +def test_table_composed_reads_emit_json_safe_output_and_location(composition, columns, rows, table_memory): + import copy + import json + snapshots = { + "Products": {"columns": ["ID", "Value"], "data": [[1, "alpha"], [2, "bravo"]]}, + "Extra": {"columns": ["ID", "Category"], "data": [[3, "external"]]}, + "Lookup": {"columns": ["ID", "Category"], "data": [[2, "tools"]]}, + "Marker": {"columns": ["Marker"], "data": [["a"], ["b"]]}, + } + original = copy.deepcopy(snapshots) + result = wrangles.recipe.run( + {"read": composition, "write": [{"excel.table": {"name": "Results", "cell": "D4"}}]}, + variables={"__excel_tables": snapshots, "recipe_name": "Clean"}, + ) + payload, = memory.dataframes.values() + assert json.loads(json.dumps(payload, allow_nan=False))["data"] == rows + assert payload["columns"] == columns + assert (payload["sheet"], payload["cell"]) == ("Clean-Resu", "D4") + assert result.columns.tolist() == columns and len(result) == len(rows) + assert snapshots == original + + +def test_table_write_normalizes_nullable_values_without_mutating_input(table_memory): + import json + frame = pd.DataFrame({"ID": [1, 2, 3], "Value": [float("nan"), pd.NA, pd.NaT]}) + original = frame.copy(deep=True) + wrangles.connectors.excel.table.write(frame, "Results") + payload, = memory.dataframes.values() + assert json.loads(json.dumps(payload, allow_nan=False))["data"] == [[1, None], [2, None], [3, None]] + pd.testing.assert_frame_equal(frame, original) + + +def test_table_write_keeps_default_location_across_external_batches(table_memory): + for number in (1, 2): + wrangles.connectors.excel.table.write( + pd.DataFrame({"ID": [number]}), "Results", + variables={"recipe_name": "Clean", "batch_number": number, "batch_total": 2}, + ) + assert [(p["sheet"], p["cell"], p["action"], p["data"]) for p in memory.dataframes.values()] == [ + ("Clean-Resu", "A1", "replace", [[1]]), ("Clean-Resu", "A1", "append", [[2]]), + ] diff --git a/wrangles/connectors/excel.py b/wrangles/connectors/excel.py index e650d83b..b5737b01 100644 --- a/wrangles/connectors/excel.py +++ b/wrangles/connectors/excel.py @@ -5,6 +5,7 @@ from . import memory as _memory from . import grid as _grid import logging as _logging +import re as _re # Names/values follow the syntax already used by the Polars/XlsxWriter @@ -290,7 +291,7 @@ def write( class table(): - """Existing workbook tables, transported by WranglesXL through recipe variables.""" + """Workbook tables, transported by WranglesXL through recipe variables.""" _schema = {} def read(name: str, variables: dict = None): @@ -314,12 +315,40 @@ def read(name: str, variables: dict = None): # The workbook snapshot is input only: never place it in memory outputs. return _pd.DataFrame(rows, columns=columns) - def write(df: _pd.DataFrame, name: str, action: str = "replace", variables: dict = None): + def write( + df: _pd.DataFrame, + name: str, + action: str = "replace", + variables: dict = None, + sheet: str = None, + cell: str = None, + ): _validate_table_name(name) _validate_table_columns(df.columns.tolist()) if action not in ("replace", "append"): raise ValueError("excel.table action must be replace or append") variables = variables or {} + if sheet is None: + recipe_name = variables.get("recipe_name") or "Recipe" + sheet = _re.sub(r"[\\/*?:\[\]]", "_", f"{recipe_name}-{name}")[:10] + sheet = sheet.strip("'") or "Recipe" + if not isinstance(sheet, str) or not sheet or len(sheet) > 31 or ( + _re.search(r"[\\/*?:\[\]]", sheet) + or sheet.startswith("'") or sheet.endswith("'") + ): + raise ValueError("excel.table sheet must be a valid worksheet name (1-31 characters)") + cell = "A1" if cell is None else cell + match = _re.fullmatch(r"\$?([A-Za-z]{1,3})\$?([1-9][0-9]*)", cell) if isinstance(cell, str) else None + if match is None: + raise ValueError("excel.table cell must be a single A1-style cell address") + column, row = match.groups() + column = column.upper() + column_number = 0 + for letter in column: + column_number = column_number * 26 + ord(letter) - ord("A") + 1 + if column_number > 16384 or int(row) > 1048576: + raise ValueError("excel.table cell exceeds Excel worksheet bounds") + cell = f"{column}{row}" batch_number = variables.get("batch_number", 1) batch_total = variables.get("batch_total", 1) for value in (batch_number, batch_total): @@ -329,7 +358,13 @@ def write(df: _pd.DataFrame, name: str, action: str = "replace", variables: dict raise ValueError("excel.table batch_number exceeds batch_total") if action == "replace" and batch_number > 1: action = "append" - _memory.write(df, connector="excel.table.write", orient="split", name=name, action=action) + # A composed read may introduce NaN/pd.NA/NaT. Emit valid JSON cells + # without changing the dataframe returned to the recipe caller. + output = df.astype(object).where(_pd.notna(df), None) + _memory.write( + output, connector="excel.table.write", orient="split", + name=name, action=action, sheet=sheet, cell=cell, + ) _schema["read"] = """ type: object @@ -348,8 +383,8 @@ def write(df: _pd.DataFrame, name: str, action: str = "replace", variables: dict _schema["write"] = """ type: object description: >- - Write to an existing named table in the calling Excel workbook. - Spreadsheet write mode table. Does not create tables. Headers must + Write to a named table in the calling Excel workbook, creating it + on the requested sheet and cell if missing. Headers must match the existing column names; column order is aligned by name. Calculated/formula columns are not supported in this initial version. additionalProperties: false @@ -359,6 +394,21 @@ def write(df: _pd.DataFrame, name: str, action: str = "replace", variables: dict type: string minLength: 1 description: Workbook-wide table name, matched case-insensitively. + sheet: + type: string + minLength: 1 + maxLength: 31 + description: >- + Worksheet for a new table. Defaults to the first 10 characters + of recipe_name-table_name (Recipe if recipe_name is unavailable). + Invalid generated worksheet characters are replaced with underscores. + Existing tables keep their current worksheet and location. + cell: + type: string + default: A1 + description: >- + Top-left cell for a new table. Defaults to A1. + Existing tables keep their current location. action: type: string enum: [replace, append] From f3bed7b1f4d22a132de5da071fbc3b6a16af785e Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Tue, 20 Oct 2026 15:26:46 +0300 Subject: [PATCH 4/4] Require explicit existing-table writes and support numbered table names --- docs/excel-tables.md | 32 ++++++++++++++++++++++++++-- tests/connectors/test_excel.py | 38 ++++++++++++++++++++++++++++++---- wrangles/connectors/excel.py | 28 +++++++++++++++++-------- 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/docs/excel-tables.md b/docs/excel-tables.md index 6686d36d..f67c428a 100644 --- a/docs/excel-tables.md +++ b/docs/excel-tables.md @@ -57,10 +57,12 @@ The 10-character prefix can collide for multiple output tables: provide distinct - Read headers and every body row, including filtered/hidden rows, excluding the totals row. A table with no body rows yields an empty dataframe with its headers. Ordinary recipe read options (`columns`, `where`, etc.) still apply. -- `replace` (default) clears/replaces the body, including empty results, and +- `error` (default) creates a missing table but rejects an existing workbook-wide + name before any workbook mutation. This prevents accidental updates. +- Explicit `replace` clears/replaces the body, including empty results, and adjusts the table's row count. Existing table name, headers and table identity remain. It does not replace a worksheet. -- `append` adds rows; empty results add nothing. Existing and output column +- Explicit `append` adds rows; empty results add nothing. Existing and output column names must match exactly, but their order may differ: XL aligns values by name. Header names must be non-empty strings and unique ignoring case. New/missing columns fail; no columns are silently dropped or invented. @@ -78,6 +80,32 @@ reads through `union`, `join` or `concatenate`. A table-only recipe runs once, independently of the user's unrelated selection. If combined with selected-data reads, each selected-data batch uses the same full table snapshot. +## Numbered output tables + +```yaml +write: + - excel.table: + name: my_table_* + sheet: FreshOutput + cell: A1 +``` + +A single trailing `*` requests a new workbook-wide name. XL chooses one greater +than the highest numeric suffix, ignoring case: with `my_table_2` and +`MY_TABLE_9`, it creates `my_table_10`. With no matches it starts at 1; gaps +are not reused. Literal output names in the same run are also reserved. All +batches of this wildcard output use the same resolved name; another run +allocates the next suffix. Read names must be literal existing table names. + +Wildcard writes require `action: error` (the default); combining a wildcard +with `append` or `replace` is rejected. Numbering allocates only the table name, +not a free worksheet or cell. The usual sheet/cell defaults and occupied-cell +checks apply. Choose a free destination on later runs, otherwise the write +fails rather than overwriting the previous table. Malformed patterns, invalid +Excel names, excessive numeric suffixes and overlapping destinations fail +before mutation. Number allocation is not a lock against simultaneous writers; +Office errors are reported if another writer claims the name first. + ## Workbook transport WranglesXL injects the reserved recipe variable `__excel_tables`: diff --git a/tests/connectors/test_excel.py b/tests/connectors/test_excel.py index 9d9235e0..886a54fb 100644 --- a/tests/connectors/test_excel.py +++ b/tests/connectors/test_excel.py @@ -4,6 +4,34 @@ from wrangles.connectors import memory +@pytest.mark.parametrize("number", [1, 2, 3]) +def test_table_default_error_is_preserved_for_all_batches(number, table_memory): + wrangles.connectors.excel.table.write( + pd.DataFrame({"ID": [number]}), "Results", + variables={"batch_number": number, "batch_total": 3}, + ) + frame = list(memory.dataframes.values())[0] + assert frame["action"] == "error" + assert frame["name"] == "Results" + assert frame["data"] == [[number]] + + +def test_table_wildcard_is_transported_for_workbook_allocation(table_memory): + wrangles.connectors.excel.table.write(pd.DataFrame({"ID": [1]}), "my_table_*", sheet="Fresh") + frame = list(memory.dataframes.values())[0] + assert frame["name"] == "my_table_*" + assert frame["action"] == "error" + assert frame["sheet"] == "Fresh" + assert frame["data"] == [[1]] + + +@pytest.mark.parametrize("name,action", [("*", "error"), ("my*table", "error"), ("my_table_**", "error"), ("my_table_*", "append"), ("my_table_*", "replace")]) +def test_table_invalid_wildcard_fails_before_output(name, action, table_memory): + with pytest.raises(ValueError, match="wildcard"): + wrangles.connectors.excel.table.write(pd.DataFrame({"ID": [1]}), name, action=action) + assert memory.dataframes == {} + + def test_default_write(): """ Test memory connector @@ -459,7 +487,7 @@ def test_table_recipe_reads_named_snapshot_and_emits_only_output(table_memory): assert result.to_dict("list") == {"ID": [1, 2], "Price": [20, 40]} assert list(memory.dataframes.values()) == [{ "index": [0, 1], "columns": ["ID", "Price"], "data": [[1, 20], [2, 40]], - "connector": "excel.table.write", "name": "Results", "action": "replace", + "connector": "excel.table.write", "name": "Results", "action": "error", "sheet": "Recipe-Res", "cell": "A1", }] assert snapshots == {"Products": {"columns": ["ID", "Price"], "data": [[1, 10], [2, 20]]}} @@ -482,7 +510,7 @@ def test_table_composed_read_uses_multiple_named_tables(table_memory): def test_table_empty_read_and_replace_keep_headers(table_memory): df = wrangles.recipe.run( {"read": [{"excel.table": {"name": "Empty"}}], - "write": [{"excel.table": {"name": "Results"}}]}, + "write": [{"excel.table": {"name": "Results", "action": "replace"}}]}, variables={"__excel_tables": {"Empty": {"columns": ["ID"], "data": []}}}, ) assert df.shape == (0, 1) @@ -581,6 +609,8 @@ def test_table_schema_requires_name_and_only_supported_actions(): write = yaml.safe_load(wrangles.connectors.excel.table._schema["write"]) jsonschema.validate({"name": "Products"}, read) jsonschema.validate({"name": "Results", "action": "append"}, write) + jsonschema.validate({"name": "my_table_*", "action": "error"}, write) + assert write["properties"]["action"]["default"] == "error" jsonschema.validate({"name": "Results", "sheet": "Sheet1", "cell": "C3"}, write) for value, schema in [({}, read), ({"name": ""}, read), ({"name": "T", "action": "create"}, write), @@ -634,7 +664,7 @@ def test_table_write_default_sheet_and_cell(recipe_name, expected, table_memory) payload, = memory.dataframes.values() assert payload == {"columns": ["ID"], "index": [0], "data": [[7]], "connector": "excel.table.write", "name": "Results", - "action": "replace", "sheet": expected, "cell": "A1"} + "action": "error", "sheet": expected, "cell": "A1"} def test_table_write_preserves_explicit_sheet_and_normalizes_cell(table_memory): @@ -714,5 +744,5 @@ def test_table_write_keeps_default_location_across_external_batches(table_memory variables={"recipe_name": "Clean", "batch_number": number, "batch_total": 2}, ) assert [(p["sheet"], p["cell"], p["action"], p["data"]) for p in memory.dataframes.values()] == [ - ("Clean-Resu", "A1", "replace", [[1]]), ("Clean-Resu", "A1", "append", [[2]]), + ("Clean-Resu", "A1", "error", [[1]]), ("Clean-Resu", "A1", "error", [[2]]), ] diff --git a/wrangles/connectors/excel.py b/wrangles/connectors/excel.py index b5737b01..930ee479 100644 --- a/wrangles/connectors/excel.py +++ b/wrangles/connectors/excel.py @@ -318,15 +318,20 @@ def read(name: str, variables: dict = None): def write( df: _pd.DataFrame, name: str, - action: str = "replace", + action: str = "error", variables: dict = None, sheet: str = None, cell: str = None, ): _validate_table_name(name) _validate_table_columns(df.columns.tolist()) - if action not in ("replace", "append"): - raise ValueError("excel.table action must be replace or append") + if action not in ("error", "replace", "append"): + raise ValueError("excel.table action must be error, replace or append") + if "*" in name: + if name.count("*") != 1 or not name.endswith("*") or len(name) == 1: + raise ValueError("excel.table wildcard must be a single trailing * after a name prefix") + if action != "error": + raise ValueError("excel.table wildcard names require action error (create a new table)") variables = variables or {} if sheet is None: recipe_name = variables.get("recipe_name") or "Recipe" @@ -393,7 +398,11 @@ def write( name: type: string minLength: 1 - description: Workbook-wide table name, matched case-insensitively. + description: >- + Workbook-wide table name, matched case-insensitively. A single + trailing * creates a numbered name, e.g. my_table_* becomes + my_table_1, then my_table_2. Requires action error. WranglesXL + allocates the name once per run, reusing it across batches. sheet: type: string minLength: 1 @@ -411,12 +420,13 @@ def write( Existing tables keep their current location. action: type: string - enum: [replace, append] - default: replace + enum: [error, replace, append] + default: error description: >- - replace clears and replaces body rows, including empty results; - append adds rows. The table resizes to fit the result. - Later external batches append after the first replacement. + error (default) creates a missing table and rejects an existing + workbook-wide name. Explicit replace clears and replaces body + rows, including empty results; explicit append adds rows. + Later external batches continue the same output table. """