feat(cli)!: return the canonical diff document from --format json (#693)

.github/workflows/ci.yml

@@ -263,6 +263,7 @@ jobs:
           tests/test_pdf_watermark_recall.py
           tests/test_formatters_text_serializer.py
           tests/test_pdf_xml_withheld_recall.py
+          tests/test_cli_api_parity.py
 
   # The required status check on `develop` and `main` is a context named exactly
   # `test`. The matrix jobs above report per-leg contexts (e.g., `fast-tests (3.12)`),

README.md

@@ -66,9 +66,9 @@ The product commands are the executable `.py` scripts in the project root — `d
 | `./tools/fetch_bills.py download-all --start_year <Y> --end_year <Y>` | Download all appropriations bills in a year range (or `--file <csv>` for a specific set; `--source govinfo\|api`) |
 | `./tools/fetch_bills.py search "<terms>" [--congress N] [--type hr] [--appropriations]` | Find bills by title over a local BILLSTATUS index (keyless, offline; **requires the index** — fetch it first with `fetch-index`, see below) |
 | `./tools/fetch_bills.py fetch-index --congress <N> [--type hr]` | Download just the scoped BILLSTATUS ZIP(s) that `search` reads (keyless; tens of MB, not the multi-GB full bulk set) — the lightweight on-ramp for `search` |
-| `./diff_bill.py compare <old.xml> <new.xml>` | Diff two XML versions (HTML by default; `--format json`, `--financial`, `--filter`, `-o`) |
+| `./diff_bill.py compare <old.xml> <new.xml>` | Diff two XML versions (HTML by default; `--format json` for canonical JSON, `--financial`, `--filter`, `-o`) |
 | `./diff_bill.py compare <slug> <n_old> <n_new>` | Diff two versions of a downloaded bill by ordinal, resolved under `--bills-dir` (default `bills/`); a bare `<slug>` lists that bill's local versions |
-| `./diff_pdf.py <old.pdf> <new.pdf> -o <out.html>` | Diff two PDF versions into the same HTML report |
+| `./diff_pdf.py <old.pdf> <new.pdf> -o <out.html>` | Diff two PDF versions into the same HTML report (`--format json` for the same canonical JSON, `--v1-label`/`--v2-label`) |
 | `./tools/fetch_bill_archives.py` | Bulk-build a full bill-metadata index (all of 112–119) from govinfo archives — **see the warning below** |
 | `./tools/fetch_bill_text_archives.py --from-congress <n> --to-congress <n>` | Bulk-download bill text from govinfo into `bills/` (no API key; `--min-versions 2` keeps only bills comparable across versions) |
 
@@ -133,21 +133,19 @@ The index is read from BILLSTATUS ZIPs in `bills/`, which are **not** part of a
 # Filter to a specific section
 ./diff_bill.py compare old.xml new.xml --filter "military construction"
 
-# Include unchanged sections
-./diff_bill.py compare old.xml new.xml --include-unchanged
-
-# Save the engine's internal diff dictionary (output defaults to HTML, so request json explicitly)
-./diff_bill.py compare old.xml new.xml --format json -o internal-diff.json
+# Save the canonical diff JSON (output defaults to HTML, so request json explicitly)
+./diff_bill.py compare old.xml new.xml --format json -o diff.json
 
 # Generate a standalone HTML report
 ./diff_bill.py compare old.xml new.xml --format html -o reports/report.html
 ```
 
-**Building something against the output?** `--format json` emits the engine's current
-*internal* diff dictionary, not the versioned canonical JSON that is the published
-interchange contract between the engine and its consumers. Use the canonical document
-instead: [`schema/canonical-diff.md`](schema/canonical-diff.md) specifies it, and the HTML
-report's **Export and share → Download `diff.json`** button produces one.
+**Building something against the output?** `--format json` gives you the canonical diff
+JSON, the versioned interchange contract between the engine and its consumers, specified
+in [`schema/canonical-diff.md`](schema/canonical-diff.md). It is the same document
+`POST /api/compare?output=json` returns and the same one the HTML report's **Export and
+share → Download `diff.json`** button saves, so a script, the web service and a browser
+download all read one shape. `./diff_pdf.py --format json` does the same for a PDF pair.
 
 ### HTML report
 

docs/architecture.md

@@ -40,9 +40,11 @@ enter through them — so one bill pair renders one way no matter who asked. Tho
 modules' docstrings name each stage they call, and are the shortest accurate map of the
 pipeline.
 
-One path deliberately does not go through them: `diff_bill.py compare --format json`
-emits the older diff-dict shape straight from `bill_diff_to_dict`, not canonical JSON.
-If you are consuming diff output programmatically, take the canonical JSON.
+Every path now goes through them, in both output formats. `diff_bill.py compare
+--format json` emitted the older diff-dict shape straight from `bill_diff_to_dict` until
+[#693](https://github.com/AgoraDMV/DeltaTrack/issues/693); that shape is a pipeline stage
+rather than a second output format, and only the XML branch has one. It remains available
+as a library call, which is why no flag was kept to select it.
 
 ## Pipeline tour
 

scripts/compare_differs.py

@@ -17,27 +17,32 @@
 """
 
 import difflib
-import json
-import subprocess
 import sys
 from collections import Counter
 from pathlib import Path
 
-HERE = Path(__file__).resolve().parent.parent
-PY = sys.executable  # the interpreter running this script already has the deps
+from deltatrack.bill_tree import normalize_bill
+from deltatrack.diff_bill import bill_diff_to_dict, diff_bills, filter_diff
 
 
 def our_tool(old_xml: Path, new_xml: Path) -> dict:
-    """DeltaTrack structured financial diff."""
-    out = subprocess.run(
-        [PY, "diff_bill.py", "compare", str(old_xml), str(new_xml), "--financial", "--format", "json"],
-        cwd=HERE,
-        capture_output=True,
-        text=True,
-    )
-    if out.returncode != 0:
-        raise SystemExit(f"diff_bill.py failed ({out.returncode}):\n{out.stderr}")
-    return json.loads(out.stdout)
+    """DeltaTrack structured financial diff.
+
+    Calls the engine as a library rather than shelling out to `diff_bill.py compare
+    --financial --format json`, which returns the canonical diff document since #693.
+    That document deliberately carries no money on a change: #671 removed
+    `amount_entries`, and the account-level model that would let a figure say what it IS
+    (top-line appropriation, sub-allocation, ceiling, limitation) is deferred to #115.
+    The paired old/new amounts this comparison exists to show are therefore not in it.
+
+    `bill_diff_to_dict` is where they do live. It is the engine's internal diff
+    dictionary, a pipeline stage rather than a published contract, and calling it here is
+    the intended way to reach it: #693 removed that shape from the command-line surface
+    precisely because it stays available as a library call. Nothing outside this
+    repository should read it; this script is inside it.
+    """
+    diff = filter_diff(diff_bills(normalize_bill(old_xml), normalize_bill(new_xml)), financial_only=True)
+    return bill_diff_to_dict(diff, financial=True)
 
 
 def xmldiff_actions(old_xml: Path, new_xml: Path) -> list:
@@ -81,7 +86,10 @@ def main() -> None:
     print(f"  {fin} accounts with dollar changes, each as paired old->new amounts")
     for c in d["changes"][:3]:
         f = c["financial"]
-        path = " > ".join(c.get("match_path", []))
+        # Subscript rather than `.get("match_path", [])`: the default silently printed an
+        # empty breadcrumb for every row once the shape changed, which reads as a bill
+        # with unnamed accounts rather than as a broken script (#693).
+        path = " > ".join(c["match_path"])
         print(f"    {path}: {f['old_amounts'][:1]} -> {f['new_amounts'][:1]} ...")
 
     print("\nxmldiff (off-the-shelf structural XML differ)")

src/deltatrack/compare/xml.py

@@ -16,8 +16,8 @@
 ``versions.v2.source == "xml"`` to drop the PDF line-number gutter.
 
 **This module is the only place a bill-XML report is assembled** (#42). The web app,
-the ``diff_bill.py compare --format html`` CLI, and ``render_examples.py`` all enter
-here, so one bill pair renders one way no matter which surface asked for it. Each of
+the ``diff_bill.py compare`` CLI (both of its formats, #693), and ``render_examples.py``
+all enter here, so one bill pair renders one way no matter which surface asked for it. Each of
 those three used to assemble the canonical → view → HTML chain itself, and the copies
 had already drifted apart in which version metadata they set. ``diff_pdf.py`` delegates
 to ``compare/pdf.py`` for the same reason; this is the XML half of that pattern.
@@ -44,7 +44,6 @@ def _build_from_trees(
     end_label: str | None,
     old_version_number: int | None = None,
     new_version_number: int | None = None,
-    include_unchanged: bool = False,
     filter_text: str | None = None,
     financial_only: bool = False,
 ) -> tuple[dict, str]:
@@ -57,11 +56,15 @@ def _build_from_trees(
     None to keep the embedded names. The version *numbers* are the bill's legislative
     ordinals, which are known when the input is a numbered corpus filename and unknown
     for a web upload — the renderer prefixes the header with ``v1:``/``v2:`` only when
-    they are supplied. Financial enrichment is unconditional on the HTML path.
+    they are supplied. Financial enrichment is unconditional here, on every caller's
+    behalf, so ``financial_only`` filters and nothing else.
+
+    Unchanged nodes are never carried: ``xml_diff_to_canonical`` drops them, so a
+    ``filter_diff(include_unchanged=True)`` here would only inflate
+    ``summary.unchanged`` to a count of entries the document does not contain (#693).
     """
     result = filter_diff(
         diff_bills(old_tree, new_tree),
-        include_unchanged=include_unchanged,
         filter_text=filter_text,
         financial_only=financial_only,
     )
@@ -137,6 +140,40 @@ def _render(canonical: dict, title: str) -> str:
     return format_diff_html(canonical, title)
 
 
+def compare_xml_trees(
+    old_tree: BillTree,
+    new_tree: BillTree,
+    *,
+    start_label: str | None = None,
+    end_label: str | None = None,
+    old_version_number: int | None = None,
+    new_version_number: int | None = None,
+    filter_text: str | None = None,
+    financial_only: bool = False,
+) -> dict:
+    """Canonical diff JSON for two already-parsed versions (see schema/canonical-diff.md).
+
+    The JSON sibling of :func:`compare_xml_trees_html`, and what
+    ``diff_bill.py compare --format json`` returns. The two formats are the same
+    document rendered two ways, which is the point of routing both through here (#693):
+    the command line used to serialize the engine's internal diff dictionary instead,
+    so a consumer could reach the published contract only by downloading it from a
+    rendered report in a browser.
+
+    See :func:`_build_from_trees` for what the version metadata does.
+    """
+    return _build_from_trees(
+        old_tree,
+        new_tree,
+        start_label=start_label,
+        end_label=end_label,
+        old_version_number=old_version_number,
+        new_version_number=new_version_number,
+        filter_text=filter_text,
+        financial_only=financial_only,
+    )[0]
+
+
 def compare_xml_trees_html(
     old_tree: BillTree,
     new_tree: BillTree,
@@ -145,7 +182,6 @@ def compare_xml_trees_html(
     end_label: str | None = None,
     old_version_number: int | None = None,
     new_version_number: int | None = None,
-    include_unchanged: bool = False,
     filter_text: str | None = None,
     financial_only: bool = False,
 ) -> str:
@@ -162,7 +198,6 @@ def compare_xml_trees_html(
         end_label=end_label,
         old_version_number=old_version_number,
         new_version_number=new_version_number,
-        include_unchanged=include_unchanged,
         filter_text=filter_text,
         financial_only=financial_only,
     )

src/deltatrack/diff_bill.py

@@ -2062,40 +2062,36 @@ def cmd_compare(args: argparse.Namespace) -> None:
     old_path, new_path = _compare_targets(args)
     old_tree = normalize_bill(old_path)
     new_tree = normalize_bill(new_path)
-    fmt = getattr(args, "format", "json")
-
-    if fmt == "html":
-        # Imported here, not at module scope: compare.xml imports this module.
-        # It owns the whole XML → HTML chain (#42), so the CLI, the web app, and
-        # render_examples.py cannot drift into rendering the same pair differently.
-        from deltatrack.compare.xml import compare_xml_trees_html
-
-        old_stem, new_stem = old_path.stem, new_path.stem
-        output = compare_xml_trees_html(
-            old_tree,
-            new_tree,
-            start_label=label_from_stem(old_stem),
-            end_label=label_from_stem(new_stem),
-            old_version_number=version_number_from_stem(old_stem),
-            new_version_number=version_number_from_stem(new_stem),
-            include_unchanged=args.include_unchanged,
-            filter_text=args.filter,
-            financial_only=args.financial,
-        )
-    else:
-        result = filter_diff(
-            diff_bills(old_tree, new_tree),
-            include_unchanged=args.include_unchanged,
-            filter_text=args.filter,
-            financial_only=args.financial,
-        )
-        diff_dict = bill_diff_to_dict(result, financial=args.financial)
-        # Extract version numbers from filenames (e.g., "1_reported-in-house.xml" -> 1)
-        for key, path in (("old_version_number", old_path), ("new_version_number", new_path)):
-            num = version_number_from_stem(path.stem)
-            if num is not None:
-                diff_dict[key] = num
-        output = json.dumps(diff_dict, indent=2)
+    fmt = getattr(args, "format", "html")
+
+    # Imported here, not at module scope: compare.xml imports this module.
+    # It owns the whole XML → report chain (#42), so the CLI, the web app, and
+    # render_examples.py cannot drift into rendering the same pair differently.
+    #
+    # BOTH formats enter it (#693). `--format json` used to serialize this module's
+    # own `bill_diff_to_dict` output, which is a pipeline stage rather than a
+    # published contract: it carries no `schema_version`, is specified nowhere, and
+    # shared only two top-level keys with what `POST /api/compare?output=json`
+    # returns. So the two surfaces answered the same question in two vocabularies,
+    # and the canonical document was reachable only by rendering HTML and clicking
+    # the download button in a browser, which a script cannot do. The internal
+    # shape stays reachable as a library call (`bill_diff_to_dict`); what is gone is
+    # its appearance on a command-line surface that documents the other one.
+    from deltatrack.compare.xml import compare_xml_trees, compare_xml_trees_html
+
+    old_stem, new_stem = old_path.stem, new_path.stem
+    build = compare_xml_trees_html if fmt == "html" else compare_xml_trees
+    result = build(
+        old_tree,
+        new_tree,
+        start_label=label_from_stem(old_stem),
+        end_label=label_from_stem(new_stem),
+        old_version_number=version_number_from_stem(old_stem),
+        new_version_number=version_number_from_stem(new_stem),
+        filter_text=args.filter,
+        financial_only=args.financial,
+    )
+    output = result if fmt == "html" else json.dumps(result, indent=2)
 
     if args.output:
         with open(args.output, "w", encoding="utf-8") as f:
@@ -2179,29 +2175,27 @@ def build_parser() -> argparse.ArgumentParser:
             "directory listing (compare <abs-dir>), which doesn't need this flag."
         ),
     )
-    compare.add_argument("-o", "--output", help="Output JSON file (default: stdout)")
-    compare.add_argument(
-        "--include-unchanged",
-        action="store_true",
-        help="Include unchanged nodes in output",
-    )
+    compare.add_argument("-o", "--output", help="Output file (default: stdout)")
     compare.add_argument(
         "--filter",
-        help="Only include nodes whose match_path contains this substring",
+        help=(
+            "Only include changes whose section breadcrumb contains this substring "
+            "(case-insensitive; the division is not part of the breadcrumb matched)."
+        ),
     )
     compare.add_argument(
         "--financial",
         action="store_true",
-        help=(
-            "Only show sections whose set of dollar figures differs between versions; "
-            "adds each side's amounts to the JSON output"
-        ),
+        help="Only show sections whose set of dollar figures differs between versions",
     )
     compare.add_argument(
         "--format",
         choices=["json", "html"],
         default="html",
-        help="Output format (default: html)",
+        help=(
+            "Output format: an HTML report, or the canonical diff JSON the web endpoint "
+            "returns (schema/canonical-diff.md). Default: html."
+        ),
     )
 
     return parser