From f70776517866c4970ec1f488db53772805769df0 Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Tue, 25 Aug 2026 04:14:10 +0200 Subject: [PATCH 1/4] Encode product reports as UTF-8 Use explicit UTF-8 at both product report output boundaries. Refs #627 Co-Authored-By: GPT-5.6 Luna --- src/deltatrack/diff_bill.py | 2 +- src/deltatrack/diff_pdf.py | 2 +- tests/test_diff_bill.py | 67 +++++++++++++++++++++++++++++++++++++ tests/test_diff_pdf_cli.py | 56 +++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/deltatrack/diff_bill.py b/src/deltatrack/diff_bill.py index dac6b1a2..037a4887 100644 --- a/src/deltatrack/diff_bill.py +++ b/src/deltatrack/diff_bill.py @@ -2096,7 +2096,7 @@ def cmd_compare(args: argparse.Namespace) -> None: output = json.dumps(diff_dict, indent=2) if args.output: - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8") as f: f.write(output) else: print(output) diff --git a/src/deltatrack/diff_pdf.py b/src/deltatrack/diff_pdf.py index 3284f14b..2878afd9 100644 --- a/src/deltatrack/diff_pdf.py +++ b/src/deltatrack/diff_pdf.py @@ -1378,7 +1378,7 @@ def main(argv: list[str] | None = None) -> None: v2_label=args.v2_label, ) if args.output: - args.output.write_text(html) + args.output.write_text(html, encoding="utf-8") print(f"Wrote {args.output}", file=sys.stderr) else: print(html) diff --git a/tests/test_diff_bill.py b/tests/test_diff_bill.py index c517dd43..fdae0973 100644 --- a/tests/test_diff_bill.py +++ b/tests/test_diff_bill.py @@ -1,5 +1,6 @@ import argparse import json +import os import subprocess import sys from pathlib import Path @@ -641,6 +642,9 @@ def _run_compare(monkeypatch, *argv: str) -> None: main() +REPORT = "

old → new

⚠ unanchored

" + + class TestIntermixedSubParserGuard: """The re-entrancy guard in _IntermixedSubParser, pinned on ANY interpreter (#426). @@ -870,6 +874,69 @@ def test_output_flag_still_writes_the_file_and_nothing_to_stdout( assert data["old_version"] == "reported-in-house" assert data["new_version"] == "enrolled-bill" + def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monkeypatch): + """The real CLI writer preserves report bytes independently of the host default.""" + old_xml = tmp_path / "old.xml" + new_xml = tmp_path / "new.xml" + output = tmp_path / "diff.html" + old_xml.write_bytes(b"") + new_xml.write_bytes(b"") + + import deltatrack.diff_bill as diff_bill + from deltatrack.compare import xml as compare_xml + + monkeypatch.setattr(diff_bill, "normalize_bill", lambda _path: object()) + monkeypatch.setattr(compare_xml, "compare_xml_trees_html", lambda *_args, **_kwargs: REPORT) + + original_open = open + + def open_with_cp1252_default( + file, + mode="r", + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None, + ): + if ( + os.fspath(file) == os.fspath(output) + and mode.startswith("w") + and "b" not in mode + and encoding in (None, "locale") + ): + encoding = "cp1252" + return original_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + + monkeypatch.setattr("builtins.open", open_with_cp1252_default) + + encoding_error = None + try: + _run_compare( + monkeypatch, + str(old_xml), + str(new_xml), + "--format", + "html", + "-o", + str(output), + ) + except UnicodeEncodeError as exc: + encoding_error = exc + + assert encoding_error is None, "report output depended on the host default encoding" + report = output.read_bytes() + assert report == REPORT.encode("utf-8") + decoded = report.decode("utf-8") + assert "→" in decoded + assert "⚠" in decoded + + unrelated = tmp_path / "unrelated.txt" + with open(unrelated, "w") as handle: + handle.write("→") + assert unrelated.read_bytes() == "→".encode("utf-8") + class TestCompareVersionAddressableForm: """`compare ` resolves under --bills-dir and diffs (#152).""" diff --git a/tests/test_diff_pdf_cli.py b/tests/test_diff_pdf_cli.py index 49bb9804..ac1d5847 100644 --- a/tests/test_diff_pdf_cli.py +++ b/tests/test_diff_pdf_cli.py @@ -2,6 +2,8 @@ from __future__ import annotations +import io +import os from pathlib import Path from deltatrack.diff_pdf import build_parser, main @@ -9,6 +11,7 @@ V1 = fixture_path("118-hr-8752", "1_reported-in-house.pdf") V2 = fixture_path("118-hr-8752", "2_engrossed-in-house.pdf") +REPORT = "

old → new

⚠ unanchored

" def test_fixtures_committed(): @@ -47,6 +50,59 @@ def test_writes_html_file(self, tmp_path): assert "full-bill" in html assert "diff.json" in html + def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monkeypatch): + """The real CLI writer preserves report bytes independently of the host default.""" + output = tmp_path / "diff.html" + v1_pdf = tmp_path / "old.pdf" + v2_pdf = tmp_path / "new.pdf" + v1_pdf.write_bytes(b"old") + v2_pdf.write_bytes(b"new") + + import deltatrack.diff_pdf as diff_pdf + + monkeypatch.setattr(diff_pdf, "render_pdf_diff_html", lambda *_args, **_kwargs: REPORT) + + original_io_open = io.open + + def io_open_with_cp1252_default( + file, + mode="r", + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None, + ): + if ( + os.fspath(file) == os.fspath(output) + and mode.startswith("w") + and "b" not in mode + and encoding in (None, "locale") + ): + encoding = "cp1252" + return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + + monkeypatch.setattr(io, "open", io_open_with_cp1252_default) + + encoding_error = None + try: + main([str(v1_pdf), str(v2_pdf), "-o", str(output)]) + except UnicodeEncodeError as exc: + encoding_error = exc + + assert encoding_error is None, "report output depended on the host default encoding" + report = output.read_bytes() + assert report == REPORT.encode("utf-8") + decoded = report.decode("utf-8") + assert "→" in decoded + assert "⚠" in decoded + + unrelated = tmp_path / "unrelated.txt" + with open(unrelated, "w") as handle: + handle.write("→") + assert unrelated.read_bytes() == "→".encode("utf-8") + def test_stdout_when_no_output(self, capsys): main([str(V1), str(V2)]) captured = capsys.readouterr() From 826c1dea09ec50ac79da4f0cb678dccb50b872f7 Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Tue, 25 Aug 2026 04:44:43 +0200 Subject: [PATCH 2/4] Harden UTF-8 writer alias tests Cover both builtins.open and io.open while preserving unrelated writes. Co-Authored-By: GPT-5.6 Luna --- tests/test_diff_bill.py | 48 +++++++++++++++++++++++++++++++------- tests/test_diff_pdf_cli.py | 39 +++++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/tests/test_diff_bill.py b/tests/test_diff_bill.py index fdae0973..3e2d372c 100644 --- a/tests/test_diff_bill.py +++ b/tests/test_diff_bill.py @@ -1,4 +1,6 @@ import argparse +import builtins +import io import json import os import subprocess @@ -888,9 +890,10 @@ def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monke monkeypatch.setattr(diff_bill, "normalize_bill", lambda _path: object()) monkeypatch.setattr(compare_xml, "compare_xml_trees_html", lambda *_args, **_kwargs: REPORT) - original_open = open + original_builtin_open = builtins.open + original_io_open = io.open - def open_with_cp1252_default( + def builtin_open_with_cp1252_default( file, mode="r", buffering=-1, @@ -901,15 +904,39 @@ def open_with_cp1252_default( opener=None, ): if ( - os.fspath(file) == os.fspath(output) + isinstance(file, (str, bytes, os.PathLike)) + and isinstance(mode, str) + and os.fspath(file) == os.fspath(output) and mode.startswith("w") and "b" not in mode and encoding in (None, "locale") ): encoding = "cp1252" - return original_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + return original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - monkeypatch.setattr("builtins.open", open_with_cp1252_default) + def io_open_with_cp1252_default( + file, + mode="r", + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None, + ): + if ( + isinstance(file, (str, bytes, os.PathLike)) + and isinstance(mode, str) + and os.fspath(file) == os.fspath(output) + and mode.startswith("w") + and "b" not in mode + and encoding in (None, "locale") + ): + encoding = "cp1252" + return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + + monkeypatch.setattr(builtins, "open", builtin_open_with_cp1252_default) + monkeypatch.setattr(io, "open", io_open_with_cp1252_default) encoding_error = None try: @@ -932,10 +959,15 @@ def open_with_cp1252_default( assert "→" in decoded assert "⚠" in decoded - unrelated = tmp_path / "unrelated.txt" - with open(unrelated, "w") as handle: + unrelated_builtin = tmp_path / "unrelated-builtins.txt" + with builtins.open(unrelated_builtin, "w") as handle: handle.write("→") - assert unrelated.read_bytes() == "→".encode("utf-8") + assert unrelated_builtin.read_bytes() == "→".encode("utf-8") + + unrelated_io = tmp_path / "unrelated-io.txt" + with io.open(unrelated_io, "w") as handle: + handle.write("⚠") + assert unrelated_io.read_bytes() == "⚠".encode("utf-8") class TestCompareVersionAddressableForm: diff --git a/tests/test_diff_pdf_cli.py b/tests/test_diff_pdf_cli.py index ac1d5847..d71c2e52 100644 --- a/tests/test_diff_pdf_cli.py +++ b/tests/test_diff_pdf_cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +import builtins import io import os from pathlib import Path @@ -62,8 +63,30 @@ def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monke monkeypatch.setattr(diff_pdf, "render_pdf_diff_html", lambda *_args, **_kwargs: REPORT) + original_builtin_open = builtins.open original_io_open = io.open + def builtin_open_with_cp1252_default( + file, + mode="r", + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None, + ): + if ( + isinstance(file, (str, bytes, os.PathLike)) + and isinstance(mode, str) + and os.fspath(file) == os.fspath(output) + and mode.startswith("w") + and "b" not in mode + and encoding in (None, "locale") + ): + encoding = "cp1252" + return original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + def io_open_with_cp1252_default( file, mode="r", @@ -75,7 +98,9 @@ def io_open_with_cp1252_default( opener=None, ): if ( - os.fspath(file) == os.fspath(output) + isinstance(file, (str, bytes, os.PathLike)) + and isinstance(mode, str) + and os.fspath(file) == os.fspath(output) and mode.startswith("w") and "b" not in mode and encoding in (None, "locale") @@ -83,6 +108,7 @@ def io_open_with_cp1252_default( encoding = "cp1252" return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + monkeypatch.setattr(builtins, "open", builtin_open_with_cp1252_default) monkeypatch.setattr(io, "open", io_open_with_cp1252_default) encoding_error = None @@ -98,10 +124,15 @@ def io_open_with_cp1252_default( assert "→" in decoded assert "⚠" in decoded - unrelated = tmp_path / "unrelated.txt" - with open(unrelated, "w") as handle: + unrelated_builtin = tmp_path / "unrelated-builtins.txt" + with builtins.open(unrelated_builtin, "w") as handle: handle.write("→") - assert unrelated.read_bytes() == "→".encode("utf-8") + assert unrelated_builtin.read_bytes() == "→".encode("utf-8") + + unrelated_io = tmp_path / "unrelated-io.txt" + with io.open(unrelated_io, "w") as handle: + handle.write("⚠") + assert unrelated_io.read_bytes() == "⚠".encode("utf-8") def test_stdout_when_no_output(self, capsys): main([str(V1), str(V2)]) From 469b886c29a4df1858dc44424a07accba4eb919b Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Tue, 25 Aug 2026 05:17:29 +0200 Subject: [PATCH 3/4] Cover descriptor-backed UTF-8 writers Associate only exact-output descriptors with the simulated default encoding and verify unrelated descriptors stay untouched. Co-Authored-By: GPT-5.6 Luna --- tests/test_diff_bill.py | 109 +++++++++++++++++++++++++++++++------ tests/test_diff_pdf_cli.py | 109 +++++++++++++++++++++++++++++++------ 2 files changed, 186 insertions(+), 32 deletions(-) diff --git a/tests/test_diff_bill.py b/tests/test_diff_bill.py index 3e2d372c..a55c7686 100644 --- a/tests/test_diff_bill.py +++ b/tests/test_diff_bill.py @@ -892,6 +892,62 @@ def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monke original_builtin_open = builtins.open original_io_open = io.open + original_os_open = os.open + target_fds = set() + + class _TrackedTextHandle: + def __init__(self, handle, fd): + self._handle = handle + self._fd = fd + + def __enter__(self): + self._handle.__enter__() + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + return self._handle.__exit__(exc_type, exc_value, traceback) + finally: + target_fds.discard(self._fd) + + def close(self): + try: + return self._handle.close() + finally: + target_fds.discard(self._fd) + + def __getattr__(self, name): + return getattr(self._handle, name) + + def __del__(self): + target_fds.discard(getattr(self, "_fd", None)) + + def is_output_path(file): + try: + return os.fspath(file) == os.fspath(output) + except TypeError: + return False + + def is_target_descriptor(file): + return isinstance(file, int) and file in target_fds + + def uses_default_text_encoding(file, mode, encoding): + return ( + (is_output_path(file) or is_target_descriptor(file)) + and isinstance(mode, str) + and mode.startswith("w") + and "b" not in mode + and encoding in (None, "locale") + ) + + def tracked_os_open(*args, **kwargs): + fd = original_os_open(*args, **kwargs) + path = args[0] if args else kwargs.get("path") + if is_output_path(path): + target_fds.add(fd) + return fd + + monkeypatch.setattr(os, "open", tracked_os_open) def builtin_open_with_cp1252_default( file, @@ -903,15 +959,13 @@ def builtin_open_with_cp1252_default( closefd=True, opener=None, ): - if ( - isinstance(file, (str, bytes, os.PathLike)) - and isinstance(mode, str) - and os.fspath(file) == os.fspath(output) - and mode.startswith("w") - and "b" not in mode - and encoding in (None, "locale") - ): + target_descriptor = is_target_descriptor(file) + if uses_default_text_encoding(file, mode, encoding): encoding = "cp1252" + handle = original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + if target_descriptor: + return _TrackedTextHandle(handle, file) + return handle return original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) def io_open_with_cp1252_default( @@ -924,15 +978,13 @@ def io_open_with_cp1252_default( closefd=True, opener=None, ): - if ( - isinstance(file, (str, bytes, os.PathLike)) - and isinstance(mode, str) - and os.fspath(file) == os.fspath(output) - and mode.startswith("w") - and "b" not in mode - and encoding in (None, "locale") - ): + target_descriptor = is_target_descriptor(file) + if uses_default_text_encoding(file, mode, encoding): encoding = "cp1252" + handle = original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + if target_descriptor: + return _TrackedTextHandle(handle, file) + return handle return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) monkeypatch.setattr(builtins, "open", builtin_open_with_cp1252_default) @@ -969,6 +1021,31 @@ def io_open_with_cp1252_default( handle.write("⚠") assert unrelated_io.read_bytes() == "⚠".encode("utf-8") + unrelated_builtin_fd_path = tmp_path / "unrelated-builtins-fd.txt" + unrelated_builtin_fd = os.open(unrelated_builtin_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) + try: + with builtins.open(unrelated_builtin_fd, "w") as handle: + handle.write("→") + finally: + try: + os.close(unrelated_builtin_fd) + except OSError: + pass + assert unrelated_builtin_fd_path.read_bytes() == "→".encode("utf-8") + + unrelated_io_fd_path = tmp_path / "unrelated-io-fd.txt" + unrelated_io_fd = os.open(unrelated_io_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) + try: + with os.fdopen(unrelated_io_fd, "w") as handle: + handle.write("⚠") + finally: + try: + os.close(unrelated_io_fd) + except OSError: + pass + assert unrelated_io_fd_path.read_bytes() == "⚠".encode("utf-8") + assert not target_fds + class TestCompareVersionAddressableForm: """`compare ` resolves under --bills-dir and diffs (#152).""" diff --git a/tests/test_diff_pdf_cli.py b/tests/test_diff_pdf_cli.py index d71c2e52..bf94230b 100644 --- a/tests/test_diff_pdf_cli.py +++ b/tests/test_diff_pdf_cli.py @@ -65,6 +65,62 @@ def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monke original_builtin_open = builtins.open original_io_open = io.open + original_os_open = os.open + target_fds = set() + + class _TrackedTextHandle: + def __init__(self, handle, fd): + self._handle = handle + self._fd = fd + + def __enter__(self): + self._handle.__enter__() + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + return self._handle.__exit__(exc_type, exc_value, traceback) + finally: + target_fds.discard(self._fd) + + def close(self): + try: + return self._handle.close() + finally: + target_fds.discard(self._fd) + + def __getattr__(self, name): + return getattr(self._handle, name) + + def __del__(self): + target_fds.discard(getattr(self, "_fd", None)) + + def is_output_path(file): + try: + return os.fspath(file) == os.fspath(output) + except TypeError: + return False + + def is_target_descriptor(file): + return isinstance(file, int) and file in target_fds + + def uses_default_text_encoding(file, mode, encoding): + return ( + (is_output_path(file) or is_target_descriptor(file)) + and isinstance(mode, str) + and mode.startswith("w") + and "b" not in mode + and encoding in (None, "locale") + ) + + def tracked_os_open(*args, **kwargs): + fd = original_os_open(*args, **kwargs) + path = args[0] if args else kwargs.get("path") + if is_output_path(path): + target_fds.add(fd) + return fd + + monkeypatch.setattr(os, "open", tracked_os_open) def builtin_open_with_cp1252_default( file, @@ -76,15 +132,13 @@ def builtin_open_with_cp1252_default( closefd=True, opener=None, ): - if ( - isinstance(file, (str, bytes, os.PathLike)) - and isinstance(mode, str) - and os.fspath(file) == os.fspath(output) - and mode.startswith("w") - and "b" not in mode - and encoding in (None, "locale") - ): + target_descriptor = is_target_descriptor(file) + if uses_default_text_encoding(file, mode, encoding): encoding = "cp1252" + handle = original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + if target_descriptor: + return _TrackedTextHandle(handle, file) + return handle return original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) def io_open_with_cp1252_default( @@ -97,15 +151,13 @@ def io_open_with_cp1252_default( closefd=True, opener=None, ): - if ( - isinstance(file, (str, bytes, os.PathLike)) - and isinstance(mode, str) - and os.fspath(file) == os.fspath(output) - and mode.startswith("w") - and "b" not in mode - and encoding in (None, "locale") - ): + target_descriptor = is_target_descriptor(file) + if uses_default_text_encoding(file, mode, encoding): encoding = "cp1252" + handle = original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) + if target_descriptor: + return _TrackedTextHandle(handle, file) + return handle return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) monkeypatch.setattr(builtins, "open", builtin_open_with_cp1252_default) @@ -134,6 +186,31 @@ def io_open_with_cp1252_default( handle.write("⚠") assert unrelated_io.read_bytes() == "⚠".encode("utf-8") + unrelated_builtin_fd_path = tmp_path / "unrelated-builtins-fd.txt" + unrelated_builtin_fd = os.open(unrelated_builtin_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) + try: + with builtins.open(unrelated_builtin_fd, "w") as handle: + handle.write("→") + finally: + try: + os.close(unrelated_builtin_fd) + except OSError: + pass + assert unrelated_builtin_fd_path.read_bytes() == "→".encode("utf-8") + + unrelated_io_fd_path = tmp_path / "unrelated-io-fd.txt" + unrelated_io_fd = os.open(unrelated_io_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) + try: + with os.fdopen(unrelated_io_fd, "w") as handle: + handle.write("⚠") + finally: + try: + os.close(unrelated_io_fd) + except OSError: + pass + assert unrelated_io_fd_path.read_bytes() == "⚠".encode("utf-8") + assert not target_fds + def test_stdout_when_no_output(self, capsys): main([str(V1), str(V2)]) captured = capsys.readouterr() From 0f94e3aba052d0ed4976da0767475d2a16742e8f Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Tue, 25 Aug 2026 05:53:08 +0200 Subject: [PATCH 4/4] Use a non-UTF-8 subprocess for report output tests Replace stateful descriptor simulation with a verified locale boundary that runs each real CLI route and reports setup, encoding, and byte outcomes distinctly. Co-Authored-By: GPT-5.6 Luna --- tests/test_diff_bill.py | 315 ++++++++++++++++++------------------- tests/test_diff_pdf_cli.py | 289 ++++++++++++++++------------------ 2 files changed, 282 insertions(+), 322 deletions(-) diff --git a/tests/test_diff_bill.py b/tests/test_diff_bill.py index a55c7686..1120b08a 100644 --- a/tests/test_diff_bill.py +++ b/tests/test_diff_bill.py @@ -1,6 +1,4 @@ import argparse -import builtins -import io import json import os import subprocess @@ -646,6 +644,152 @@ def _run_compare(monkeypatch, *argv: str) -> None: REPORT = "

old → new

⚠ unanchored

" +_UTF8_ROUTE_CHILD = r""" +import locale +import os +import sys +from pathlib import Path + +REPORT = "

old \u2192 new

\u26a0 unanchored

" + + +def _ascii(value): + return str(value).encode("ascii", "backslashreplace").decode("ascii") + + +def _status(name): + print("DT627_STATUS=" + name) + return 0 + + +def _utf8_name(value): + return str(value).lower().replace("-", "").replace("_", "") in {"utf8", "utf"} + + +def main(): + route, output, old_input, new_input = sys.argv[1:] + try: + locale_encoding = locale.getencoding() + with open(os.devnull, "w") as probe: + file_encoding = probe.encoding + except Exception as exc: + print("DT627_STATUS=locale-unavailable") + print("DT627_DETAIL=" + type(exc).__name__) + return 11 + + print("DT627_DEFAULT_ENCODING=" + _ascii(locale_encoding)) + print("DT627_FILE_ENCODING=" + _ascii(file_encoding)) + if _utf8_name(locale_encoding) or _utf8_name(file_encoding): + return _status("locale-unavailable") + + try: + if route == "xml": + import deltatrack.diff_bill as diff_bill + from deltatrack.compare import xml as compare_xml + + diff_bill.normalize_bill = lambda _path: object() + compare_xml.compare_xml_trees_html = lambda *_args, **_kwargs: REPORT + sys.argv = [ + "diff_bill.py", + "compare", + old_input, + new_input, + "--format", + "html", + "-o", + output, + ] + route_main = diff_bill.main + elif route == "pdf": + import deltatrack.diff_pdf as diff_pdf + + diff_pdf.render_pdf_diff_html = lambda *_args, **_kwargs: REPORT + route_main = lambda: diff_pdf.main( + [old_input, new_input, "--output", output] + ) + else: + print("DT627_STATUS=setup-error") + print("DT627_DETAIL=unknown-route") + return 12 + except Exception as exc: + print("DT627_STATUS=setup-error") + print("DT627_DETAIL=" + type(exc).__name__) + return 12 + + try: + route_main() + except UnicodeEncodeError: + return _status("unicode-error") + except SystemExit as exc: + print("DT627_STATUS=route-error") + print("DT627_DETAIL=SystemExit:" + _ascii(exc.code)) + return 13 + except Exception as exc: + print("DT627_STATUS=route-error") + print("DT627_DETAIL=" + type(exc).__name__) + return 13 + + try: + report = Path(output).read_bytes() + except Exception as exc: + print("DT627_STATUS=byte-mismatch") + print("DT627_DETAIL=" + type(exc).__name__) + return 0 + if report != REPORT.encode("utf-8"): + return _status("byte-mismatch") + try: + decoded = report.decode("utf-8") + except UnicodeDecodeError: + return _status("decode-error") + if "\u2192" not in decoded or "\u26a0" not in decoded: + return _status("marker-mismatch") + return _status("ok") + + +raise SystemExit(main()) +""" + + +def _run_non_utf8_report_child(route: str, output: Path, old_input: Path, new_input: Path) -> None: + repo_root = Path(__file__).resolve().parents[1] + environment = os.environ.copy() + environment.update({"PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0", "LC_ALL": "C", "LANG": "C"}) + pythonpath = [str(repo_root / "src"), str(repo_root)] + if environment.get("PYTHONPATH"): + pythonpath.append(environment["PYTHONPATH"]) + environment["PYTHONPATH"] = os.pathsep.join(pythonpath) + result = subprocess.run( + [ + sys.executable, + "-c", + _UTF8_ROUTE_CHILD, + route, + str(output), + str(old_input), + str(new_input), + ], + cwd=repo_root, + env=environment, + capture_output=True, + text=True, + ) + status = next( + (line for line in result.stdout.splitlines() if line.startswith("DT627_STATUS=")), + "", + ) + assert result.returncode == 0, ( + f"child setup/route failure ({status}); stdout={result.stdout!r}; stderr={result.stderr!r}" + ) + assert status == "DT627_STATUS=ok", ( + f"report route did not preserve UTF-8 ({status}); stdout={result.stdout!r}; stderr={result.stderr!r}" + ) + + report = output.read_bytes() + assert report == REPORT.encode("utf-8") + decoded = report.decode("utf-8") + assert "→" in decoded + assert "⚠" in decoded + class TestIntermixedSubParserGuard: """The re-entrancy guard in _IntermixedSubParser, pinned on ANY interpreter (#426). @@ -876,175 +1020,14 @@ def test_output_flag_still_writes_the_file_and_nothing_to_stdout( assert data["old_version"] == "reported-in-house" assert data["new_version"] == "enrolled-bill" - def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monkeypatch): - """The real CLI writer preserves report bytes independently of the host default.""" + def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path): + """The real CLI writer is tested under a verified non-UTF-8 child locale.""" old_xml = tmp_path / "old.xml" new_xml = tmp_path / "new.xml" output = tmp_path / "diff.html" old_xml.write_bytes(b"") new_xml.write_bytes(b"") - - import deltatrack.diff_bill as diff_bill - from deltatrack.compare import xml as compare_xml - - monkeypatch.setattr(diff_bill, "normalize_bill", lambda _path: object()) - monkeypatch.setattr(compare_xml, "compare_xml_trees_html", lambda *_args, **_kwargs: REPORT) - - original_builtin_open = builtins.open - original_io_open = io.open - original_os_open = os.open - target_fds = set() - - class _TrackedTextHandle: - def __init__(self, handle, fd): - self._handle = handle - self._fd = fd - - def __enter__(self): - self._handle.__enter__() - return self - - def __exit__(self, exc_type, exc_value, traceback): - try: - return self._handle.__exit__(exc_type, exc_value, traceback) - finally: - target_fds.discard(self._fd) - - def close(self): - try: - return self._handle.close() - finally: - target_fds.discard(self._fd) - - def __getattr__(self, name): - return getattr(self._handle, name) - - def __del__(self): - target_fds.discard(getattr(self, "_fd", None)) - - def is_output_path(file): - try: - return os.fspath(file) == os.fspath(output) - except TypeError: - return False - - def is_target_descriptor(file): - return isinstance(file, int) and file in target_fds - - def uses_default_text_encoding(file, mode, encoding): - return ( - (is_output_path(file) or is_target_descriptor(file)) - and isinstance(mode, str) - and mode.startswith("w") - and "b" not in mode - and encoding in (None, "locale") - ) - - def tracked_os_open(*args, **kwargs): - fd = original_os_open(*args, **kwargs) - path = args[0] if args else kwargs.get("path") - if is_output_path(path): - target_fds.add(fd) - return fd - - monkeypatch.setattr(os, "open", tracked_os_open) - - def builtin_open_with_cp1252_default( - file, - mode="r", - buffering=-1, - encoding=None, - errors=None, - newline=None, - closefd=True, - opener=None, - ): - target_descriptor = is_target_descriptor(file) - if uses_default_text_encoding(file, mode, encoding): - encoding = "cp1252" - handle = original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - if target_descriptor: - return _TrackedTextHandle(handle, file) - return handle - return original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - - def io_open_with_cp1252_default( - file, - mode="r", - buffering=-1, - encoding=None, - errors=None, - newline=None, - closefd=True, - opener=None, - ): - target_descriptor = is_target_descriptor(file) - if uses_default_text_encoding(file, mode, encoding): - encoding = "cp1252" - handle = original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - if target_descriptor: - return _TrackedTextHandle(handle, file) - return handle - return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - - monkeypatch.setattr(builtins, "open", builtin_open_with_cp1252_default) - monkeypatch.setattr(io, "open", io_open_with_cp1252_default) - - encoding_error = None - try: - _run_compare( - monkeypatch, - str(old_xml), - str(new_xml), - "--format", - "html", - "-o", - str(output), - ) - except UnicodeEncodeError as exc: - encoding_error = exc - - assert encoding_error is None, "report output depended on the host default encoding" - report = output.read_bytes() - assert report == REPORT.encode("utf-8") - decoded = report.decode("utf-8") - assert "→" in decoded - assert "⚠" in decoded - - unrelated_builtin = tmp_path / "unrelated-builtins.txt" - with builtins.open(unrelated_builtin, "w") as handle: - handle.write("→") - assert unrelated_builtin.read_bytes() == "→".encode("utf-8") - - unrelated_io = tmp_path / "unrelated-io.txt" - with io.open(unrelated_io, "w") as handle: - handle.write("⚠") - assert unrelated_io.read_bytes() == "⚠".encode("utf-8") - - unrelated_builtin_fd_path = tmp_path / "unrelated-builtins-fd.txt" - unrelated_builtin_fd = os.open(unrelated_builtin_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) - try: - with builtins.open(unrelated_builtin_fd, "w") as handle: - handle.write("→") - finally: - try: - os.close(unrelated_builtin_fd) - except OSError: - pass - assert unrelated_builtin_fd_path.read_bytes() == "→".encode("utf-8") - - unrelated_io_fd_path = tmp_path / "unrelated-io-fd.txt" - unrelated_io_fd = os.open(unrelated_io_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) - try: - with os.fdopen(unrelated_io_fd, "w") as handle: - handle.write("⚠") - finally: - try: - os.close(unrelated_io_fd) - except OSError: - pass - assert unrelated_io_fd_path.read_bytes() == "⚠".encode("utf-8") - assert not target_fds + _run_non_utf8_report_child("xml", output, old_xml, new_xml) class TestCompareVersionAddressableForm: diff --git a/tests/test_diff_pdf_cli.py b/tests/test_diff_pdf_cli.py index bf94230b..2923b82e 100644 --- a/tests/test_diff_pdf_cli.py +++ b/tests/test_diff_pdf_cli.py @@ -2,9 +2,9 @@ from __future__ import annotations -import builtins -import io import os +import subprocess +import sys from pathlib import Path from deltatrack.diff_pdf import build_parser, main @@ -14,6 +14,134 @@ V2 = fixture_path("118-hr-8752", "2_engrossed-in-house.pdf") REPORT = "

old → new

⚠ unanchored

" +_UTF8_ROUTE_CHILD = r""" +import locale +import os +import sys +from pathlib import Path + +REPORT = "

old \u2192 new

\u26a0 unanchored

" + + +def _ascii(value): + return str(value).encode("ascii", "backslashreplace").decode("ascii") + + +def _status(name): + print("DT627_STATUS=" + name) + return 0 + + +def _utf8_name(value): + return str(value).lower().replace("-", "").replace("_", "") in {"utf8", "utf"} + + +def main(): + route, output, old_input, new_input = sys.argv[1:] + try: + locale_encoding = locale.getencoding() + with open(os.devnull, "w") as probe: + file_encoding = probe.encoding + except Exception as exc: + print("DT627_STATUS=locale-unavailable") + print("DT627_DETAIL=" + type(exc).__name__) + return 11 + + print("DT627_DEFAULT_ENCODING=" + _ascii(locale_encoding)) + print("DT627_FILE_ENCODING=" + _ascii(file_encoding)) + if _utf8_name(locale_encoding) or _utf8_name(file_encoding): + return _status("locale-unavailable") + + try: + if route != "pdf": + print("DT627_STATUS=setup-error") + print("DT627_DETAIL=unknown-route") + return 12 + import deltatrack.diff_pdf as diff_pdf + + diff_pdf.render_pdf_diff_html = lambda *_args, **_kwargs: REPORT + route_main = lambda: diff_pdf.main( + [old_input, new_input, "--output", output] + ) + except Exception as exc: + print("DT627_STATUS=setup-error") + print("DT627_DETAIL=" + type(exc).__name__) + return 12 + + try: + route_main() + except UnicodeEncodeError: + return _status("unicode-error") + except SystemExit as exc: + print("DT627_STATUS=route-error") + print("DT627_DETAIL=SystemExit:" + _ascii(exc.code)) + return 13 + except Exception as exc: + print("DT627_STATUS=route-error") + print("DT627_DETAIL=" + type(exc).__name__) + return 13 + + try: + report = Path(output).read_bytes() + except Exception as exc: + print("DT627_STATUS=byte-mismatch") + print("DT627_DETAIL=" + type(exc).__name__) + return 0 + if report != REPORT.encode("utf-8"): + return _status("byte-mismatch") + try: + decoded = report.decode("utf-8") + except UnicodeDecodeError: + return _status("decode-error") + if "\u2192" not in decoded or "\u26a0" not in decoded: + return _status("marker-mismatch") + return _status("ok") + + +raise SystemExit(main()) +""" + + +def _run_non_utf8_report_child(route: str, output: Path, old_input: Path, new_input: Path) -> None: + repo_root = Path(__file__).resolve().parents[1] + environment = os.environ.copy() + environment.update({"PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0", "LC_ALL": "C", "LANG": "C"}) + pythonpath = [str(repo_root / "src"), str(repo_root)] + if environment.get("PYTHONPATH"): + pythonpath.append(environment["PYTHONPATH"]) + environment["PYTHONPATH"] = os.pathsep.join(pythonpath) + result = subprocess.run( + [ + sys.executable, + "-c", + _UTF8_ROUTE_CHILD, + route, + str(output), + str(old_input), + str(new_input), + ], + cwd=repo_root, + env=environment, + capture_output=True, + text=True, + ) + status = next( + (line for line in result.stdout.splitlines() if line.startswith("DT627_STATUS=")), + "", + ) + assert result.returncode == 0, ( + f"child setup/route failure ({status}); stdout={result.stdout!r}; stderr={result.stderr!r}" + ) + assert status == "DT627_STATUS=ok", ( + f"report route did not preserve UTF-8 ({status}); stdout={result.stdout!r}; stderr={result.stderr!r}" + ) + + report = output.read_bytes() + assert report == REPORT.encode("utf-8") + decoded = report.decode("utf-8") + assert "→" in decoded + assert "⚠" in decoded + def test_fixtures_committed(): """Fail-closed floor (#326): both PDFs are committed and manifested, so an absent @@ -51,165 +179,14 @@ def test_writes_html_file(self, tmp_path): assert "full-bill" in html assert "diff.json" in html - def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path, monkeypatch): - """The real CLI writer preserves report bytes independently of the host default.""" + def test_html_output_uses_utf8_when_host_default_is_cp1252(self, tmp_path): + """The real CLI writer is tested under a verified non-UTF-8 child locale.""" output = tmp_path / "diff.html" v1_pdf = tmp_path / "old.pdf" v2_pdf = tmp_path / "new.pdf" v1_pdf.write_bytes(b"old") v2_pdf.write_bytes(b"new") - - import deltatrack.diff_pdf as diff_pdf - - monkeypatch.setattr(diff_pdf, "render_pdf_diff_html", lambda *_args, **_kwargs: REPORT) - - original_builtin_open = builtins.open - original_io_open = io.open - original_os_open = os.open - target_fds = set() - - class _TrackedTextHandle: - def __init__(self, handle, fd): - self._handle = handle - self._fd = fd - - def __enter__(self): - self._handle.__enter__() - return self - - def __exit__(self, exc_type, exc_value, traceback): - try: - return self._handle.__exit__(exc_type, exc_value, traceback) - finally: - target_fds.discard(self._fd) - - def close(self): - try: - return self._handle.close() - finally: - target_fds.discard(self._fd) - - def __getattr__(self, name): - return getattr(self._handle, name) - - def __del__(self): - target_fds.discard(getattr(self, "_fd", None)) - - def is_output_path(file): - try: - return os.fspath(file) == os.fspath(output) - except TypeError: - return False - - def is_target_descriptor(file): - return isinstance(file, int) and file in target_fds - - def uses_default_text_encoding(file, mode, encoding): - return ( - (is_output_path(file) or is_target_descriptor(file)) - and isinstance(mode, str) - and mode.startswith("w") - and "b" not in mode - and encoding in (None, "locale") - ) - - def tracked_os_open(*args, **kwargs): - fd = original_os_open(*args, **kwargs) - path = args[0] if args else kwargs.get("path") - if is_output_path(path): - target_fds.add(fd) - return fd - - monkeypatch.setattr(os, "open", tracked_os_open) - - def builtin_open_with_cp1252_default( - file, - mode="r", - buffering=-1, - encoding=None, - errors=None, - newline=None, - closefd=True, - opener=None, - ): - target_descriptor = is_target_descriptor(file) - if uses_default_text_encoding(file, mode, encoding): - encoding = "cp1252" - handle = original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - if target_descriptor: - return _TrackedTextHandle(handle, file) - return handle - return original_builtin_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - - def io_open_with_cp1252_default( - file, - mode="r", - buffering=-1, - encoding=None, - errors=None, - newline=None, - closefd=True, - opener=None, - ): - target_descriptor = is_target_descriptor(file) - if uses_default_text_encoding(file, mode, encoding): - encoding = "cp1252" - handle = original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - if target_descriptor: - return _TrackedTextHandle(handle, file) - return handle - return original_io_open(file, mode, buffering, encoding, errors, newline, closefd, opener) - - monkeypatch.setattr(builtins, "open", builtin_open_with_cp1252_default) - monkeypatch.setattr(io, "open", io_open_with_cp1252_default) - - encoding_error = None - try: - main([str(v1_pdf), str(v2_pdf), "-o", str(output)]) - except UnicodeEncodeError as exc: - encoding_error = exc - - assert encoding_error is None, "report output depended on the host default encoding" - report = output.read_bytes() - assert report == REPORT.encode("utf-8") - decoded = report.decode("utf-8") - assert "→" in decoded - assert "⚠" in decoded - - unrelated_builtin = tmp_path / "unrelated-builtins.txt" - with builtins.open(unrelated_builtin, "w") as handle: - handle.write("→") - assert unrelated_builtin.read_bytes() == "→".encode("utf-8") - - unrelated_io = tmp_path / "unrelated-io.txt" - with io.open(unrelated_io, "w") as handle: - handle.write("⚠") - assert unrelated_io.read_bytes() == "⚠".encode("utf-8") - - unrelated_builtin_fd_path = tmp_path / "unrelated-builtins-fd.txt" - unrelated_builtin_fd = os.open(unrelated_builtin_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) - try: - with builtins.open(unrelated_builtin_fd, "w") as handle: - handle.write("→") - finally: - try: - os.close(unrelated_builtin_fd) - except OSError: - pass - assert unrelated_builtin_fd_path.read_bytes() == "→".encode("utf-8") - - unrelated_io_fd_path = tmp_path / "unrelated-io-fd.txt" - unrelated_io_fd = os.open(unrelated_io_fd_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) - try: - with os.fdopen(unrelated_io_fd, "w") as handle: - handle.write("⚠") - finally: - try: - os.close(unrelated_io_fd) - except OSError: - pass - assert unrelated_io_fd_path.read_bytes() == "⚠".encode("utf-8") - assert not target_fds + _run_non_utf8_report_child("pdf", output, v1_pdf, v2_pdf) def test_stdout_when_no_output(self, capsys): main([str(V1), str(V2)])