From 489675a2cdb87e0990d717e4127408855052f014 Mon Sep 17 00:00:00 2001 From: Rio Yu <52408936+rioyu123@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:59:44 +0800 Subject: [PATCH 1/2] fix(reporting): preserve colliding report files --- rampart/reporting/json_file.py | 36 ++++++++++++++-- tests/unit/reporting/test_json_file.py | 59 ++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/rampart/reporting/json_file.py b/rampart/reporting/json_file.py index 5bf576ec..31841ad8 100644 --- a/rampart/reporting/json_file.py +++ b/rampart/reporting/json_file.py @@ -47,12 +47,15 @@ class JsonFileReportSink: Each run produces a timestamped file: ``/run_report_2026-03-19T21-30-00.json`` + If that name already exists, the sink appends ``_1``, ``_2``, and so on. Args: output_dir (Path): Directory to write report files into. Created automatically if it does not exist. """ + _MAX_FILENAME_ATTEMPTS: int = 1000 + def __init__(self, *, output_dir: Path) -> None: """Initialize with an output directory for report files.""" self._output_dir = output_dir @@ -62,14 +65,41 @@ async def emit_async(self, *, report: TestRunReport) -> None: Args: report (TestRunReport): The aggregated test run results. + + Raises: + FileExistsError: If no collision-free filename can be reserved. """ self._output_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S") - filepath = self._output_dir / f"run_report_{timestamp}.json" - data = self._serialize_report(report) - filepath.write_text(json.dumps(data, indent=2, default=str)) + self._write_report_file( + stem=f"run_report_{timestamp}", + content=json.dumps(data, indent=2, default=str), + ) + + def _write_report_file(self, *, stem: str, content: str) -> None: + """Write a report without replacing an existing file. + + Args: + stem (str): Filename without a collision suffix or extension. + content (str): Serialized report content. + + Raises: + FileExistsError: If every collision suffix is already in use. + """ + for suffix in range(self._MAX_FILENAME_ATTEMPTS): + suffix_text = "" if suffix == 0 else f"_{suffix}" + filepath = self._output_dir / f"{stem}{suffix_text}.json" + try: + report_file = filepath.open("x", encoding="utf-8") + except FileExistsError: + continue + with report_file: + report_file.write(content) + return + msg = f"Unable to reserve a report filename for {stem!r}" + raise FileExistsError(msg) def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: """Convert a TestRunReport to a JSON-serializable dict. diff --git a/tests/unit/reporting/test_json_file.py b/tests/unit/reporting/test_json_file.py index ded7f08c..b676e90e 100644 --- a/tests/unit/reporting/test_json_file.py +++ b/tests/unit/reporting/test_json_file.py @@ -6,8 +6,10 @@ from __future__ import annotations import json +from datetime import UTC, datetime from pathlib import Path from typing import Any +from unittest.mock import patch import pytest @@ -366,6 +368,63 @@ async def test_emitted_file_contains_metadata_async(self, tmp_path: Path) -> Non "page_url": "https://example.com/chat", } + async def test_same_timestamp_preserves_every_report_async( + self, + tmp_path: Path, + ) -> None: + sink = JsonFileReportSink(output_dir=tmp_path) + fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + + with patch("rampart.reporting.json_file.datetime") as clock: + clock.now.return_value = fixed + for run in range(3): + await sink.emit_async(report=TestRunReport(metadata={"run": run})) + + files = {path.name: path for path in tmp_path.glob("run_report_*.json")} + assert set(files) == { + "run_report_2026-08-27T12-00-00.json", + "run_report_2026-08-27T12-00-00_1.json", + "run_report_2026-08-27T12-00-00_2.json", + } + for run, suffix in enumerate(("", "_1", "_2")): + path = files[f"run_report_2026-08-27T12-00-00{suffix}.json"] + assert json.loads(path.read_text(encoding="utf-8"))["metadata"] == { + "run": run, + } + + async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> None: + original = tmp_path / "run_report_2026-08-27T12-00-00.json" + original.write_text("keep me", encoding="utf-8") + sink = JsonFileReportSink(output_dir=tmp_path) + fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + + with patch("rampart.reporting.json_file.datetime") as clock: + clock.now.return_value = fixed + await sink.emit_async(report=TestRunReport(metadata={"run": "new"})) + + assert original.read_text(encoding="utf-8") == "keep me" + collision = tmp_path / "run_report_2026-08-27T12-00-00_1.json" + assert json.loads(collision.read_text(encoding="utf-8"))["metadata"] == { + "run": "new", + } + + async def test_raises_after_all_suffixes_are_taken_async( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stem = "run_report_2026-08-27T12-00-00" + (tmp_path / f"{stem}.json").write_text("first", encoding="utf-8") + (tmp_path / f"{stem}_1.json").write_text("second", encoding="utf-8") + monkeypatch.setattr(JsonFileReportSink, "_MAX_FILENAME_ATTEMPTS", 2) + sink = JsonFileReportSink(output_dir=tmp_path) + fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + + with patch("rampart.reporting.json_file.datetime") as clock: + clock.now.return_value = fixed + with pytest.raises(FileExistsError, match="Unable to reserve"): + await sink.emit_async(report=TestRunReport()) + class TestReportMetadata: """Run-level TestRunReport.metadata is projected into the JSON output.""" From 6bc2bc23429b921bbd9036ae48dde1b98c1ca9cb Mon Sep 17 00:00:00 2001 From: Rio Yu <52408936+rioyu123@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:01:04 +0800 Subject: [PATCH 2/2] fix(reporting): use millisecond timestamps and UUID filenames Signed-off-by: Rio Yu <52408936+rioyu123@users.noreply.github.com> --- docs/usage/results-and-reporting.md | 4 +- rampart/reporting/json_file.py | 45 +++++------------- tests/unit/reporting/test_json_file.py | 65 +++++++++++++++++--------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/docs/usage/results-and-reporting.md b/docs/usage/results-and-reporting.md index 7750afe0..67d4ca61 100644 --- a/docs/usage/results-and-reporting.md +++ b/docs/usage/results-and-reporting.md @@ -87,7 +87,9 @@ from rampart.reporting import JsonFileReportSink sink = JsonFileReportSink(output_dir=Path(".report")) ``` -Output: `.report/run_report_2026-04-25T14-30-00.json` +Output: `.report/run_report_2026-04-25T14-30-00-123_a3f18c92654d4b75ad15687d383d951b.json` + +The filename contains a UTC timestamp (millisecond precision) and a random UUID. Reports created in the same millisecond receive different filenames. An exact filename collision raises `FileExistsError` instead of overwriting an existing report. Reports written within the same millisecond have no defined filename order relative to each other. ### Custom Sinks diff --git a/rampart/reporting/json_file.py b/rampart/reporting/json_file.py index 31841ad8..4328e1bc 100644 --- a/rampart/reporting/json_file.py +++ b/rampart/reporting/json_file.py @@ -31,6 +31,7 @@ def rampart_sinks(): import json from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from uuid import uuid4 from rampart.common.text import safe_float, safe_str, safe_str_list @@ -45,17 +46,15 @@ def rampart_sinks(): class JsonFileReportSink: """Writes the test run report to a JSON file. - Each run produces a timestamped file: - ``/run_report_2026-03-19T21-30-00.json`` - If that name already exists, the sink appends ``_1``, ``_2``, and so on. + Each run produces a file named ``run_report__.json``. + The UTC timestamp includes milliseconds; the UUID distinguishes runs + created in the same millisecond. Existing files are never overwritten. Args: output_dir (Path): Directory to write report files into. Created automatically if it does not exist. """ - _MAX_FILENAME_ATTEMPTS: int = 1000 - def __init__(self, *, output_dir: Path) -> None: """Initialize with an output directory for report files.""" self._output_dir = output_dir @@ -67,39 +66,17 @@ async def emit_async(self, *, report: TestRunReport) -> None: report (TestRunReport): The aggregated test run results. Raises: - FileExistsError: If no collision-free filename can be reserved. + FileExistsError: If the generated filename already exists, or + ``output_dir`` exists and is not a directory. """ self._output_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S") + timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] + filepath = self._output_dir / f"run_report_{timestamp}_{uuid4().hex}.json" data = self._serialize_report(report) - self._write_report_file( - stem=f"run_report_{timestamp}", - content=json.dumps(data, indent=2, default=str), - ) - - def _write_report_file(self, *, stem: str, content: str) -> None: - """Write a report without replacing an existing file. - - Args: - stem (str): Filename without a collision suffix or extension. - content (str): Serialized report content. - - Raises: - FileExistsError: If every collision suffix is already in use. - """ - for suffix in range(self._MAX_FILENAME_ATTEMPTS): - suffix_text = "" if suffix == 0 else f"_{suffix}" - filepath = self._output_dir / f"{stem}{suffix_text}.json" - try: - report_file = filepath.open("x", encoding="utf-8") - except FileExistsError: - continue - with report_file: - report_file.write(content) - return - msg = f"Unable to reserve a report filename for {stem!r}" - raise FileExistsError(msg) + content = json.dumps(data, indent=2, default=str) + with filepath.open("x", encoding="utf-8") as report_file: + report_file.write(content) def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: """Convert a TestRunReport to a JSON-serializable dict. diff --git a/tests/unit/reporting/test_json_file.py b/tests/unit/reporting/test_json_file.py index b676e90e..07093b4a 100644 --- a/tests/unit/reporting/test_json_file.py +++ b/tests/unit/reporting/test_json_file.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any from unittest.mock import patch +from uuid import UUID import pytest @@ -373,24 +374,25 @@ async def test_same_timestamp_preserves_every_report_async( tmp_path: Path, ) -> None: sink = JsonFileReportSink(output_dir=tmp_path) - fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + fixed = datetime(2026, 8, 27, 12, 0, 0, 123456, tzinfo=UTC) with patch("rampart.reporting.json_file.datetime") as clock: clock.now.return_value = fixed for run in range(3): await sink.emit_async(report=TestRunReport(metadata={"run": run})) + clock.now.assert_called_with(UTC) - files = {path.name: path for path in tmp_path.glob("run_report_*.json")} - assert set(files) == { - "run_report_2026-08-27T12-00-00.json", - "run_report_2026-08-27T12-00-00_1.json", - "run_report_2026-08-27T12-00-00_2.json", - } - for run, suffix in enumerate(("", "_1", "_2")): - path = files[f"run_report_2026-08-27T12-00-00{suffix}.json"] - assert json.loads(path.read_text(encoding="utf-8"))["metadata"] == { - "run": run, - } + files = list(tmp_path.glob("run_report_*.json")) + assert len(files) == 3 + assert { + json.loads(path.read_text(encoding="utf-8"))["metadata"]["run"] + for path in files + } == {0, 1, 2} + for path in files: + assert path.name.startswith("run_report_2026-08-27T12-00-00-123_") + identifier = path.stem.rsplit("_", 1)[1] + assert len(identifier) == 32 + assert UUID(hex=identifier).version == 4 async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> None: original = tmp_path / "run_report_2026-08-27T12-00-00.json" @@ -403,28 +405,47 @@ async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> No await sink.emit_async(report=TestRunReport(metadata={"run": "new"})) assert original.read_text(encoding="utf-8") == "keep me" - collision = tmp_path / "run_report_2026-08-27T12-00-00_1.json" - assert json.loads(collision.read_text(encoding="utf-8"))["metadata"] == { + new_files = list(tmp_path.glob("run_report_2026-08-27T12-00-00-000_*.json")) + assert len(new_files) == 1 + assert json.loads(new_files[0].read_text(encoding="utf-8"))["metadata"] == { "run": "new", } - async def test_raises_after_all_suffixes_are_taken_async( + async def test_uuid_collision_does_not_overwrite_existing_report_async( self, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - stem = "run_report_2026-08-27T12-00-00" - (tmp_path / f"{stem}.json").write_text("first", encoding="utf-8") - (tmp_path / f"{stem}_1.json").write_text("second", encoding="utf-8") - monkeypatch.setattr(JsonFileReportSink, "_MAX_FILENAME_ATTEMPTS", 2) + identifier = UUID("a3f18c92-654d-4b75-ad15-687d383d951b") + original = ( + tmp_path / f"run_report_2026-08-27T12-00-00-000_{identifier.hex}.json" + ) + original.write_text("keep me", encoding="utf-8") sink = JsonFileReportSink(output_dir=tmp_path) fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) - with patch("rampart.reporting.json_file.datetime") as clock: + with ( + patch("rampart.reporting.json_file.datetime") as clock, + patch("rampart.reporting.json_file.uuid4", return_value=identifier), + ): clock.now.return_value = fixed - with pytest.raises(FileExistsError, match="Unable to reserve"): + with pytest.raises(FileExistsError, match=identifier.hex): await sink.emit_async(report=TestRunReport()) + assert original.read_text(encoding="utf-8") == "keep me" + assert list(tmp_path.glob("run_report_*.json")) == [original] + + async def test_serialization_failure_does_not_create_a_file_async( + self, + tmp_path: Path, + ) -> None: + sink = JsonFileReportSink(output_dir=tmp_path) + report = TestRunReport(metadata={"bad": {("tuple", "key"): "value"}}) + + with pytest.raises(TypeError, match="keys must be"): + await sink.emit_async(report=report) + + assert list(tmp_path.glob("run_report_*.json")) == [] + class TestReportMetadata: """Run-level TestRunReport.metadata is projected into the JSON output."""