From 489675a2cdb87e0990d717e4127408855052f014 Mon Sep 17 00:00:00 2001 From: Rio Yu <52408936+rioyu123@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:59:44 +0800 Subject: [PATCH 1/3] fix(reporting): preserve colliding report files --- rampart/reporting/json_file.py | 36 ++++++++++++++-- tests/unit/reporting/test_json_file.py | 59 ++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/rampart/reporting/json_file.py b/rampart/reporting/json_file.py index 5bf576ec..31841ad8 100644 --- a/rampart/reporting/json_file.py +++ b/rampart/reporting/json_file.py @@ -47,12 +47,15 @@ class JsonFileReportSink: Each run produces a timestamped file: ``/run_report_2026-03-19T21-30-00.json`` + If that name already exists, the sink appends ``_1``, ``_2``, and so on. Args: output_dir (Path): Directory to write report files into. Created automatically if it does not exist. """ + _MAX_FILENAME_ATTEMPTS: int = 1000 + def __init__(self, *, output_dir: Path) -> None: """Initialize with an output directory for report files.""" self._output_dir = output_dir @@ -62,14 +65,41 @@ async def emit_async(self, *, report: TestRunReport) -> None: Args: report (TestRunReport): The aggregated test run results. + + Raises: + FileExistsError: If no collision-free filename can be reserved. """ self._output_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S") - filepath = self._output_dir / f"run_report_{timestamp}.json" - data = self._serialize_report(report) - filepath.write_text(json.dumps(data, indent=2, default=str)) + self._write_report_file( + stem=f"run_report_{timestamp}", + content=json.dumps(data, indent=2, default=str), + ) + + def _write_report_file(self, *, stem: str, content: str) -> None: + """Write a report without replacing an existing file. + + Args: + stem (str): Filename without a collision suffix or extension. + content (str): Serialized report content. + + Raises: + FileExistsError: If every collision suffix is already in use. + """ + for suffix in range(self._MAX_FILENAME_ATTEMPTS): + suffix_text = "" if suffix == 0 else f"_{suffix}" + filepath = self._output_dir / f"{stem}{suffix_text}.json" + try: + report_file = filepath.open("x", encoding="utf-8") + except FileExistsError: + continue + with report_file: + report_file.write(content) + return + msg = f"Unable to reserve a report filename for {stem!r}" + raise FileExistsError(msg) def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: """Convert a TestRunReport to a JSON-serializable dict. diff --git a/tests/unit/reporting/test_json_file.py b/tests/unit/reporting/test_json_file.py index ded7f08c..b676e90e 100644 --- a/tests/unit/reporting/test_json_file.py +++ b/tests/unit/reporting/test_json_file.py @@ -6,8 +6,10 @@ from __future__ import annotations import json +from datetime import UTC, datetime from pathlib import Path from typing import Any +from unittest.mock import patch import pytest @@ -366,6 +368,63 @@ async def test_emitted_file_contains_metadata_async(self, tmp_path: Path) -> Non "page_url": "https://example.com/chat", } + async def test_same_timestamp_preserves_every_report_async( + self, + tmp_path: Path, + ) -> None: + sink = JsonFileReportSink(output_dir=tmp_path) + fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + + with patch("rampart.reporting.json_file.datetime") as clock: + clock.now.return_value = fixed + for run in range(3): + await sink.emit_async(report=TestRunReport(metadata={"run": run})) + + files = {path.name: path for path in tmp_path.glob("run_report_*.json")} + assert set(files) == { + "run_report_2026-08-27T12-00-00.json", + "run_report_2026-08-27T12-00-00_1.json", + "run_report_2026-08-27T12-00-00_2.json", + } + for run, suffix in enumerate(("", "_1", "_2")): + path = files[f"run_report_2026-08-27T12-00-00{suffix}.json"] + assert json.loads(path.read_text(encoding="utf-8"))["metadata"] == { + "run": run, + } + + async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> None: + original = tmp_path / "run_report_2026-08-27T12-00-00.json" + original.write_text("keep me", encoding="utf-8") + sink = JsonFileReportSink(output_dir=tmp_path) + fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + + with patch("rampart.reporting.json_file.datetime") as clock: + clock.now.return_value = fixed + await sink.emit_async(report=TestRunReport(metadata={"run": "new"})) + + assert original.read_text(encoding="utf-8") == "keep me" + collision = tmp_path / "run_report_2026-08-27T12-00-00_1.json" + assert json.loads(collision.read_text(encoding="utf-8"))["metadata"] == { + "run": "new", + } + + async def test_raises_after_all_suffixes_are_taken_async( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stem = "run_report_2026-08-27T12-00-00" + (tmp_path / f"{stem}.json").write_text("first", encoding="utf-8") + (tmp_path / f"{stem}_1.json").write_text("second", encoding="utf-8") + monkeypatch.setattr(JsonFileReportSink, "_MAX_FILENAME_ATTEMPTS", 2) + sink = JsonFileReportSink(output_dir=tmp_path) + fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + + with patch("rampart.reporting.json_file.datetime") as clock: + clock.now.return_value = fixed + with pytest.raises(FileExistsError, match="Unable to reserve"): + await sink.emit_async(report=TestRunReport()) + class TestReportMetadata: """Run-level TestRunReport.metadata is projected into the JSON output.""" From 6bc2bc23429b921bbd9036ae48dde1b98c1ca9cb Mon Sep 17 00:00:00 2001 From: Rio Yu <52408936+rioyu123@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:01:04 +0800 Subject: [PATCH 2/3] fix(reporting): use millisecond timestamps and UUID filenames Signed-off-by: Rio Yu <52408936+rioyu123@users.noreply.github.com> --- docs/usage/results-and-reporting.md | 4 +- rampart/reporting/json_file.py | 45 +++++------------- tests/unit/reporting/test_json_file.py | 65 +++++++++++++++++--------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/docs/usage/results-and-reporting.md b/docs/usage/results-and-reporting.md index 7750afe0..67d4ca61 100644 --- a/docs/usage/results-and-reporting.md +++ b/docs/usage/results-and-reporting.md @@ -87,7 +87,9 @@ from rampart.reporting import JsonFileReportSink sink = JsonFileReportSink(output_dir=Path(".report")) ``` -Output: `.report/run_report_2026-04-25T14-30-00.json` +Output: `.report/run_report_2026-04-25T14-30-00-123_a3f18c92654d4b75ad15687d383d951b.json` + +The filename contains a UTC timestamp (millisecond precision) and a random UUID. Reports created in the same millisecond receive different filenames. An exact filename collision raises `FileExistsError` instead of overwriting an existing report. Reports written within the same millisecond have no defined filename order relative to each other. ### Custom Sinks diff --git a/rampart/reporting/json_file.py b/rampart/reporting/json_file.py index 31841ad8..4328e1bc 100644 --- a/rampart/reporting/json_file.py +++ b/rampart/reporting/json_file.py @@ -31,6 +31,7 @@ def rampart_sinks(): import json from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from uuid import uuid4 from rampart.common.text import safe_float, safe_str, safe_str_list @@ -45,17 +46,15 @@ def rampart_sinks(): class JsonFileReportSink: """Writes the test run report to a JSON file. - Each run produces a timestamped file: - ``/run_report_2026-03-19T21-30-00.json`` - If that name already exists, the sink appends ``_1``, ``_2``, and so on. + Each run produces a file named ``run_report__.json``. + The UTC timestamp includes milliseconds; the UUID distinguishes runs + created in the same millisecond. Existing files are never overwritten. Args: output_dir (Path): Directory to write report files into. Created automatically if it does not exist. """ - _MAX_FILENAME_ATTEMPTS: int = 1000 - def __init__(self, *, output_dir: Path) -> None: """Initialize with an output directory for report files.""" self._output_dir = output_dir @@ -67,39 +66,17 @@ async def emit_async(self, *, report: TestRunReport) -> None: report (TestRunReport): The aggregated test run results. Raises: - FileExistsError: If no collision-free filename can be reserved. + FileExistsError: If the generated filename already exists, or + ``output_dir`` exists and is not a directory. """ self._output_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S") + timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] + filepath = self._output_dir / f"run_report_{timestamp}_{uuid4().hex}.json" data = self._serialize_report(report) - self._write_report_file( - stem=f"run_report_{timestamp}", - content=json.dumps(data, indent=2, default=str), - ) - - def _write_report_file(self, *, stem: str, content: str) -> None: - """Write a report without replacing an existing file. - - Args: - stem (str): Filename without a collision suffix or extension. - content (str): Serialized report content. - - Raises: - FileExistsError: If every collision suffix is already in use. - """ - for suffix in range(self._MAX_FILENAME_ATTEMPTS): - suffix_text = "" if suffix == 0 else f"_{suffix}" - filepath = self._output_dir / f"{stem}{suffix_text}.json" - try: - report_file = filepath.open("x", encoding="utf-8") - except FileExistsError: - continue - with report_file: - report_file.write(content) - return - msg = f"Unable to reserve a report filename for {stem!r}" - raise FileExistsError(msg) + content = json.dumps(data, indent=2, default=str) + with filepath.open("x", encoding="utf-8") as report_file: + report_file.write(content) def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: """Convert a TestRunReport to a JSON-serializable dict. diff --git a/tests/unit/reporting/test_json_file.py b/tests/unit/reporting/test_json_file.py index b676e90e..07093b4a 100644 --- a/tests/unit/reporting/test_json_file.py +++ b/tests/unit/reporting/test_json_file.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any from unittest.mock import patch +from uuid import UUID import pytest @@ -373,24 +374,25 @@ async def test_same_timestamp_preserves_every_report_async( tmp_path: Path, ) -> None: sink = JsonFileReportSink(output_dir=tmp_path) - fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) + fixed = datetime(2026, 8, 27, 12, 0, 0, 123456, tzinfo=UTC) with patch("rampart.reporting.json_file.datetime") as clock: clock.now.return_value = fixed for run in range(3): await sink.emit_async(report=TestRunReport(metadata={"run": run})) + clock.now.assert_called_with(UTC) - files = {path.name: path for path in tmp_path.glob("run_report_*.json")} - assert set(files) == { - "run_report_2026-08-27T12-00-00.json", - "run_report_2026-08-27T12-00-00_1.json", - "run_report_2026-08-27T12-00-00_2.json", - } - for run, suffix in enumerate(("", "_1", "_2")): - path = files[f"run_report_2026-08-27T12-00-00{suffix}.json"] - assert json.loads(path.read_text(encoding="utf-8"))["metadata"] == { - "run": run, - } + files = list(tmp_path.glob("run_report_*.json")) + assert len(files) == 3 + assert { + json.loads(path.read_text(encoding="utf-8"))["metadata"]["run"] + for path in files + } == {0, 1, 2} + for path in files: + assert path.name.startswith("run_report_2026-08-27T12-00-00-123_") + identifier = path.stem.rsplit("_", 1)[1] + assert len(identifier) == 32 + assert UUID(hex=identifier).version == 4 async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> None: original = tmp_path / "run_report_2026-08-27T12-00-00.json" @@ -403,28 +405,47 @@ async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> No await sink.emit_async(report=TestRunReport(metadata={"run": "new"})) assert original.read_text(encoding="utf-8") == "keep me" - collision = tmp_path / "run_report_2026-08-27T12-00-00_1.json" - assert json.loads(collision.read_text(encoding="utf-8"))["metadata"] == { + new_files = list(tmp_path.glob("run_report_2026-08-27T12-00-00-000_*.json")) + assert len(new_files) == 1 + assert json.loads(new_files[0].read_text(encoding="utf-8"))["metadata"] == { "run": "new", } - async def test_raises_after_all_suffixes_are_taken_async( + async def test_uuid_collision_does_not_overwrite_existing_report_async( self, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - stem = "run_report_2026-08-27T12-00-00" - (tmp_path / f"{stem}.json").write_text("first", encoding="utf-8") - (tmp_path / f"{stem}_1.json").write_text("second", encoding="utf-8") - monkeypatch.setattr(JsonFileReportSink, "_MAX_FILENAME_ATTEMPTS", 2) + identifier = UUID("a3f18c92-654d-4b75-ad15-687d383d951b") + original = ( + tmp_path / f"run_report_2026-08-27T12-00-00-000_{identifier.hex}.json" + ) + original.write_text("keep me", encoding="utf-8") sink = JsonFileReportSink(output_dir=tmp_path) fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) - with patch("rampart.reporting.json_file.datetime") as clock: + with ( + patch("rampart.reporting.json_file.datetime") as clock, + patch("rampart.reporting.json_file.uuid4", return_value=identifier), + ): clock.now.return_value = fixed - with pytest.raises(FileExistsError, match="Unable to reserve"): + with pytest.raises(FileExistsError, match=identifier.hex): await sink.emit_async(report=TestRunReport()) + assert original.read_text(encoding="utf-8") == "keep me" + assert list(tmp_path.glob("run_report_*.json")) == [original] + + async def test_serialization_failure_does_not_create_a_file_async( + self, + tmp_path: Path, + ) -> None: + sink = JsonFileReportSink(output_dir=tmp_path) + report = TestRunReport(metadata={"bad": {("tuple", "key"): "value"}}) + + with pytest.raises(TypeError, match="keys must be"): + await sink.emit_async(report=report) + + assert list(tmp_path.glob("run_report_*.json")) == [] + class TestReportMetadata: """Run-level TestRunReport.metadata is projected into the JSON output.""" From bf8df80a9236c8b6940a521ad53b227c78ade7b1 Mon Sep 17 00:00:00 2001 From: Rio Yu <52408936+rioyu123@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:36:58 +0800 Subject: [PATCH 3/3] fix(reporting): append UUID only on collision --- docs/usage/results-and-reporting.md | 4 ++-- rampart/reporting/json_file.py | 26 ++++++++++++++++++++------ tests/unit/reporting/test_json_file.py | 22 ++++++++++++++++------ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/usage/results-and-reporting.md b/docs/usage/results-and-reporting.md index 67d4ca61..889dff8a 100644 --- a/docs/usage/results-and-reporting.md +++ b/docs/usage/results-and-reporting.md @@ -87,9 +87,9 @@ from rampart.reporting import JsonFileReportSink sink = JsonFileReportSink(output_dir=Path(".report")) ``` -Output: `.report/run_report_2026-04-25T14-30-00-123_a3f18c92654d4b75ad15687d383d951b.json` +Output: `.report/run_report_2026-04-25T14-30-00-123.json` -The filename contains a UTC timestamp (millisecond precision) and a random UUID. Reports created in the same millisecond receive different filenames. An exact filename collision raises `FileExistsError` instead of overwriting an existing report. Reports written within the same millisecond have no defined filename order relative to each other. +The filename contains a UTC timestamp with millisecond precision. If another report already has the same timestamp, a random UUID is appended to the new filename. Files are created atomically and existing reports are never overwritten. Reports written within the same millisecond have no defined filename order relative to each other. ### Custom Sinks diff --git a/rampart/reporting/json_file.py b/rampart/reporting/json_file.py index 4328e1bc..e734eb9c 100644 --- a/rampart/reporting/json_file.py +++ b/rampart/reporting/json_file.py @@ -46,9 +46,10 @@ def rampart_sinks(): class JsonFileReportSink: """Writes the test run report to a JSON file. - Each run produces a file named ``run_report__.json``. - The UTC timestamp includes milliseconds; the UUID distinguishes runs - created in the same millisecond. Existing files are never overwritten. + Each run normally produces a file named ``run_report_.json``. + The UTC timestamp includes milliseconds. If that filename already exists, + a UUID suffix distinguishes the colliding run. Existing files are never + overwritten. Args: output_dir (Path): Directory to write report files into. @@ -66,16 +67,29 @@ async def emit_async(self, *, report: TestRunReport) -> None: report (TestRunReport): The aggregated test run results. Raises: - FileExistsError: If the generated filename already exists, or + FileExistsError: If the random fallback filename also exists, or ``output_dir`` exists and is not a directory. """ self._output_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] - filepath = self._output_dir / f"run_report_{timestamp}_{uuid4().hex}.json" data = self._serialize_report(report) content = json.dumps(data, indent=2, default=str) - with filepath.open("x", encoding="utf-8") as report_file: + + filepath = self._output_dir / f"run_report_{timestamp}.json" + try: + report_file = filepath.open("x", encoding="utf-8") + except FileExistsError: + filepath = self._output_dir / f"run_report_{timestamp}_{uuid4().hex}.json" + + # Leave the exception handler before opening the fallback so a + # second collision reports only the path that actually collided. + report_file = None + + if report_file is None: + report_file = filepath.open("x", encoding="utf-8") + + with report_file: report_file.write(content) def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: diff --git a/tests/unit/reporting/test_json_file.py b/tests/unit/reporting/test_json_file.py index 07093b4a..e162dc76 100644 --- a/tests/unit/reporting/test_json_file.py +++ b/tests/unit/reporting/test_json_file.py @@ -388,14 +388,21 @@ async def test_same_timestamp_preserves_every_report_async( json.loads(path.read_text(encoding="utf-8"))["metadata"]["run"] for path in files } == {0, 1, 2} - for path in files: + + concise = tmp_path / "run_report_2026-08-27T12-00-00-123.json" + assert json.loads(concise.read_text(encoding="utf-8"))["metadata"] == { + "run": 0, + } + colliding_files = [path for path in files if path != concise] + assert len(colliding_files) == 2 + for path in colliding_files: assert path.name.startswith("run_report_2026-08-27T12-00-00-123_") identifier = path.stem.rsplit("_", 1)[1] assert len(identifier) == 32 assert UUID(hex=identifier).version == 4 async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> None: - original = tmp_path / "run_report_2026-08-27T12-00-00.json" + original = tmp_path / "run_report_2026-08-27T12-00-00-000.json" original.write_text("keep me", encoding="utf-8") sink = JsonFileReportSink(output_dir=tmp_path) fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) @@ -416,10 +423,12 @@ async def test_uuid_collision_does_not_overwrite_existing_report_async( tmp_path: Path, ) -> None: identifier = UUID("a3f18c92-654d-4b75-ad15-687d383d951b") - original = ( + timestamp_file = tmp_path / "run_report_2026-08-27T12-00-00-000.json" + timestamp_file.write_text("keep timestamp", encoding="utf-8") + uuid_file = ( tmp_path / f"run_report_2026-08-27T12-00-00-000_{identifier.hex}.json" ) - original.write_text("keep me", encoding="utf-8") + uuid_file.write_text("keep uuid", encoding="utf-8") sink = JsonFileReportSink(output_dir=tmp_path) fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC) @@ -431,8 +440,9 @@ async def test_uuid_collision_does_not_overwrite_existing_report_async( with pytest.raises(FileExistsError, match=identifier.hex): await sink.emit_async(report=TestRunReport()) - assert original.read_text(encoding="utf-8") == "keep me" - assert list(tmp_path.glob("run_report_*.json")) == [original] + assert timestamp_file.read_text(encoding="utf-8") == "keep timestamp" + assert uuid_file.read_text(encoding="utf-8") == "keep uuid" + assert set(tmp_path.glob("run_report_*.json")) == {timestamp_file, uuid_file} async def test_serialization_failure_does_not_create_a_file_async( self,