From 779b96b5cddb9e7a2a81382458b496832a197575 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Thu, 13 Aug 2026 13:39:15 +0200 Subject: [PATCH] junit xml generation --- doc/reporting.rst | 6 +++ src/cloudai/core.py | 3 +- src/cloudai/registration.py | 3 +- src/cloudai/reporter.py | 85 +++++++++++++++++++++++++++++++++++++ tests/test_init.py | 5 ++- tests/test_reporter.py | 57 ++++++++++++++++++++++++- 6 files changed, 155 insertions(+), 4 deletions(-) diff --git a/doc/reporting.rst b/doc/reporting.rst index 4eed11ed4..7e29ee8a6 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -58,6 +58,12 @@ Enabling or disabling a report needs to be done in the system configuration: [reports] per_test = { enable = false } status = { enable = true } + junit = { enable = true } + +The ``junit`` scenario reporter writes ``junit.xml`` in the scenario results directory. It emits one test case for +every regular test iteration and every DSE step, including pass/fail status, failure details, scheduler duration when +available, and the contents of ``stdout.txt`` and ``stderr.txt``. The artifact can be consumed directly by Jenkins, +GitLab, GitHub Actions, and other CI systems that support JUnit XML. Speed-of-Light comparisons -------------------------- diff --git a/src/cloudai/core.py b/src/cloudai/core.py index 06601f7f4..7dffb5493 100644 --- a/src/cloudai/core.py +++ b/src/cloudai/core.py @@ -69,7 +69,7 @@ from .configurator.gymnasium_adapter import GymnasiumAdapter from .models.workload import CmdArgs, NsysConfiguration, PredictorConfig, TestDefinition from .parser import Parser -from .reporter import PerTestReporter, StatusReporter, TarballReporter +from .reporter import JUnitReporter, PerTestReporter, StatusReporter, TarballReporter from .test_parser import TestParser from .test_scenario_parser import TestScenarioParser @@ -97,6 +97,7 @@ "HFModel", "InstallStatusResult", "Installable", + "JUnitReporter", "JobIdRetrievalError", "JobStatusResult", "JsonGenStrategy", diff --git a/src/cloudai/registration.py b/src/cloudai/registration.py index 9ead24bbf..5bc9a46fc 100644 --- a/src/cloudai/registration.py +++ b/src/cloudai/registration.py @@ -39,7 +39,7 @@ def register_all(): from cloudai.core import Registry from cloudai.models.scenario import ReportConfig from cloudai.report_generator.training import TrainingReporter - from cloudai.reporter import DSEReporter, PerTestReporter, StatusReporter, TarballReporter + from cloudai.reporter import DSEReporter, JUnitReporter, PerTestReporter, StatusReporter, TarballReporter # Import systems from cloudai.systems.kubernetes import KubernetesInstaller, KubernetesRunner, KubernetesSystem @@ -332,6 +332,7 @@ def register_all(): ReportConfig(enable=True), ) Registry().add_scenario_report("status", StatusReporter, ReportConfig(enable=True)) + Registry().add_scenario_report("junit", JUnitReporter, ReportConfig(enable=True)) Registry().add_scenario_report("dse", DSEReporter, ReportConfig(enable=True)) Registry().add_scenario_report("tarball", TarballReporter, ReportConfig(enable=True)) Registry().add_scenario_report( diff --git a/src/cloudai/reporter.py b/src/cloudai/reporter.py index a97ef2fc3..cba3ebf37 100644 --- a/src/cloudai/reporter.py +++ b/src/cloudai/reporter.py @@ -17,6 +17,7 @@ import contextlib import logging import tarfile +import xml.etree.ElementTree as ET from dataclasses import dataclass from pathlib import Path from typing import Optional @@ -134,6 +135,90 @@ def print_summary(self) -> None: logging.info(capture.get()) +class JUnitReporter(Reporter): + """Generate a JUnit XML report for all scenario test executions.""" + + REPORT_FILE_NAME = "junit.xml" + + def generate(self) -> None: + self.load_test_runs() + + results = [(tr, tr.test.was_run_successful(tr), self._duration(tr.output_path)) for tr in self.trs] + failures = sum(not status.is_successful for _, status, _ in results) + durations = [duration for _, _, duration in results if duration is not None] + + suite_attributes = { + "name": self.test_scenario.name, + "tests": str(len(results)), + "failures": str(failures), + "errors": "0", + "skipped": "0", + } + if durations: + suite_attributes["time"] = self._format_duration(sum(durations)) + + root = ET.Element("testsuites", suite_attributes) + suite = ET.SubElement(root, "testsuite", suite_attributes) + for tr, status, duration in results: + attributes = {"name": case_name(tr), "classname": self.test_scenario.name} + if duration is not None: + attributes["time"] = self._format_duration(duration) + + testcase = ET.SubElement(suite, "testcase", attributes) + if not status.is_successful: + message = status.error_message or "Test run failed" + failure = ET.SubElement(testcase, "failure", {"message": self._xml_text(message)}) + failure.text = self._xml_text(message) + + self._add_log(testcase, "system-out", tr.output_path / "stdout.txt") + self._add_log(testcase, "system-err", tr.output_path / "stderr.txt") + + ET.indent(root) + report_path = self.results_root / self.REPORT_FILE_NAME + ET.ElementTree(root).write(report_path, encoding="utf-8", xml_declaration=True) + logging.info("Generated JUnit report at %s", report_path) + + @staticmethod + def _duration(output_path: Path) -> float | None: + # Duration is currently available only for Slurm workloads, which persist slurm-job.toml. + metadata_path = output_path / "slurm-job.toml" + if not metadata_path.is_file(): + return None + try: + duration = toml.load(metadata_path).get("elapsed_time_sec") + return float(duration) if duration is not None else None + except (OSError, TypeError, ValueError, toml.TomlDecodeError) as exc: + logging.debug("Could not read execution duration from %s: %s", metadata_path, exc) + return None + + @staticmethod + def _format_duration(duration: float) -> str: + return f"{duration:g}" + + @classmethod + def _add_log(cls, testcase: ET.Element, tag: str, path: Path) -> None: + if not path.is_file(): + return + try: + content = path.read_text(errors="replace") + except OSError as exc: + logging.debug("Could not read test log %s: %s", path, exc) + return + ET.SubElement(testcase, tag).text = cls._xml_text(content) + + @staticmethod + def _xml_text(value: str) -> str: + """Remove control characters that XML 1.0 cannot represent.""" + return "".join( + char + for char in value + if char in "\t\n\r" + or "\u0020" <= char <= "\ud7ff" + or "\ue000" <= char <= "\ufffd" + or "\U00010000" <= char <= "\U0010ffff" + ) + + class DSEReporter(Reporter): """ Generate DSE-specific scenario artifacts. diff --git a/tests/test_init.py b/tests/test_init.py index 466fbae66..f215baece 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -17,7 +17,7 @@ from cloudai.core import Registry from cloudai.report_generator.training import TrainingReporter -from cloudai.reporter import DSEReporter, PerTestReporter, StatusReporter, TarballReporter +from cloudai.reporter import DSEReporter, JUnitReporter, PerTestReporter, StatusReporter, TarballReporter from cloudai.systems.kubernetes import KubernetesInstaller, KubernetesSystem from cloudai.systems.lsf import LSFInstaller, LSFSystem from cloudai.systems.runai import RunAISystem @@ -284,6 +284,7 @@ def test_scenario_reports(): "training", "moe_benchmark_throughput", "status", + "junit", "dse", "tarball", "nixl_bench_summary", @@ -299,6 +300,7 @@ def test_scenario_reports(): TrainingReporter, MoEBenchmarkThroughputReporter, StatusReporter, + JUnitReporter, DSEReporter, TarballReporter, NIXLBenchComparisonReport, @@ -318,6 +320,7 @@ def test_report_configs(): "training", "moe_benchmark_throughput", "status", + "junit", "dse", "tarball", "nixl_bench_summary", diff --git a/tests/test_reporter.py b/tests/test_reporter.py index 95acd8ac9..31942ccda 100644 --- a/tests/test_reporter.py +++ b/tests/test_reporter.py @@ -17,6 +17,7 @@ import copy import csv import tarfile +import xml.etree.ElementTree as ET from dataclasses import asdict from pathlib import Path from typing import Any @@ -29,7 +30,7 @@ from cloudai.core import CommandGenStrategy, Registry, Reporter, System from cloudai.models.scenario import ReportConfig, TestRunDetails from cloudai.report_generator.dse_report import build_dse_summaries -from cloudai.reporter import DSEReporter, PerTestReporter, ReportItem, StatusReporter, TarballReporter +from cloudai.reporter import DSEReporter, JUnitReporter, PerTestReporter, ReportItem, StatusReporter, TarballReporter from cloudai.systems.slurm.slurm_metadata import ( MetadataCUDA, MetadataMPI, @@ -344,6 +345,60 @@ def test_report_order() -> None: assert reports[-1][0] == "tarball" +def test_junit_reporter_generates_testcases_with_status_logs_and_duration( + slurm_system: SlurmSystem, benchmark_tr: TestRun, monkeypatch: pytest.MonkeyPatch +) -> None: + run_dirs = [slurm_system.output_path / benchmark_tr.name / str(i) for i in range(benchmark_tr.iterations)] + for i, run_dir in enumerate(run_dirs): + (run_dir / "stdout.txt").write_text(f"stdout {i}\n") + (run_dir / "stderr.txt").write_text(f"stderr {i}\n") + _write_slurm_job(run_dir, i + 1) + + statuses = { + run_dirs[0]: (True, ""), + run_dirs[1]: (False, "benchmark failed\x01"), + run_dirs[2]: (True, ""), + } + + def was_run_successful(tr: TestRun): + successful, message = statuses[tr.output_path] + from cloudai.core import JobStatusResult + + return JobStatusResult(successful, message) + + monkeypatch.setattr(type(benchmark_tr.test), "was_run_successful", lambda self, tr: was_run_successful(tr)) + reporter = JUnitReporter( + slurm_system, + TestScenario(name="test-scenario", test_runs=[benchmark_tr]), + slurm_system.output_path, + ReportConfig(), + ) + reporter.generate() + + root = ET.parse(slurm_system.output_path / "junit.xml").getroot() + suite = root.find("testsuite") + assert root.attrib == { + "name": "test-scenario", + "tests": "3", + "failures": "1", + "errors": "0", + "skipped": "0", + "time": "6", + } + assert suite is not None + cases = suite.findall("testcase") + assert [case.attrib for case in cases] == [ + {"name": "benchmark", "classname": "test-scenario", "time": "1"}, + {"name": "benchmark iter=1", "classname": "test-scenario", "time": "2"}, + {"name": "benchmark iter=2", "classname": "test-scenario", "time": "3"}, + ] + assert cases[0].findtext("system-out") == "stdout 0\n" + failure = cases[1].find("failure") + assert failure is not None + assert failure.attrib["message"] == "benchmark failed" + assert cases[1].findtext("system-err") == "stderr 1\n" + + def _write_slurm_job(step_dir: Path, elapsed_time_sec: int) -> None: metadata = SlurmJobMetadata( job_id=12345,