[Reporting] JUnit

doc/reporting.rst

@@ -58,6 +58,12 @@ Enabling or disabling a report needs to be done in the system configuration:
    [reports]
    per_test = { enable = false }
    status = { enable = true }
+   junit = { enable = true }
+
+The ``junit`` scenario reporter writes ``junit.xml`` in the scenario results directory. It emits one test case for
+every regular test iteration and every DSE step, including pass/fail status, failure details, scheduler duration when
+available, and the contents of ``stdout.txt`` and ``stderr.txt``. The artifact can be consumed directly by Jenkins,
+GitLab, GitHub Actions, and other CI systems that support JUnit XML.
 
 Speed-of-Light comparisons
 --------------------------

src/cloudai/core.py

@@ -69,7 +69,7 @@
 from .configurator.gymnasium_adapter import GymnasiumAdapter
 from .models.workload import CmdArgs, NsysConfiguration, PredictorConfig, TestDefinition
 from .parser import Parser
-from .reporter import PerTestReporter, StatusReporter, TarballReporter
+from .reporter import JUnitReporter, PerTestReporter, StatusReporter, TarballReporter
 from .test_parser import TestParser
 from .test_scenario_parser import TestScenarioParser
 
@@ -97,6 +97,7 @@
     "HFModel",
     "InstallStatusResult",
     "Installable",
+    "JUnitReporter",
     "JobIdRetrievalError",
     "JobStatusResult",
     "JsonGenStrategy",

src/cloudai/registration.py

@@ -39,7 +39,7 @@ def register_all():
     from cloudai.core import Registry
     from cloudai.models.scenario import ReportConfig
     from cloudai.report_generator.training import TrainingReporter
-    from cloudai.reporter import DSEReporter, PerTestReporter, StatusReporter, TarballReporter
+    from cloudai.reporter import DSEReporter, JUnitReporter, PerTestReporter, StatusReporter, TarballReporter
 
     # Import systems
     from cloudai.systems.kubernetes import KubernetesInstaller, KubernetesRunner, KubernetesSystem
@@ -332,6 +332,7 @@ def register_all():
         ReportConfig(enable=True),
     )
     Registry().add_scenario_report("status", StatusReporter, ReportConfig(enable=True))
+    Registry().add_scenario_report("junit", JUnitReporter, ReportConfig(enable=True))
     Registry().add_scenario_report("dse", DSEReporter, ReportConfig(enable=True))
     Registry().add_scenario_report("tarball", TarballReporter, ReportConfig(enable=True))
     Registry().add_scenario_report(

src/cloudai/reporter.py

@@ -17,6 +17,7 @@
 import contextlib
 import logging
 import tarfile
+import xml.etree.ElementTree as ET
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional
@@ -134,6 +135,90 @@ def print_summary(self) -> None:
         logging.info(capture.get())
 
 
+class JUnitReporter(Reporter):
+    """Generate a JUnit XML report for all scenario test executions."""
+
+    REPORT_FILE_NAME = "junit.xml"
+
+    def generate(self) -> None:
+        self.load_test_runs()
+
+        results = [(tr, tr.test.was_run_successful(tr), self._duration(tr.output_path)) for tr in self.trs]
+        failures = sum(not status.is_successful for _, status, _ in results)
+        durations = [duration for _, _, duration in results if duration is not None]
+
+        suite_attributes = {
+            "name": self.test_scenario.name,
+            "tests": str(len(results)),
+            "failures": str(failures),
+            "errors": "0",
+            "skipped": "0",
+        }
+        if durations:
+            suite_attributes["time"] = self._format_duration(sum(durations))
+
+        root = ET.Element("testsuites", suite_attributes)
+        suite = ET.SubElement(root, "testsuite", suite_attributes)
+        for tr, status, duration in results:
+            attributes = {"name": case_name(tr), "classname": self.test_scenario.name}
+            if duration is not None:
+                attributes["time"] = self._format_duration(duration)
+
+            testcase = ET.SubElement(suite, "testcase", attributes)
+            if not status.is_successful:
+                message = status.error_message or "Test run failed"
+                failure = ET.SubElement(testcase, "failure", {"message": self._xml_text(message)})
+                failure.text = self._xml_text(message)
+
+            self._add_log(testcase, "system-out", tr.output_path / "stdout.txt")
+            self._add_log(testcase, "system-err", tr.output_path / "stderr.txt")
+
+        ET.indent(root)
+        report_path = self.results_root / self.REPORT_FILE_NAME
+        ET.ElementTree(root).write(report_path, encoding="utf-8", xml_declaration=True)
+        logging.info("Generated JUnit report at %s", report_path)
+
+    @staticmethod
+    def _duration(output_path: Path) -> float | None:
+        # Duration is currently available only for Slurm workloads, which persist slurm-job.toml.
+        metadata_path = output_path / "slurm-job.toml"
+        if not metadata_path.is_file():
+            return None
+        try:
+            duration = toml.load(metadata_path).get("elapsed_time_sec")
+            return float(duration) if duration is not None else None
+        except (OSError, TypeError, ValueError, toml.TomlDecodeError) as exc:
+            logging.debug("Could not read execution duration from %s: %s", metadata_path, exc)
+            return None
+
+    @staticmethod
+    def _format_duration(duration: float) -> str:
+        return f"{duration:g}"
+
+    @classmethod
+    def _add_log(cls, testcase: ET.Element, tag: str, path: Path) -> None:
+        if not path.is_file():
+            return
+        try:
+            content = path.read_text(errors="replace")
+        except OSError as exc:
+            logging.debug("Could not read test log %s: %s", path, exc)
+            return
+        ET.SubElement(testcase, tag).text = cls._xml_text(content)
+
+    @staticmethod
+    def _xml_text(value: str) -> str:
+        """Remove control characters that XML 1.0 cannot represent."""
+        return "".join(
+            char
+            for char in value
+            if char in "\t\n\r"
+            or "\u0020" <= char <= "\ud7ff"
+            or "\ue000" <= char <= "\ufffd"
+            or "\U00010000" <= char <= "\U0010ffff"
+        )
+
+
 class DSEReporter(Reporter):
     """
     Generate DSE-specific scenario artifacts.

tests/test_init.py

@@ -17,7 +17,7 @@
 
 from cloudai.core import Registry
 from cloudai.report_generator.training import TrainingReporter
-from cloudai.reporter import DSEReporter, PerTestReporter, StatusReporter, TarballReporter
+from cloudai.reporter import DSEReporter, JUnitReporter, PerTestReporter, StatusReporter, TarballReporter
 from cloudai.systems.kubernetes import KubernetesInstaller, KubernetesSystem
 from cloudai.systems.lsf import LSFInstaller, LSFSystem
 from cloudai.systems.runai import RunAISystem
@@ -284,6 +284,7 @@ def test_scenario_reports():
         "training",
         "moe_benchmark_throughput",
         "status",
+        "junit",
         "dse",
         "tarball",
         "nixl_bench_summary",
@@ -299,6 +300,7 @@ def test_scenario_reports():
         TrainingReporter,
         MoEBenchmarkThroughputReporter,
         StatusReporter,
+        JUnitReporter,
         DSEReporter,
         TarballReporter,
         NIXLBenchComparisonReport,
@@ -318,6 +320,7 @@ def test_report_configs():
         "training",
         "moe_benchmark_throughput",
         "status",
+        "junit",
         "dse",
         "tarball",
         "nixl_bench_summary",

tests/test_reporter.py

@@ -17,6 +17,7 @@
 import copy
 import csv
 import tarfile
+import xml.etree.ElementTree as ET
 from dataclasses import asdict
 from pathlib import Path
 from typing import Any
@@ -29,7 +30,7 @@
 from cloudai.core import CommandGenStrategy, Registry, Reporter, System
 from cloudai.models.scenario import ReportConfig, TestRunDetails
 from cloudai.report_generator.dse_report import build_dse_summaries
-from cloudai.reporter import DSEReporter, PerTestReporter, ReportItem, StatusReporter, TarballReporter
+from cloudai.reporter import DSEReporter, JUnitReporter, PerTestReporter, ReportItem, StatusReporter, TarballReporter
 from cloudai.systems.slurm.slurm_metadata import (
     MetadataCUDA,
     MetadataMPI,
@@ -344,6 +345,60 @@ def test_report_order() -> None:
     assert reports[-1][0] == "tarball"
 
 
+def test_junit_reporter_generates_testcases_with_status_logs_and_duration(
+    slurm_system: SlurmSystem, benchmark_tr: TestRun, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    run_dirs = [slurm_system.output_path / benchmark_tr.name / str(i) for i in range(benchmark_tr.iterations)]
+    for i, run_dir in enumerate(run_dirs):
+        (run_dir / "stdout.txt").write_text(f"stdout {i}\n")
+        (run_dir / "stderr.txt").write_text(f"stderr {i}\n")
+        _write_slurm_job(run_dir, i + 1)
+
+    statuses = {
+        run_dirs[0]: (True, ""),
+        run_dirs[1]: (False, "benchmark failed\x01"),
+        run_dirs[2]: (True, ""),
+    }
+
+    def was_run_successful(tr: TestRun):
+        successful, message = statuses[tr.output_path]
+        from cloudai.core import JobStatusResult
+
+        return JobStatusResult(successful, message)
+
+    monkeypatch.setattr(type(benchmark_tr.test), "was_run_successful", lambda self, tr: was_run_successful(tr))
+    reporter = JUnitReporter(
+        slurm_system,
+        TestScenario(name="test-scenario", test_runs=[benchmark_tr]),
+        slurm_system.output_path,
+        ReportConfig(),
+    )
+    reporter.generate()
+
+    root = ET.parse(slurm_system.output_path / "junit.xml").getroot()
+    suite = root.find("testsuite")
+    assert root.attrib == {
+        "name": "test-scenario",
+        "tests": "3",
+        "failures": "1",
+        "errors": "0",
+        "skipped": "0",
+        "time": "6",
+    }
+    assert suite is not None
+    cases = suite.findall("testcase")
+    assert [case.attrib for case in cases] == [
+        {"name": "benchmark", "classname": "test-scenario", "time": "1"},
+        {"name": "benchmark iter=1", "classname": "test-scenario", "time": "2"},
+        {"name": "benchmark iter=2", "classname": "test-scenario", "time": "3"},
+    ]
+    assert cases[0].findtext("system-out") == "stdout 0\n"
+    failure = cases[1].find("failure")
+    assert failure is not None
+    assert failure.attrib["message"] == "benchmark failed"
+    assert cases[1].findtext("system-err") == "stderr 1\n"
+
+
 def _write_slurm_job(step_dir: Path, elapsed_time_sec: int) -> None:
     metadata = SlurmJobMetadata(
         job_id=12345,