diff --git a/src/cloudai/reporter.py b/src/cloudai/reporter.py index d708264d6..2fbdf711e 100644 --- a/src/cloudai/reporter.py +++ b/src/cloudai/reporter.py @@ -43,6 +43,8 @@ class ReportItem: description: str logs_path: Optional[str] = None nodes: Optional[str] = None + is_successful: Optional[bool] = None + error_message: str = "" @classmethod def from_test_runs(cls, test_runs: list[TestRun], results_root: Path) -> list["ReportItem"]: @@ -53,6 +55,9 @@ def from_test_runs(cls, test_runs: list[TestRun], results_root: Path) -> list["R ri.logs_path = f"./{tr.output_path.relative_to(results_root)}" if metadata := load_system_metadata(tr.output_path, results_root): ri.nodes = metadata.slurm.node_list + status = tr.test.was_run_successful(tr) + ri.is_successful = status.is_successful + ri.error_message = status.error_message report_items.append(ri) return report_items diff --git a/src/cloudai/util/general-report.jinja2 b/src/cloudai/util/general-report.jinja2 index 7f3f88cc0..4f98ca3fc 100644 --- a/src/cloudai/util/general-report.jinja2 +++ b/src/cloudai/util/general-report.jinja2 @@ -1,10 +1,18 @@ {% extends "base-report.jinja2" %} +{% block extra_head %} + +{% endblock %} + {% block content %} + @@ -12,6 +20,13 @@ + {% if item.is_successful is none %} + + {% elif item.is_successful %} + + {% else %} + + {% endif %} {% if item.logs_path %} {% else %} diff --git a/tests/test_reporter.py b/tests/test_reporter.py index 3e827e57c..0b7364128 100644 --- a/tests/test_reporter.py +++ b/tests/test_reporter.py @@ -336,6 +336,80 @@ def test_metadata_for_single_sbatch(self, slurm_system: SlurmSystem, slurm_metad [report_item] = ReportItem.from_test_runs([tr], slurm_system.output_path) assert report_item.nodes == slurm_metadata.slurm.node_list + def test_records_passing_status(self, slurm_system: SlurmSystem, monkeypatch: pytest.MonkeyPatch) -> None: + from cloudai.core import JobStatusResult + + run_dir = slurm_system.output_path / "run_dir" + run_dir.mkdir(parents=True, exist_ok=True) + tr = TestRun( + name="run_dir", + test=NCCLTestDefinition( + name="nccl", + description="NCCL test", + test_template_name="NcclTest", + cmd_args=NCCLCmdArgs(docker_image_url="fake://url/nccl"), + ), + num_nodes=1, + nodes=["node1"], + output_path=run_dir, + ) + monkeypatch.setattr(type(tr.test), "was_run_successful", lambda self, tr: JobStatusResult(True, "")) + + [report_item] = ReportItem.from_test_runs([tr], slurm_system.output_path) + assert report_item.is_successful is True + assert report_item.error_message == "" + + def test_records_failing_status_and_message( + self, slurm_system: SlurmSystem, monkeypatch: pytest.MonkeyPatch + ) -> None: + from cloudai.core import JobStatusResult + + run_dir = slurm_system.output_path / "run_dir" + run_dir.mkdir(parents=True, exist_ok=True) + tr = TestRun( + name="run_dir", + test=NCCLTestDefinition( + name="nccl", + description="NCCL test", + test_template_name="NcclTest", + cmd_args=NCCLCmdArgs(docker_image_url="fake://url/nccl"), + ), + num_nodes=1, + nodes=["node1"], + output_path=run_dir, + ) + monkeypatch.setattr( + type(tr.test), "was_run_successful", lambda self, tr: JobStatusResult(False, "command failed") + ) + + [report_item] = ReportItem.from_test_runs([tr], slurm_system.output_path) + assert report_item.is_successful is False + assert report_item.error_message == "command failed" + + +def test_scenario_report_shows_pass_fail_status( + slurm_system: SlurmSystem, benchmark_tr: TestRun, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression test for the bug where the saved HTML report never recorded pass/fail, + only the terminal summary did.""" + from cloudai.core import JobStatusResult + + monkeypatch.setattr( + type(benchmark_tr.test), "was_run_successful", lambda self, tr: JobStatusResult(False, "command failed") + ) + + reporter = StatusReporter( + slurm_system, + TestScenario(name="test-scenario", test_runs=[benchmark_tr]), + slurm_system.output_path, + ReportConfig(), + ) + reporter.generate() + + report_html = (slurm_system.output_path / "test-scenario.html").read_text() + assert "FAILED" in report_html + assert "command failed" in report_html + def test_report_order() -> None: reports = Registry().ordered_scenario_reports()
Test DescriptionStatus Results Nodes
{{ item.name }} {{ item.description }}UnknownPASSEDFAILED{% if item.error_message %}
{{ item.error_message }}{% endif %}
logs