From 5362f501af6ac457302ab48b4c0db7d4fa716871 Mon Sep 17 00:00:00 2001 From: Dan Calavrezo <195309321+dcalavrezo-qorix@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:14:39 +0300 Subject: [PATCH 1/2] coverage: emit a markdown job summary from the report generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every consumer that wants a human-readable coverage summary on its workflow run page has been re-parsing the LCOV itself; the report generator now emits it directly. - New py_binary coverage_summary (stdlib only): parses the pipeline's LCOV (which includes the exact-0% baseline records), aggregates line/branch/file totals, and renders markdown — overall table with text progress bars, raw-vs-effective when a justification report exists, a per-directory rollup (first one/two path segments, worst first), and collapsible least-covered / exact-0% file lists, closing with a pointer to the HTML artifact. - generate_coverage_html: new optional --summary-md ; when the flag is absent and GITHUB_STEP_SUMMARY is set (GitHub Actions), the summary is appended there automatically; with neither, behavior is unchanged. Emitted BEFORE the threshold gate decides the exit code, so a failing gate still leaves the summary on the run page. - Edge cases covered by unit tests: missing/empty LCOV (note instead of crash), records with LF but no BRF (branch cells render as an em dash), BRDA fallback counting, zero denominators, non-UTF8 bytes in paths, markdown cell escaping. - Integration workspace: asserts summary content markers, append semantics for GITHUB_STEP_SUMMARY (existing content preserved), and that the summary survives a failing gate. The test script unsets GITHUB_STEP_SUMMARY first so CI runs don't spam the real run page. - Docs: adoption guide (README) and COVERAGE_GUIDE updated. --- .gitignore | 2 + coverage/BUILD | 13 + coverage/COVERAGE_GUIDE.md | 8 + coverage/README.md | 12 + coverage/coverage_summary.py | 333 ++++++++++++++++++ coverage/generate_coverage_html.sh | 39 ++ .../integration_tests/run_integration_test.sh | 39 ++ coverage/tests/BUILD | 6 + coverage/tests/coverage_summary_test.py | 181 ++++++++++ 9 files changed, 633 insertions(+) create mode 100644 coverage/coverage_summary.py create mode 100644 coverage/tests/coverage_summary_test.py diff --git a/.gitignore b/.gitignore index c2df75cc..4af5e20f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ coverage-html/ coverage/integration_tests/coverage_artifacts.zip coverage/integration_tests/lcov.dat coverage/integration_tests/coverage_linux/ +coverage/integration_tests/summary.md +coverage/integration_tests/step_summary.md diff --git a/coverage/BUILD b/coverage/BUILD index 614491dd..4ea1af08 100644 --- a/coverage/BUILD +++ b/coverage/BUILD @@ -56,6 +56,13 @@ py_binary( srcs = ["effective_coverage.py"], ) +# Markdown job summary (GITHUB_STEP_SUMMARY / --summary-md), invoked by +# generate_coverage_html.sh. Stdlib only. +py_binary( + name = "coverage_summary", + srcs = ["coverage_summary.py"], +) + sh_binary( name = "generate_coverage_html", srcs = ["generate_coverage_html.sh"], @@ -75,6 +82,12 @@ py_library( deps = ["@rules_python//python/runfiles"], ) +py_library( + name = "coverage_summary_lib", + srcs = ["coverage_summary.py"], + imports = [".."], +) + # These compile time options are required to cover abnormal termination cases # (death tests). LLVM provides them in combination with a specific profile # setting which is enabled in Bazel via LLVM_PROFILE_CONTINUOUS_MODE. diff --git a/coverage/COVERAGE_GUIDE.md b/coverage/COVERAGE_GUIDE.md index 75deba79..9c66f2c5 100644 --- a/coverage/COVERAGE_GUIDE.md +++ b/coverage/COVERAGE_GUIDE.md @@ -181,6 +181,14 @@ bazel run @score_tooling//coverage:generate_coverage_html -- \ (exit 1)** when below. The model: every uncovered line must eventually be either tested or justified; the threshold is ratcheted up as gaps close. +- Optionally a **markdown job summary** is emitted (`--summary-md `, or + appended to `GITHUB_STEP_SUMMARY` automatically inside GitHub Actions when + the flag is absent): overall/per-directory tables computed from the LCOV + data (which includes the exact-0% baseline records), raw-vs-effective + numbers when justifications ran, and collapsible least-covered/0% file + lists. It is written before the gate decides the exit code, so a failing + gate still leaves the summary on the run page. + ### 2.3 Day-to-day commands ```bash diff --git a/coverage/README.md b/coverage/README.md index dc9368d5..f1abcacd 100644 --- a/coverage/README.md +++ b/coverage/README.md @@ -43,6 +43,7 @@ is copied from it. | `@score_tooling//coverage:generate_coverage_html` | Orchestration: unpacks the report, runs justifications, enforces the threshold, optionally archives. | | `@score_tooling//coverage:justify` | Parses the justification YAML + in-code markers into a manifest. | | `@score_tooling//coverage:effective_coverage` | Post-processes the HTML: restyles justified lines, computes effective coverage, detects stale justifications. | +| `@score_tooling//coverage:coverage_summary` | Renders the markdown job summary from the LCOV data (invoked by `generate_coverage_html` for `--summary-md` / `GITHUB_STEP_SUMMARY`). | | `@score_tooling//coverage:enable_llvm_coverage_for_death_tests` | `cc_feature` adding `-mllvm -runtime-counter-relocation` (continuous-mode profiling for death tests). | ## Prerequisites @@ -207,6 +208,16 @@ COVERAGE_THRESHOLD=95 bazel run @score_tooling//coverage:generate_coverage_html a YAML; add one (`version: 1` + `justifications: []`) when you introduce your first `COV_JUSTIFIED` marker. +**GitHub job summary:** inside GitHub Actions no extra flags are needed — when +`GITHUB_STEP_SUMMARY` is set (and `--summary-md` is not given), a markdown +summary is appended to the workflow run page automatically: overall +line/branch/file tables with progress bars, raw-vs-effective when a +justification YAML is in play, a per-directory rollup (worst first), and +collapsible lists of the least-covered and exact-0% files. Outside Actions, +pass `--summary-md ` to write the same summary to a file. The summary is +emitted before the threshold gate decides the exit code, so a failing gate +still leaves it on the run page. No consumer-side LCOV parsing needed. + `--build_tests_only` matters: without it, coverage builds (not runs) every target matched by the pattern, including e.g. `manual`-tagged or platform-incompatible test binaries. @@ -219,6 +230,7 @@ platform-incompatible test binaries. | Output directory | positional `output-dir` argument (default `coverage_`) | | Platform-specific justifications | `--platform linux\|qnx` (default linux) | | JUnit XMLs subtree in the archive | `--testlogs-subdir ` (default: whole `bazel-testlogs`) | +| Markdown job summary | `--summary-md `; auto-append to `GITHUB_STEP_SUMMARY` when the flag is absent and the variable is set | | Different LLVM version | your own `llvm.toolchain(...)`; pass its labels in step 3 | | Rust branch coverage | `-Zcoverage-options=branch` (needs a nightly-based/rolling Ferrocene; drop the flag on stable) | diff --git a/coverage/coverage_summary.py b/coverage/coverage_summary.py new file mode 100644 index 00000000..cc2e160b --- /dev/null +++ b/coverage/coverage_summary.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Render a markdown coverage summary from the pipeline's LCOV output. + +Invoked by generate_coverage_html.sh to produce a human-readable summary for +GitHub job summary pages (GITHUB_STEP_SUMMARY) or an arbitrary markdown file +(--summary-md). Standard library only. + +Inputs: + --lcov LCOV trace produced by the coverage reporter + (includes exact-0% baseline records). + --justification-report Optional report.json from effective_coverage.py + (raw vs effective metrics, justified counts). + --output Markdown destination. + --append Append to --output instead of overwriting + (GITHUB_STEP_SUMMARY convention). +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict, List, Optional + +BAR_WIDTH = 10 +LEAST_COVERED_LIMIT = 15 + + +class FileCoverage: + """Line/branch counters for one SF record.""" + + def __init__(self, path: str) -> None: + self.path = path + self.lines_found = 0 + self.lines_hit = 0 + # None means "no branch data in the record" (rendered as an em dash). + self.branches_found: Optional[int] = None + self.branches_hit: Optional[int] = None + + @property + def line_pct(self) -> Optional[float]: + return percent(self.lines_hit, self.lines_found) + + +def percent(hit: int, total: int) -> Optional[float]: + """Percentage, or None when the denominator is zero.""" + if total <= 0: + return None + return 100.0 * hit / total + + +def fmt_pct(pct: Optional[float]) -> str: + return "—" if pct is None else f"{pct:.2f}%" + + +def progress_bar(pct: Optional[float], width: int = BAR_WIDTH) -> str: + """Inline-code text progress bar, e.g. `███████░░░`.""" + if pct is None: + return "—" + filled = int(round(pct / 100.0 * width)) + filled = max(0, min(width, filled)) + return "`" + "█" * filled + "░" * (width - filled) + "`" + + +def escape_cell(text: str) -> str: + """Make a path safe inside a markdown table cell.""" + return text.replace("|", "\\|") + + +def parse_lcov(path: Path) -> Optional[List[FileCoverage]]: + """Parse an LCOV trace into per-file counters. + + Returns None when the file does not exist; an empty list when it exists + but contains no records. Branch data prefers BRF/BRH sums and falls back + to counting BRDA entries (taken > 0 counts as hit). + """ + if not path.is_file(): + return None + + files: List[FileCoverage] = [] + current: Optional[FileCoverage] = None + brf = brh = 0 + brda_total = brda_hit = 0 + saw_brf = False + + def flush() -> None: + nonlocal current, brf, brh, brda_total, brda_hit, saw_brf + if current is not None: + if saw_brf: + current.branches_found = brf + current.branches_hit = brh + elif brda_total > 0: + current.branches_found = brda_total + current.branches_hit = brda_hit + files.append(current) + current = None + brf = brh = 0 + brda_total = brda_hit = 0 + saw_brf = False + + # errors="replace" keeps non-UTF8 bytes in paths from crashing the parse. + with open(path, encoding="utf-8", errors="replace") as f: + for raw_line in f: + line = raw_line.strip() + if line.startswith("SF:"): + flush() + current = FileCoverage(line[3:]) + elif current is None: + continue + elif line.startswith("LF:"): + current.lines_found += _int_suffix(line) + elif line.startswith("LH:"): + current.lines_hit += _int_suffix(line) + elif line.startswith("BRF:"): + saw_brf = True + brf += _int_suffix(line) + elif line.startswith("BRH:"): + saw_brf = True + brh += _int_suffix(line) + elif line.startswith("BRDA:"): + # BRDA:,,, — "-" means never + # evaluated, any positive count means taken. + brda_total += 1 + taken = line.rsplit(",", 1)[-1] + if taken not in ("-", "0"): + brda_hit += 1 + elif line == "end_of_record": + flush() + flush() + return files + + +def _int_suffix(line: str) -> int: + try: + return int(line.split(":", 1)[1]) + except (IndexError, ValueError): + return 0 + + +def load_justification_summary(path: Path) -> Optional[Dict]: + """Load the summary block of effective_coverage.py's report.json.""" + try: + with open(path, encoding="utf-8") as f: + report = json.load(f) + except (OSError, json.JSONDecodeError) as e: + print(f"WARNING: could not read justification report {path}: {e}", file=sys.stderr) + return None + summary = report.get("summary") + if not isinstance(summary, dict): + return None + summary = dict(summary) + applied = report.get("applied_justifications") + summary["applied_justification_count"] = len(applied) if isinstance(applied, list) else 0 + return summary + + +def directory_key(path: str) -> str: + """Group by the first one or two path segments (generic, layout-agnostic).""" + parts = path.split("/") + if len(parts) <= 1: + return "(root)" + if len(parts) == 2: + return parts[0] + return "/".join(parts[:2]) + + +def rollup_by_directory(files: List[FileCoverage]) -> List[Dict]: + groups: Dict[str, Dict] = {} + for fc in files: + g = groups.setdefault( + directory_key(fc.path), + {"lines_found": 0, "lines_hit": 0, "files": 0}, + ) + g["lines_found"] += fc.lines_found + g["lines_hit"] += fc.lines_hit + g["files"] += 1 + rows = [ + { + "directory": name, + "pct": percent(g["lines_hit"], g["lines_found"]), + **g, + } + for name, g in groups.items() + ] + # Worst first; groups without countable lines sink to the end. + rows.sort(key=lambda r: (r["pct"] is None, r["pct"], r["directory"])) + return rows + + +def render_markdown(files: List[FileCoverage], justification: Optional[Dict]) -> str: + out: List[str] = ["## Coverage summary", ""] + + if not files: + out.append("_No coverage records found in the LCOV report._") + out.append("") + return "\n".join(out) + + total_lf = sum(f.lines_found for f in files) + total_lh = sum(f.lines_hit for f in files) + branch_files = [f for f in files if f.branches_found is not None] + total_brf = sum(f.branches_found for f in branch_files) if branch_files else 0 + total_brh = sum(f.branches_hit for f in branch_files) if branch_files else 0 + touched = [f for f in files if f.lines_hit > 0] + zero = [f for f in files if f.lines_found > 0 and f.lines_hit == 0] + + line_pct = percent(total_lh, total_lf) + branch_pct = percent(total_brh, total_brf) if branch_files else None + touched_pct = percent(len(touched), len(files)) + + out.append("| Metric | Covered | Total | % | |") + out.append("|---|---:|---:|---:|---|") + out.append(f"| Lines | {total_lh} | {total_lf} | {fmt_pct(line_pct)} | {progress_bar(line_pct)} |") + out.append( + f"| Branches | {total_brh if branch_files else '—'} | " + f"{total_brf if branch_files else '—'} | {fmt_pct(branch_pct)} | {progress_bar(branch_pct)} |" + ) + out.append( + f"| Files with coverage | {len(touched)} | {len(files)} | " + f"{fmt_pct(touched_pct)} | {progress_bar(touched_pct)} |" + ) + out.append(f"| Files at exact 0% | {len(zero)} | {len(files)} | | |") + out.append("") + + if justification is not None: + out.append("### Raw vs effective (justifications applied)") + out.append("") + out.append("| Metric | Raw | Effective |") + out.append("|---|---:|---:|") + out.append( + f"| Line coverage | {justification.get('raw_line_coverage_pct', 0)}% " + f"| {justification.get('effective_line_coverage_pct', 0)}% |" + ) + out.append( + f"| Branch coverage | {justification.get('raw_branch_coverage_pct', 0)}% " + f"| {justification.get('effective_branch_coverage_pct', 0)}% |" + ) + out.append("") + out.append( + f"Justified: {justification.get('justified_lines', 0)} lines, " + f"{justification.get('justified_branches', 0)} branches " + f"({justification.get('applied_justification_count', 0)} justification entries applied, " + f"{justification.get('stale_justifications', 0)} stale)." + ) + out.append("") + + out.append("### Coverage by directory (worst first)") + out.append("") + out.append("| Directory | Files | Lines hit/total | % | |") + out.append("|---|---:|---:|---:|---|") + for row in rollup_by_directory(files): + out.append( + f"| {escape_cell(row['directory'])} | {row['files']} " + f"| {row['lines_hit']}/{row['lines_found']} " + f"| {fmt_pct(row['pct'])} | {progress_bar(row['pct'])} |" + ) + out.append("") + + least = sorted( + (f for f in touched if f.lines_hit < f.lines_found), + key=lambda f: (f.line_pct is None, f.line_pct, f.path), + )[:LEAST_COVERED_LIMIT] + if least: + out.append("
") + out.append(f"Least-covered files with coverage (top {len(least)})") + out.append("") + out.append("| File | Lines hit/total | % | |") + out.append("|---|---:|---:|---|") + for fc in least: + out.append( + f"| {escape_cell(fc.path)} | {fc.lines_hit}/{fc.lines_found} " + f"| {fmt_pct(fc.line_pct)} | {progress_bar(fc.line_pct)} |" + ) + out.append("") + out.append("
") + out.append("") + + if zero: + out.append("
") + out.append(f"Files at exact 0% ({len(zero)})") + out.append("") + for fc in sorted(zero, key=lambda f: f.path): + out.append(f"- `{fc.path}` ({fc.lines_found} lines)") + out.append("") + out.append("
") + out.append("") + + out.append("_Full per-line HTML report: download the coverage artifact of this run._") + out.append("") + return "\n".join(out) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Markdown coverage summary from LCOV") + parser.add_argument("--lcov", type=Path, required=True) + parser.add_argument("--justification-report", type=Path, default=None) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--append", action="store_true") + args = parser.parse_args() + + files = parse_lcov(args.lcov) + if files is None: + print(f"WARNING: LCOV file not found: {args.lcov}", file=sys.stderr) + files = [] + + justification = None + if args.justification_report is not None: + justification = load_justification_summary(args.justification_report) + + markdown = render_markdown(files, justification) + + args.output.parent.mkdir(parents=True, exist_ok=True) + mode = "a" if args.append else "w" + with open(args.output, mode, encoding="utf-8") as f: + f.write(markdown) + print( + f"INFO: coverage summary {'appended to' if args.append else 'written to'} {args.output}", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/coverage/generate_coverage_html.sh b/coverage/generate_coverage_html.sh index 7c77d366..ab919edf 100755 --- a/coverage/generate_coverage_html.sh +++ b/coverage/generate_coverage_html.sh @@ -39,6 +39,15 @@ # --testlogs-subdir Subdirectory of bazel-testlogs to collect JUnit # XMLs from when archiving (default: entire # bazel-testlogs tree). +# --summary-md Write a markdown coverage summary (tables, +# per-directory rollup, 0%-file list) to . +# When ABSENT and the GITHUB_STEP_SUMMARY +# environment variable is set (GitHub Actions), +# the summary is appended there automatically; +# when neither is present, no summary is emitted. +# The summary is written before the threshold +# gate decides the exit code, so a failing gate +# still leaves it on the workflow run page. # output-dir Directory to write the HTML report to # (default: coverage_) # @@ -56,6 +65,7 @@ PLATFORM="linux" OUTPUT_DIR="" JUSTIFICATION_YAML_REL="" TESTLOGS_SUBDIR="" +SUMMARY_MD="" while [[ $# -gt 0 ]]; do case "$1" in @@ -79,6 +89,10 @@ while [[ $# -gt 0 ]]; do TESTLOGS_SUBDIR="${2:?--testlogs-subdir requires a path argument}" shift 2 ;; + --summary-md) + SUMMARY_MD="${2:?--summary-md requires a path argument}" + shift 2 + ;; *) OUTPUT_DIR="$1" shift @@ -206,6 +220,31 @@ else GATE_KIND="Raw" fi +# --------------------------------------------------------------------------- +# Optional markdown summary (--summary-md, or GITHUB_STEP_SUMMARY when the +# flag is absent). Emitted BEFORE the threshold gate so a failing gate still +# leaves the summary on the workflow run page. +# --------------------------------------------------------------------------- +if [[ -n "${SUMMARY_MD}" || -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + SUMMARY_ARGS=(--lcov "${TMPDIR_EXTRACT}/lcov_report/lcov.dat") + if [[ -n "${JUSTIFICATION_DIR}" && -f "${JUSTIFICATION_DIR}/report.json" ]]; then + SUMMARY_ARGS+=(--justification-report "${JUSTIFICATION_DIR}/report.json") + fi + if [[ -n "${SUMMARY_MD}" ]]; then + case "${SUMMARY_MD}" in + /*) : ;; + *) SUMMARY_MD="${BUILD_WORKSPACE_DIRECTORY}/${SUMMARY_MD}" ;; + esac + bazel run @score_tooling//coverage:coverage_summary -- \ + "${SUMMARY_ARGS[@]}" --output "${SUMMARY_MD}" + echo "Coverage summary written to: ${SUMMARY_MD}" + else + bazel run @score_tooling//coverage:coverage_summary -- \ + "${SUMMARY_ARGS[@]}" --output "${GITHUB_STEP_SUMMARY}" --append + echo "Coverage summary appended to GITHUB_STEP_SUMMARY" + fi +fi + # Threshold check (default: 100%). Fails the run when below. if ! awk "BEGIN {exit (${GATE_PCT} >= ${THRESHOLD}) ? 0 : 1}"; then echo "ERROR: ${GATE_KIND} coverage ${GATE_PCT}% is below threshold ${THRESHOLD}%" >&2 diff --git a/coverage/integration_tests/run_integration_test.sh b/coverage/integration_tests/run_integration_test.sh index 927bf428..2387ecc3 100755 --- a/coverage/integration_tests/run_integration_test.sh +++ b/coverage/integration_tests/run_integration_test.sh @@ -22,6 +22,11 @@ set -euo pipefail cd "$(dirname "$0")" +# In GitHub Actions GITHUB_STEP_SUMMARY is set for THIS job; unset it so the +# many generate_coverage_html invocations below don't each append to the real +# run page. The dedicated summary test sets its own target file. +unset GITHUB_STEP_SUMMARY || true + echo "=== Running coverage build ===" bazel coverage --config=llvm_cov //... --build_tests_only @@ -52,6 +57,40 @@ if [[ ! -f coverage_linux/index.html ]]; then fi echo "OK: no-yaml mode works (HTML produced, raw gate enforced)" +echo "=== --summary-md must produce a markdown job summary ===" +rm -f summary.md +COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml "${YAML}" --summary-md summary.md +for marker in "## Coverage summary" "| Lines |" "Raw vs effective" \ + "Coverage by directory" "Files at exact 0% (2)"; do + if ! grep -qF "${marker}" summary.md; then + echo "ERROR: '${marker}' missing from summary.md" >&2 + exit 1 + fi +done +grep -q "█" summary.md || { echo "ERROR: progress bars missing from summary.md" >&2; exit 1; } +echo "OK: --summary-md works" + +echo "=== Summary must still be written when the gate FAILS ===" +rm -f summary.md +if COVERAGE_THRESHOLD=100 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml "${YAML}" --summary-md summary.md; then + echo "ERROR: gate unexpectedly passed at threshold 100" >&2 + exit 1 +fi +[[ -s summary.md ]] || { echo "ERROR: summary.md missing after failing gate" >&2; exit 1; } +echo "OK: summary survives a failing gate" + +echo "=== GITHUB_STEP_SUMMARY convenience default (no flag) ===" +rm -f step_summary.md +printf '# existing content\n' > step_summary.md +GITHUB_STEP_SUMMARY="$(pwd)/step_summary.md" COVERAGE_THRESHOLD=10 \ + bazel run @score_tooling//coverage:generate_coverage_html -- --yaml "${YAML}" +grep -qF "# existing content" step_summary.md || { echo "ERROR: append mode overwrote the step summary" >&2; exit 1; } +grep -qF "## Coverage summary" step_summary.md || { echo "ERROR: summary not appended to GITHUB_STEP_SUMMARY" >&2; exit 1; } +rm -f summary.md step_summary.md +echo "OK: GITHUB_STEP_SUMMARY convenience works" + echo "=== --archive-dir must produce an unzipped artifacts tree ===" COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html -- \ --yaml "${YAML}" --archive-dir artifacts_dir diff --git a/coverage/tests/BUILD b/coverage/tests/BUILD index 68ca4de9..010aef5b 100644 --- a/coverage/tests/BUILD +++ b/coverage/tests/BUILD @@ -24,3 +24,9 @@ py_test( srcs = ["reporter_test.py"], deps = ["//coverage:reporter_lib"], ) + +py_test( + name = "coverage_summary_test", + srcs = ["coverage_summary_test.py"], + deps = ["//coverage:coverage_summary_lib"], +) diff --git a/coverage/tests/coverage_summary_test.py b/coverage/tests/coverage_summary_test.py new file mode 100644 index 00000000..8efda310 --- /dev/null +++ b/coverage/tests/coverage_summary_test.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Unit tests for the markdown coverage summary.""" + +import json +import tempfile +import unittest +from pathlib import Path + +from coverage.coverage_summary import ( + directory_key, + load_justification_summary, + parse_lcov, + percent, + progress_bar, + render_markdown, + rollup_by_directory, +) + +LCOV_TWO_FILES = ( + "SF:src/foo/a.cpp\n" + "DA:1,1\nDA:2,0\n" + "BRF:4\nBRH:3\n" + "LF:2\nLH:1\n" + "end_of_record\n" + "SF:rust/main.rs\n" + "DA:1,0\n" + "LF:3\nLH:0\n" + "end_of_record\n" +) + + +def _write(tmp: str, name: str, content: str) -> Path: + p = Path(tmp) / name + p.write_text(content, encoding="utf-8") + return p + + +class ParseLcovTest(unittest.TestCase): + def test_missing_file_returns_none(self): + self.assertIsNone(parse_lcov(Path("/nonexistent/lcov.dat"))) + + def test_empty_file_returns_empty_list(self): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(parse_lcov(_write(tmp, "e.dat", "")), []) + + def test_line_and_branch_counters(self): + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", LCOV_TWO_FILES)) + self.assertEqual(len(files), 2) + a, main_rs = files + self.assertEqual((a.lines_hit, a.lines_found), (1, 2)) + self.assertEqual((a.branches_hit, a.branches_found), (3, 4)) + self.assertEqual((main_rs.lines_hit, main_rs.lines_found), (0, 3)) + + def test_lf_without_brf_yields_no_branch_data(self): + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", "SF:a.cpp\nLF:5\nLH:2\nend_of_record\n")) + self.assertIsNone(files[0].branches_found) + + def test_brda_fallback_when_no_brf(self): + lcov = "SF:a.cpp\nBRDA:1,0,0,3\nBRDA:1,0,1,-\nBRDA:2,0,0,0\nLF:2\nLH:2\nend_of_record\n" + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", lcov)) + self.assertEqual((files[0].branches_hit, files[0].branches_found), (1, 3)) + + def test_record_without_end_of_record_is_flushed(self): + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", "SF:a.cpp\nLF:1\nLH:1\n")) + self.assertEqual(len(files), 1) + + def test_non_utf8_bytes_do_not_crash(self): + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp) / "l.dat" + p.write_bytes(b"SF:src/\xff\xfe.cpp\nLF:1\nLH:0\nend_of_record\n") + files = parse_lcov(p) + self.assertEqual(len(files), 1) + self.assertEqual(files[0].lines_found, 1) + + +class MathHelpersTest(unittest.TestCase): + def test_percent_zero_denominator_is_none(self): + self.assertIsNone(percent(0, 0)) + self.assertIsNone(percent(5, 0)) + + def test_progress_bar_bounds(self): + self.assertEqual(progress_bar(0.0), "`" + "░" * 10 + "`") + self.assertEqual(progress_bar(100.0), "`" + "█" * 10 + "`") + self.assertEqual(progress_bar(None), "—") + + def test_directory_key_grouping(self): + self.assertEqual(directory_key("a.cpp"), "(root)") + self.assertEqual(directory_key("src/a.cpp"), "src") + self.assertEqual(directory_key("src/foo/a.cpp"), "src/foo") + self.assertEqual(directory_key("src/foo/bar/a.cpp"), "src/foo") + + +class RollupTest(unittest.TestCase): + def test_worst_directory_first(self): + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", LCOV_TWO_FILES)) + rows = rollup_by_directory(files) + self.assertEqual(rows[0]["directory"], "rust") + self.assertEqual(rows[0]["pct"], 0.0) + self.assertEqual(rows[1]["directory"], "src/foo") + + +class RenderTest(unittest.TestCase): + def _render(self, justification=None): + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", LCOV_TWO_FILES)) + return render_markdown(files, justification) + + def test_empty_input_renders_note(self): + md = render_markdown([], None) + self.assertIn("No coverage records", md) + + def test_overall_table_and_zero_section(self): + md = self._render() + self.assertIn("| Lines | 1 | 5 | 20.00% |", md) + self.assertIn("| Branches | 3 | 4 | 75.00% |", md) + self.assertIn("Files at exact 0% (1)", md) + self.assertIn("`rust/main.rs` (3 lines)", md) + self.assertIn("█", md) # progress bars present + self.assertIn("
", md) + + def test_justification_section(self): + justification = { + "raw_line_coverage_pct": 20.0, + "effective_line_coverage_pct": 40.0, + "raw_branch_coverage_pct": 75.0, + "effective_branch_coverage_pct": 75.0, + "justified_lines": 1, + "justified_branches": 0, + "stale_justifications": 0, + "applied_justification_count": 1, + } + md = self._render(justification) + self.assertIn("Raw vs effective", md) + self.assertIn("| Line coverage | 20.0% | 40.0% |", md) + self.assertIn("1 justification entries applied", md) + + def test_branch_dash_when_no_branch_data(self): + with tempfile.TemporaryDirectory() as tmp: + files = parse_lcov(_write(tmp, "l.dat", "SF:a.cpp\nLF:2\nLH:1\nend_of_record\n")) + md = render_markdown(files, None) + self.assertIn("| Branches | — | — | — | — |", md) + + +class JustificationReportTest(unittest.TestCase): + def test_loads_summary_and_counts_applied(self): + report = { + "version": 1, + "summary": {"raw_line_coverage_pct": 50.0, "justified_lines": 2}, + "applied_justifications": [{"id": "a"}, {"id": "b"}], + } + with tempfile.TemporaryDirectory() as tmp: + p = _write(tmp, "report.json", json.dumps(report)) + summary = load_justification_summary(p) + self.assertEqual(summary["applied_justification_count"], 2) + self.assertEqual(summary["justified_lines"], 2) + + def test_malformed_json_returns_none(self): + with tempfile.TemporaryDirectory() as tmp: + p = _write(tmp, "report.json", "{not json") + self.assertIsNone(load_justification_summary(p)) + + +if __name__ == "__main__": + unittest.main() From 9f3aaf6ca160ca1cd92aa89bde3252334cab10db Mon Sep 17 00:00:00 2001 From: Dan Calavrezo <195309321+dcalavrezo-qorix@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:47:36 +0300 Subject: [PATCH 2/2] coverage: clarify summary test resets, import-lib pattern and bar docstring Review follow-up for #440: comment why each summary test section deletes summary.md before its own run (stale-file false passes; the failing-gate section must prove re-creation), note that imports=[".."] is confined to the *_lib unit-test import helpers, and state in the docstring that the progress bar visualizes the coverage percentage. --- coverage/BUILD | 3 +++ coverage/coverage_summary.py | 7 ++++++- coverage/integration_tests/run_integration_test.sh | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/coverage/BUILD b/coverage/BUILD index 4ea1af08..e2ca9524 100644 --- a/coverage/BUILD +++ b/coverage/BUILD @@ -82,6 +82,9 @@ py_library( deps = ["@rules_python//python/runfiles"], ) +# Import-only test helper like merger_lib/reporter_lib above: imports=[".."] +# makes `from coverage.coverage_summary import ...` resolvable from the unit +# tests. Only the *_lib targets carry it; the binaries are unaffected. py_library( name = "coverage_summary_lib", srcs = ["coverage_summary.py"], diff --git a/coverage/coverage_summary.py b/coverage/coverage_summary.py index cc2e160b..923a0f46 100644 --- a/coverage/coverage_summary.py +++ b/coverage/coverage_summary.py @@ -65,7 +65,12 @@ def fmt_pct(pct: Optional[float]) -> str: def progress_bar(pct: Optional[float], width: int = BAR_WIDTH) -> str: - """Inline-code text progress bar, e.g. `███████░░░`.""" + """Render a COVERAGE percentage as an inline-code bar, e.g. `███████░░░`. + + Purely a visual aid next to the numeric cell so table rows can be + compared at a glance on the run page — it shows how much of the code is + covered, nothing else. + """ if pct is None: return "—" filled = int(round(pct / 100.0 * width)) diff --git a/coverage/integration_tests/run_integration_test.sh b/coverage/integration_tests/run_integration_test.sh index 2387ecc3..e3fd5e14 100755 --- a/coverage/integration_tests/run_integration_test.sh +++ b/coverage/integration_tests/run_integration_test.sh @@ -57,6 +57,10 @@ if [[ ! -f coverage_linux/index.html ]]; then fi echo "OK: no-yaml mode works (HTML produced, raw gate enforced)" +# The following sections all run, in order. Each one deletes summary.md +# before its own generate_coverage_html invocation so a stale file from the +# previous section cannot produce a false pass — in particular, the +# failing-gate section must prove the file was RE-created by THAT run. echo "=== --summary-md must produce a markdown job summary ===" rm -f summary.md COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html -- \