Skip to content

Commit bed235c

Browse files
matmenclaude
andcommitted
Measure test coverage in CI
npm run coverage builds with source maps and runs the tests under c8 (V8's coverage, mapped back to src), and scripts/coverage-report.js writes coverage/report.md: the totals and the files with the most untested lines. coverage.yml runs it for every push and pull requests from forks, and also measures the base in a second checkout (the previous commit on master, else where the branch left master) to show what changed. The report goes to the job summary, to a commit comment on pushes to master, and to one comment on a branch's pull request, updated by later pushes. Its own workflow: node.yml is called by release.yml, which doesn't grant the write permissions. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YJgRK5qpwJEbHPuFv6kgAQ
1 parent 46108dd commit bed235c

7 files changed

Lines changed: 147 additions & 2 deletions

File tree

‎.c8rc.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"include": ["dist/**", "src/**"],
3+
"reporter": ["text-summary", "json-summary", "html"],
4+
"report-dir": "coverage"
5+
}

‎.github/workflows/coverage.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: coverage
2+
3+
# measures the test coverage (npm run coverage) of every pushed branch and pull requests from forks, and how it
4+
# changed: compared with the commit before the push on master, else with where the branch (or pull request) left
5+
# master, measured in the same job. The job summary shows it; pushes to master get it as a commit comment, pushes to a
6+
# branch with a pull request as a comment on it (one, updated by later pushes). Pull requests from forks have no
7+
# write access: only the job summary
8+
on:
9+
push:
10+
branches: ['**']
11+
pull_request:
12+
13+
concurrency:
14+
group: coverage-${{ github.event_name }}-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
coverage:
19+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
20+
runs-on: ubuntu-latest
21+
permissions:
22+
contents: write
23+
pull-requests: write
24+
steps:
25+
- uses: actions/checkout@v7
26+
with:
27+
fetch-depth: 0
28+
- uses: actions/setup-node@v7
29+
with:
30+
node-version: 24.x
31+
32+
- run: npm install
33+
- run: npm run coverage
34+
35+
# the base, measured in a second checkout: skipped without one (a new branch, a force push that dropped it, a
36+
# base from before npm run coverage existed), and a base whose tests fail just gives no comparison
37+
- id: base
38+
continue-on-error: true
39+
run: |
40+
if [ "$EVENT" = pull_request ]; then base=$(git merge-base "$PR_BASE" HEAD); name="the pull request's base"
41+
elif [ "$REF" = refs/heads/master ]; then base=$BEFORE; name="the previous commit on master"
42+
else base=$(git merge-base origin/master HEAD); name="master where the branch left it"; fi
43+
if [ -z "$base" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null || [ "$base" = "$(git rev-parse HEAD)" ]; then
44+
echo "no base to compare with"; exit 0
45+
fi
46+
git worktree add -q ../base "$base"
47+
cd ../base
48+
if ! node -e "process.exit(require('./package.json').scripts?.coverage ? 0 : 1)"; then echo "the base has no npm run coverage"; exit 0; fi
49+
npm install && npm run coverage
50+
echo "summary=$PWD/coverage/coverage-summary.json" >> "$GITHUB_OUTPUT"
51+
echo "root=$PWD" >> "$GITHUB_OUTPUT"
52+
echo "name=$name (${base:0:7})" >> "$GITHUB_OUTPUT"
53+
env:
54+
EVENT: ${{ github.event_name }}
55+
REF: ${{ github.ref }}
56+
BEFORE: ${{ github.event.before }}
57+
PR_BASE: ${{ github.event.pull_request.base.sha }}
58+
- if: steps.base.outputs.summary
59+
run: node scripts/coverage-report.js "$SUMMARY" "$ROOT"
60+
env:
61+
SUMMARY: ${{ steps.base.outputs.summary }}
62+
ROOT: ${{ steps.base.outputs.root }}
63+
BASE_NAME: ${{ steps.base.outputs.name }}
64+
- run: cat coverage/report.md >> "$GITHUB_STEP_SUMMARY"
65+
66+
- if: github.event_name == 'push' && github.ref == 'refs/heads/master'
67+
run: gh api "repos/$REPOSITORY/commits/$SHA/comments" -F body=@coverage/report.md
68+
env:
69+
GH_TOKEN: ${{ github.token }}
70+
REPOSITORY: ${{ github.repository }}
71+
SHA: ${{ github.sha }}
72+
73+
# the pull request of the pushed branch, if any: its coverage comment is updated, or posted the first time
74+
- if: github.event_name == 'push' && github.ref != 'refs/heads/master'
75+
run: |
76+
pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number')
77+
if [ -z "$pr" ]; then echo "no pull request"; exit 0; fi
78+
comment=$(gh api "repos/$REPOSITORY/issues/$pr/comments" --paginate --jq '.[] | select(.body | startswith("<!-- coverage -->")) | .id' | head -1)
79+
if [ -n "$comment" ]; then gh api -X PATCH "repos/$REPOSITORY/issues/comments/$comment" -F body=@coverage/report.md
80+
else gh api "repos/$REPOSITORY/issues/$pr/comments" -F body=@coverage/report.md; fi
81+
env:
82+
GH_TOKEN: ${{ github.token }}
83+
REPOSITORY: ${{ github.repository }}
84+
BRANCH: ${{ github.ref_name }}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ dist/
33
docs/
44
.idea/
55
.DS_Store
6+
coverage/

‎eslint.config.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import sonarjs from 'eslint-plugin-sonarjs';
55
import tseslint from 'typescript-eslint';
66

77
export default tseslint.config(
8-
{ ignores: ['dist/', 'docs/', 'bench/', 'node_modules/'] },
8+
{ ignores: ['dist/', 'docs/', 'bench/', 'coverage/', 'node_modules/'] },
99
{
1010
files: ['src/**/*.ts'],
1111
extends: [js.configs.recommended, ...tseslint.configs.recommendedTypeChecked],

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
"lint": "eslint",
2828
"docs": "node scripts/docs.js",
2929
"test": "npm run build && node ./tests/run.js",
30+
"coverage": "node scripts/build.js --source-maps && c8 node ./tests/run.js && node scripts/coverage-report.js",
3031
"prepack": "npm run build",
3132
"version": "node scripts/version.js && git add src/version.ts jsr.json"
3233
},
@@ -68,6 +69,7 @@
6869
},
6970
"devDependencies": {
7071
"@eslint/js": "9.39.5",
72+
"c8": "12.0.0",
7173
"eslint": "9.39.5",
7274
"eslint-plugin-sonarjs": "4.2.2",
7375
"typedoc": "0.27.9",

‎scripts/build.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ require('./version.js').setVersion();
1414
fs.rmSync(path.join(root, 'dist'), { recursive: true, force: true });
1515
// declarations keep the doc comments (editors show them), the JavaScript doesn't need them
1616
run(['-p', 'tsconfig.json', '--emitDeclarationOnly']);
17-
run(['-p', 'tsconfig.json', '--declaration', 'false', '--removeComments']);
17+
// --source-maps (npm run coverage) maps the built JavaScript back to src
18+
run(['-p', 'tsconfig.json', '--declaration', 'false', '--removeComments', ...process.argv.includes('--source-maps') ? ['--sourceMap'] : []]);
1819

1920
// declarations keep the .ts import specifiers of the sources, point them at the built files
2021
const rewrite = dir => {

‎scripts/coverage-report.js‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// turns coverage/coverage-summary.json (npm run coverage) into coverage/report.md for the CI's job summary and
2+
// comments: the totals and the files with the most untested lines, and with the summary of a base commit
3+
// (node scripts/coverage-report.js <base summary> <base checkout>) how they changed and which files changed most
4+
const fs = require('fs');
5+
const path = require('path');
6+
7+
const root = path.join(__dirname, '..');
8+
// per file (relative to its checkout) the line, branch and function coverage, and the untested lines
9+
const read = (file, checkout) => {
10+
const summary = JSON.parse(fs.readFileSync(file, 'utf8'));
11+
const files = new Map();
12+
for (const [name, s] of Object.entries(summary))
13+
if (name !== 'total') files.set(path.relative(checkout, name), {...s, missing: s.lines.total - s.lines.covered});
14+
return {total: summary.total, files};
15+
};
16+
17+
const head = read(path.join(root, 'coverage/coverage-summary.json'), root);
18+
const base = process.argv[2] ? read(process.argv[2], process.argv[3]) : null;
19+
20+
const percent = ({pct}) => `${pct.toFixed(1)}%`;
21+
const change = (now, before) => {
22+
if (!before) return '';
23+
const d = now.pct - before.pct;
24+
return Math.abs(d) < 0.05 ? '' : ` (${d > 0 ? '+' : ''}${d.toFixed(1)})`;
25+
};
26+
const row = (name, s, b) => `| ${name} | ${percent(s.lines)}${change(s.lines, b?.lines)} | ${percent(s.branches)}${change(s.branches, b?.branches)} | ${percent(s.functions)}${change(s.functions, b?.functions)} |`;
27+
const header = ['| | Lines | Branches | Functions |', '|---|--:|--:|--:|'];
28+
29+
const report = ['<!-- coverage -->', '### Test coverage', ''];
30+
if (base) report.push(`Compared with ${process.env.BASE_NAME || 'the base commit'}.`, '');
31+
report.push(...header, row('**total**', head.total, base?.total));
32+
const untested = [...head.files].sort((a, b) => b[1].missing - a[1].missing).slice(0, 10);
33+
for (const [file, s] of untested) report.push(row(`\`${file}\` (${s.missing} lines untested)`, s, base?.files.get(file)));
34+
35+
if (base) {
36+
// the files whose line coverage changed, new and removed ones
37+
const changed = [];
38+
for (const [file, s] of head.files) {
39+
const b = base.files.get(file);
40+
if (!b) changed.push([file, Infinity, `| \`${file}\` (new) | ${percent(s.lines)} | ${percent(s.branches)} | ${percent(s.functions)} |`]);
41+
else if (Math.abs(s.lines.pct - b.lines.pct) >= 0.05) changed.push([file, Math.abs(s.lines.pct - b.lines.pct), row(`\`${file}\``, s, b)]);
42+
}
43+
for (const file of base.files.keys()) if (!head.files.has(file)) changed.push([file, Infinity, `| \`${file}\` (removed) | | | |`]);
44+
changed.sort((a, b) => b[1] - a[1]);
45+
report.push('', changed.length ? '#### Changed' : 'No file\'s line coverage changed.');
46+
if (changed.length) report.push('', ...header, ...changed.slice(0, 20).map(c => c[2]));
47+
if (changed.length > 20) report.push('', `and ${changed.length - 20} more`);
48+
}
49+
report.push('');
50+
51+
fs.writeFileSync(path.join(root, 'coverage/report.md'), report.join('\n'));
52+
console.log(report.join('\n'));

0 commit comments

Comments
 (0)