Add performance benchmark CI to compare PR vs base branch (#32)

.github/workflows/benchmark.yml

@@ -0,0 +1,108 @@
+name: Benchmark
+
+on:
+  pull_request:
+    branches: [ master, main ]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          path: pr
+
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-benchmark
+
+      - name: Run PR benchmarks (time)
+        working-directory: pr
+        run: pytest benchmarks/test_time.py --benchmark-only --benchmark-json=../pr-time.json
+
+      - name: Run PR benchmarks (memory)
+        working-directory: pr
+        run: python benchmarks/memory_bench.py --output ../pr-memory.json
+
+      - name: Run base benchmarks (time)
+        working-directory: base
+        run: |
+          if [ -f benchmarks/test_time.py ]; then
+            pytest benchmarks/test_time.py --benchmark-only --benchmark-json=../base-time.json
+          else
+            echo '{"benchmarks": []}' > ../base-time.json
+          fi
+
+      - name: Run base benchmarks (memory)
+        working-directory: base
+        run: |
+          if [ -f benchmarks/memory_bench.py ]; then
+            python benchmarks/memory_bench.py --output ../base-memory.json
+          else
+            echo '{"benchmarks": []}' > ../base-memory.json
+          fi
+
+      - name: Compare results
+        run: |
+          python pr/benchmarks/compare.py \
+            --base-time base-time.json --pr-time pr-time.json \
+            --base-memory base-memory.json --pr-memory pr-memory.json \
+            --output report.md
+
+      - name: Upload raw results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: |
+            pr-time.json
+            base-time.json
+            pr-memory.json
+            base-memory.json
+            report.md
+
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('report.md', 'utf8');
+            const marker = '<!-- acl-python-benchmark-report -->';
+            const commentBody = `${marker}\n${body}`;
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const existing = comments.find(c => c.body && c.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body: commentBody,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: commentBody,
+              });
+            }

benchmarks/compare.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Compare PR-branch vs base-branch benchmark results and render a Markdown
+report suitable for posting as a PR comment.
+
+Usage:
+    python benchmarks/compare.py \\
+        --base-time base-time.json --pr-time pr-time.json \\
+        --base-memory base-memory.json --pr-memory pr-memory.json \\
+        --output report.md
+"""
+
+import argparse
+import json
+import re
+
+NAME_RE = re.compile(r"\[(.+)\]$")
+
+
+def _benchmark_name(raw_name):
+    m = NAME_RE.search(raw_name)
+    if m:
+        return m.group(1)
+    if raw_name.startswith("test_"):
+        return raw_name[len("test_") :]
+    return raw_name
+
+
+def load_time_results(path):
+    with open(path) as f:
+        data = json.load(f)
+    return {
+        _benchmark_name(b["name"]): b["stats"]["mean"]
+        for b in data.get("benchmarks", [])
+    }
+
+
+def load_memory_results(path):
+    with open(path) as f:
+        data = json.load(f)
+    return {b["name"]: b["peak_bytes"] for b in data.get("benchmarks", [])}
+
+
+def fmt_time(seconds):
+    if seconds < 1e-3:
+        return f"{seconds * 1e6:.1f}µs"
+    if seconds < 1:
+        return f"{seconds * 1e3:.2f}ms"
+    return f"{seconds:.3f}s"
+
+
+def fmt_bytes(n):
+    value = float(n)
+    for unit in ("B", "KiB", "MiB", "GiB"):
+        if abs(value) < 1024:
+            return f"{value:.1f}{unit}"
+        value /= 1024
+    return f"{value:.1f}TiB"
+
+
+def fmt_delta(base, pr):
+    if base == 0:
+        return "n/a"
+    pct = (pr - base) / base * 100
+    if pct <= -1:
+        mark = "✅"
+    elif pct > 5:
+        mark = "⚠️"
+    else:
+        mark = ""
+    sign = "+" if pct >= 0 else ""
+    return f"{sign}{pct:.1f}% {mark}".strip()
+
+
+def build_table(names, base_map, pr_map, fmt_value):
+    lines = ["| Benchmark | Base | PR | Delta |", "|---|---|---|---|"]
+    for name in names:
+        base_v = base_map.get(name)
+        pr_v = pr_map.get(name)
+        if base_v is None or pr_v is None:
+            lines.append(
+                f"| {name} | {'n/a' if base_v is None else fmt_value(base_v)} | "
+                f"{'n/a' if pr_v is None else fmt_value(pr_v)} | n/a |"
+            )
+            continue
+        lines.append(
+            f"| {name} | {fmt_value(base_v)} | {fmt_value(pr_v)} | {fmt_delta(base_v, pr_v)} |"
+        )
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-time", required=True)
+    parser.add_argument("--pr-time", required=True)
+    parser.add_argument("--base-memory", required=True)
+    parser.add_argument("--pr-memory", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    base_time = load_time_results(args.base_time)
+    pr_time = load_time_results(args.pr_time)
+    base_memory = load_memory_results(args.base_memory)
+    pr_memory = load_memory_results(args.pr_memory)
+
+    time_names = sorted(set(base_time) | set(pr_time))
+    memory_names = sorted(set(base_memory) | set(pr_memory))
+
+    report = ["## \U0001f4ca Performance Benchmark Results", ""]
+    if not time_names and not memory_names:
+        report.append("No benchmark results were produced.")
+    else:
+        report.append("### ⏱️ Execution time (mean, lower is better)")
+        report.append(
+            build_table(time_names, base_time, pr_time, fmt_time)
+            if time_names
+            else "n/a"
+        )
+        report.append("")
+        report.append("### \U0001f4be Peak memory (lower is better)")
+        report.append(
+            build_table(memory_names, base_memory, pr_memory, fmt_bytes)
+            if memory_names
+            else "n/a"
+        )
+        report.append("")
+        report.append(
+            "<sub>Base = target branch, PR = this branch. Negative delta = improvement. "
+            "✅ improvement ≥ 1%, ⚠️ regression > 5%. "
+            '"n/a" base entries mean the target branch has no benchmark suite yet.</sub>'
+        )
+
+    text = "\n".join(report)
+    with open(args.output, "w") as f:
+        f.write(text)
+    print(text)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/memory_bench.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Measure peak memory usage (via tracemalloc) for ACL-python's core data
+structures, using the same workloads as benchmarks/test_time.py.
+
+Usage:
+    python benchmarks/memory_bench.py --output result.json
+"""
+
+import argparse
+import json
+import os
+import sys
+import tracemalloc
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import workloads  # noqa: E402
+
+
+def measure(build_fn, run_fn):
+    args = build_fn()
+    tracemalloc.start()
+    before, _ = tracemalloc.get_traced_memory()
+    result = run_fn(*args)
+    _, peak = tracemalloc.get_traced_memory()
+    tracemalloc.stop()
+    del result
+    return peak - before
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    results = []
+    for name, build_fn, run_fn in workloads.BENCHMARKS:
+        peak_bytes = measure(build_fn, run_fn)
+        results.append({"name": name, "peak_bytes": peak_bytes})
+        print(f"{name}: {peak_bytes / 1024:.1f} KiB")
+
+    with open(args.output, "w") as f:
+        json.dump({"benchmarks": results}, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/test_time.py

@@ -0,0 +1,24 @@
+"""Execution-time benchmarks, run via pytest-benchmark.
+
+Usage:
+    pytest benchmarks/test_time.py --benchmark-json=result.json
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import workloads  # noqa: E402
+
+
+@pytest.mark.parametrize(
+    "name,build_fn,run_fn",
+    workloads.BENCHMARKS,
+    ids=[b[0] for b in workloads.BENCHMARKS],
+)
+def test_benchmark(benchmark, name, build_fn, run_fn):
+    args = build_fn()
+    benchmark.pedantic(run_fn, args=args, rounds=3, warmup_rounds=1)