diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..f49640f --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,108 @@ +name: Benchmark + +on: + pull_request: + branches: [ master, main ] + +permissions: + contents: read + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + path: pr + + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-benchmark + + - name: Run PR benchmarks (time) + working-directory: pr + run: pytest benchmarks/test_time.py --benchmark-only --benchmark-json=../pr-time.json + + - name: Run PR benchmarks (memory) + working-directory: pr + run: python benchmarks/memory_bench.py --output ../pr-memory.json + + - name: Run base benchmarks (time) + working-directory: base + run: | + if [ -f benchmarks/test_time.py ]; then + pytest benchmarks/test_time.py --benchmark-only --benchmark-json=../base-time.json + else + echo '{"benchmarks": []}' > ../base-time.json + fi + + - name: Run base benchmarks (memory) + working-directory: base + run: | + if [ -f benchmarks/memory_bench.py ]; then + python benchmarks/memory_bench.py --output ../base-memory.json + else + echo '{"benchmarks": []}' > ../base-memory.json + fi + + - name: Compare results + run: | + python pr/benchmarks/compare.py \ + --base-time base-time.json --pr-time pr-time.json \ + --base-memory base-memory.json --pr-memory pr-memory.json \ + --output report.md + + - name: Upload raw results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: | + pr-time.json + base-time.json + pr-memory.json + base-memory.json + report.md + + - name: Comment on PR + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('report.md', 'utf8'); + const marker = ''; + const commentBody = `${marker}\n${body}`; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: commentBody, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: commentBody, + }); + } diff --git a/benchmarks/compare.py b/benchmarks/compare.py new file mode 100644 index 0000000..0d783d6 --- /dev/null +++ b/benchmarks/compare.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Compare PR-branch vs base-branch benchmark results and render a Markdown +report suitable for posting as a PR comment. + +Usage: + python benchmarks/compare.py \\ + --base-time base-time.json --pr-time pr-time.json \\ + --base-memory base-memory.json --pr-memory pr-memory.json \\ + --output report.md +""" + +import argparse +import json +import re + +NAME_RE = re.compile(r"\[(.+)\]$") + + +def _benchmark_name(raw_name): + m = NAME_RE.search(raw_name) + if m: + return m.group(1) + if raw_name.startswith("test_"): + return raw_name[len("test_") :] + return raw_name + + +def load_time_results(path): + with open(path) as f: + data = json.load(f) + return { + _benchmark_name(b["name"]): b["stats"]["mean"] + for b in data.get("benchmarks", []) + } + + +def load_memory_results(path): + with open(path) as f: + data = json.load(f) + return {b["name"]: b["peak_bytes"] for b in data.get("benchmarks", [])} + + +def fmt_time(seconds): + if seconds < 1e-3: + return f"{seconds * 1e6:.1f}µs" + if seconds < 1: + return f"{seconds * 1e3:.2f}ms" + return f"{seconds:.3f}s" + + +def fmt_bytes(n): + value = float(n) + for unit in ("B", "KiB", "MiB", "GiB"): + if abs(value) < 1024: + return f"{value:.1f}{unit}" + value /= 1024 + return f"{value:.1f}TiB" + + +def fmt_delta(base, pr): + if base == 0: + return "n/a" + pct = (pr - base) / base * 100 + if pct <= -1: + mark = "✅" + elif pct > 5: + mark = "⚠️" + else: + mark = "" + sign = "+" if pct >= 0 else "" + return f"{sign}{pct:.1f}% {mark}".strip() + + +def build_table(names, base_map, pr_map, fmt_value): + lines = ["| Benchmark | Base | PR | Delta |", "|---|---|---|---|"] + for name in names: + base_v = base_map.get(name) + pr_v = pr_map.get(name) + if base_v is None or pr_v is None: + lines.append( + f"| {name} | {'n/a' if base_v is None else fmt_value(base_v)} | " + f"{'n/a' if pr_v is None else fmt_value(pr_v)} | n/a |" + ) + continue + lines.append( + f"| {name} | {fmt_value(base_v)} | {fmt_value(pr_v)} | {fmt_delta(base_v, pr_v)} |" + ) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-time", required=True) + parser.add_argument("--pr-time", required=True) + parser.add_argument("--base-memory", required=True) + parser.add_argument("--pr-memory", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + base_time = load_time_results(args.base_time) + pr_time = load_time_results(args.pr_time) + base_memory = load_memory_results(args.base_memory) + pr_memory = load_memory_results(args.pr_memory) + + time_names = sorted(set(base_time) | set(pr_time)) + memory_names = sorted(set(base_memory) | set(pr_memory)) + + report = ["## \U0001f4ca Performance Benchmark Results", ""] + if not time_names and not memory_names: + report.append("No benchmark results were produced.") + else: + report.append("### ⏱️ Execution time (mean, lower is better)") + report.append( + build_table(time_names, base_time, pr_time, fmt_time) + if time_names + else "n/a" + ) + report.append("") + report.append("### \U0001f4be Peak memory (lower is better)") + report.append( + build_table(memory_names, base_memory, pr_memory, fmt_bytes) + if memory_names + else "n/a" + ) + report.append("") + report.append( + "Base = target branch, PR = this branch. Negative delta = improvement. " + "✅ improvement ≥ 1%, ⚠️ regression > 5%. " + '"n/a" base entries mean the target branch has no benchmark suite yet.' + ) + + text = "\n".join(report) + with open(args.output, "w") as f: + f.write(text) + print(text) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/memory_bench.py b/benchmarks/memory_bench.py new file mode 100644 index 0000000..bbfae88 --- /dev/null +++ b/benchmarks/memory_bench.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Measure peak memory usage (via tracemalloc) for ACL-python's core data +structures, using the same workloads as benchmarks/test_time.py. + +Usage: + python benchmarks/memory_bench.py --output result.json +""" + +import argparse +import json +import os +import sys +import tracemalloc + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import workloads # noqa: E402 + + +def measure(build_fn, run_fn): + args = build_fn() + tracemalloc.start() + before, _ = tracemalloc.get_traced_memory() + result = run_fn(*args) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + del result + return peak - before + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + args = parser.parse_args() + + results = [] + for name, build_fn, run_fn in workloads.BENCHMARKS: + peak_bytes = measure(build_fn, run_fn) + results.append({"name": name, "peak_bytes": peak_bytes}) + print(f"{name}: {peak_bytes / 1024:.1f} KiB") + + with open(args.output, "w") as f: + json.dump({"benchmarks": results}, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/test_time.py b/benchmarks/test_time.py new file mode 100644 index 0000000..82e6f6f --- /dev/null +++ b/benchmarks/test_time.py @@ -0,0 +1,24 @@ +"""Execution-time benchmarks, run via pytest-benchmark. + +Usage: + pytest benchmarks/test_time.py --benchmark-json=result.json +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import workloads # noqa: E402 + + +@pytest.mark.parametrize( + "name,build_fn,run_fn", + workloads.BENCHMARKS, + ids=[b[0] for b in workloads.BENCHMARKS], +) +def test_benchmark(benchmark, name, build_fn, run_fn): + args = build_fn() + benchmark.pedantic(run_fn, args=args, rounds=3, warmup_rounds=1) diff --git a/benchmarks/workloads.py b/benchmarks/workloads.py new file mode 100644 index 0000000..4a29a66 --- /dev/null +++ b/benchmarks/workloads.py @@ -0,0 +1,190 @@ +"""Deterministic workloads shared by the time (pytest-benchmark) and memory +(tracemalloc) benchmarks, so both measure exactly the same operations. +""" + +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import segtree as segtree_mod +import lazysegtree as lazysegtree_mod +import dsu as dsu_mod +import maxflow as maxflow_mod +import mincostflow as mincostflow_mod +import convolution as convolution_mod +import scc as scc_mod + +MOD = 998244353 + + +def _rng(seed): + return random.Random(seed) + + +# --------------------------------------------------------------------------- +# segtree: point-set / range-max on a large array +# --------------------------------------------------------------------------- +def build_segtree_workload(): + r = _rng(1) + n, q = 50_000, 50_000 + values = [r.randint(1, 10**9) for _ in range(n)] + queries = [] + for _ in range(q): + if r.random() < 0.5: + queries.append(("set", r.randrange(n), r.randint(1, 10**9))) + else: + l = r.randrange(n) + queries.append(("prod", l, r.randrange(l, n + 1))) + return values, queries + + +def run_segtree(values, queries): + seg = segtree_mod.segtree(list(values), max, -1) + for op, a, b in queries: + if op == "set": + seg.set(a, b) + else: + seg.prod(a, b) + return seg + + +# --------------------------------------------------------------------------- +# lazysegtree: range-add / range-min +# --------------------------------------------------------------------------- +def build_lazysegtree_workload(): + r = _rng(2) + n, q = 50_000, 50_000 + values = [0] * n + queries = [] + for _ in range(q): + l = r.randrange(n) + rr = r.randrange(l, n + 1) + if r.random() < 0.5: + queries.append(("apply", l, rr, r.randint(-1000, 1000))) + else: + queries.append(("prod", l, rr)) + return values, queries + + +def run_lazysegtree(values, queries): + inf = float("inf") + seg = lazysegtree_mod.lazy_segtree( + list(values), min, inf, lambda f, x: f + x, lambda f, g: f + g, 0 + ) + for q in queries: + if q[0] == "apply": + seg.apply(q[1], q[2], q[3]) + else: + seg.prod(q[1], q[2]) + return seg + + +# --------------------------------------------------------------------------- +# dsu: merge / same on a large number of elements +# --------------------------------------------------------------------------- +def build_dsu_workload(): + r = _rng(3) + n = 100_000 + ops = [] + for _ in range(n): + ops.append(("merge", r.randrange(n), r.randrange(n))) + for _ in range(n): + ops.append(("same", r.randrange(n), r.randrange(n))) + return n, ops + + +def run_dsu(n, ops): + d = dsu_mod.dsu(n) + for op, a, b in ops: + if op == "merge": + d.merge(a, b) + else: + d.same(a, b) + return d + + +# --------------------------------------------------------------------------- +# maxflow: Dinic's algorithm on a random DAG +# --------------------------------------------------------------------------- +def build_maxflow_workload(): + r = _rng(4) + n, m = 2_000, 8_000 + edges = [] + for _ in range(m): + u = r.randrange(n - 1) + v = r.randrange(u + 1, n) + edges.append((u, v, r.randint(1, 1000))) + return n, edges + + +def run_maxflow(n, edges): + g = maxflow_mod.mf_graph(n) + for u, v, cap in edges: + g.add_edge(u, v, cap) + g.flow(0, n - 1) + return g + + +# --------------------------------------------------------------------------- +# mincostflow: successive shortest paths on a smaller random DAG +# --------------------------------------------------------------------------- +def build_mincostflow_workload(): + r = _rng(5) + n, m = 300, 1_500 + edges = [] + for _ in range(m): + u = r.randrange(n - 1) + v = r.randrange(u + 1, n) + edges.append((u, v, r.randint(1, 20), r.randint(1, 100))) + return n, edges + + +def run_mincostflow(n, edges): + g = mincostflow_mod.mcf_graph(n) + for u, v, cap, cost in edges: + g.add_edge(u, v, cap, cost) + g.flow(0, n - 1) + return g + + +# --------------------------------------------------------------------------- +# convolution: NTT-based convolution of two large arrays +# --------------------------------------------------------------------------- +def build_convolution_workload(): + r = _rng(6) + n = 1 << 16 + a = [r.randrange(MOD) for _ in range(n)] + b = [r.randrange(MOD) for _ in range(n)] + return a, b + + +def run_convolution(a, b): + fft = convolution_mod.FFT(MOD) + return fft.convolution(a, b) + + +# --------------------------------------------------------------------------- +# scc: Tarjan's algorithm on a large random directed graph +# --------------------------------------------------------------------------- +def build_scc_workload(): + r = _rng(7) + n, m = 50_000, 150_000 + edges = [(r.randrange(n), r.randrange(n)) for _ in range(m)] + return n, edges + + +def run_scc(n, edges): + return scc_mod.scc(n, edges) + + +BENCHMARKS = [ + ("segtree", build_segtree_workload, run_segtree), + ("lazysegtree", build_lazysegtree_workload, run_lazysegtree), + ("dsu", build_dsu_workload, run_dsu), + ("maxflow", build_maxflow_workload, run_maxflow), + ("mincostflow", build_mincostflow_workload, run_mincostflow), + ("convolution", build_convolution_workload, run_convolution), + ("scc", build_scc_workload, run_scc), +] diff --git a/pyproject.toml b/pyproject.toml index 032ec31..a497ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,9 @@ classifiers = [ "Topic :: Scientific/Engineering :: Mathematics", ] +[project.optional-dependencies] +dev = ["pytest", "pytest-cov", "pytest-benchmark", "flake8", "black"] + [project.urls] Homepage = "https://github.com/shakayami/ACL-for-python" Repository = "https://github.com/shakayami/ACL-for-python"