diff --git a/benchmarks/test_regalloc/__init__.py b/benchmarks/test_regalloc/__init__.py new file mode 100644 index 0000000..287906d --- /dev/null +++ b/benchmarks/test_regalloc/__init__.py @@ -0,0 +1,20 @@ +# flake8: noqa +""" +Register Allocation Benchmarks for ScratchV. + +This package contains three benchmark suites for the linear scan +register allocator (``LinearScanAllocator``): + +1. **bench1_simple** — Simple arithmetic (3-5 vregs, no spills) +2. **bench2_dense** — Dense computation (20+ vregs, triggers spilling) +3. **bench3_cnn** — CNN model integration (operations from ``models/graph/cnn.onnx``) + +Each suite measures allocation time, spill count, peak register pressure, +and validates output correctness. + +Usage:: + + python -m benchmarks.test_regalloc.run_all +""" + +from __future__ import annotations diff --git a/benchmarks/test_regalloc/bench_cnn.py b/benchmarks/test_regalloc/bench_cnn.py new file mode 100644 index 0000000..0f34eb4 --- /dev/null +++ b/benchmarks/test_regalloc/bench_cnn.py @@ -0,0 +1,384 @@ +# flake8: noqa +"""Benchmark 3 — CNN model integration with emulator verification. + +Compiles the CNN model through the full ScratchV pipeline, runs the +linear scan allocator, validates the output assembly, and optionally +verifies execution via the RV32 emulator. +""" + +import argparse +import os +import re +import statistics +import sys +import time + +from scratchv.backend.regalloc_linear import ( + LinearScanAllocator, + block_from_machine_instrs, + _INT_REGS, +) +from scratchv.backend.register_alloc import RegisterAllocator +from scratchv.standalone.compare_codegen import count_riscv_instrs + + +# --------------------------------------------------------------------------- +# Compilation helpers +# --------------------------------------------------------------------------- + + +def _compile_onnx(onnx_path: str) -> tuple: + """ONNX → IR → MachineInstr (with virtual registers). + + Returns ``(machine_instrs, ir_inst_count, vreg_count)``. + """ + from scratchv.frontend.onnx_parser import ONNXParser + from scratchv.backend.instruction_select import InstructionSelector + from scratchv.optimizer.constant_folding import ConstantFolder + from scratchv.optimizer.dead_code import DeadCodeEliminator + + program = ONNXParser().parse(onnx_path) + ir_count = sum( + 1 for f in program.functions for bb in f.blocks for _ in bb.instructions + ) + ConstantFolder(program).run() + DeadCodeEliminator(program).run() + machine = InstructionSelector(program).run() + + vregs: set[str] = set() + for mi in machine: + for op in (mi.dst, mi.src1, mi.src2): + if op and getattr(op, "kind", None) == "vreg": + vregs.add(str(op.value)) + return machine, ir_count, len(vregs) + + +# --------------------------------------------------------------------------- +# Assembly validation +# --------------------------------------------------------------------------- +from .bench_utils import _KNOWN_OPS + + +def _validate_asm(asm: str) -> list[str]: + """Check no unresolved vregs, valid opcodes.""" + errors: list[str] = [] + for lineno, line in enumerate(asm.splitlines(), start=1): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + content = stripped.lstrip() + if content.endswith(":") or not content: + continue + if "#" in content: + content = content[: content.index("#")].strip() + parts = content.split() + if not parts: + continue + if parts[0] not in _KNOWN_OPS: + errors.append(f"Line {lineno}: unknown opcode '{parts[0]}'") + for token in parts: + if token.startswith("v") and token[1:].isdigit(): + errors.append(f"Line {lineno}: unresolved vreg '{token}'") + return errors + + +# --------------------------------------------------------------------------- +# Emulator verification +# --------------------------------------------------------------------------- + + +def _run_emulator(cnn_path: str) -> dict: + """Compile *cnn_path* via standalone pipeline, run through RV32Emulator.""" + try: + from scratchv.standalone.onnx_to_riscv_standalone import ( + ONNXModel, + MemoryPlan, + CNNRISCVGenerator, + ) + from scratchv.simulator.rv32_emulator import RV32Emulator + except ImportError as e: + return {"passed": False, "error": f"import error: {e}"} + + try: + model = ONNXModel.from_file(cnn_path) + memory = MemoryPlan() + memory.layout_weights(model.initializers) + if model.inputs: + inp = model.inputs[0] + el = 1 + for d in model.get_shape(inp.name): + el *= d + memory.alloc_workspace(inp.name, el) + + generator = CNNRISCVGenerator(model, memory) + code_bytes = generator.generate() + + emu = RV32Emulator() + emu.load_code(code_bytes) + emu.run(max_instr=100_000) + return {"passed": True, "error": ""} + except Exception as exc: + return {"passed": False, "error": str(exc)[:120]} + + +# --------------------------------------------------------------------------- +# LLVM comparison +# --------------------------------------------------------------------------- + +# Instruction category buckets used to compare opcode mixes between the +# ScratchV backend and the LLVM backend. `sd`/`ld` are ABI stack +# save/restore pairs — the classic LLVM frame-management cost. +from .bench_utils import ( + _CAT_ALU, + _CAT_LOAD, + _CAT_STORE, + _CAT_BRANCH, + _CAT_MUL, + _CAT_STACK, +) +from .bench_utils import _op_categories + +# RV64 ABI callee-saved registers — `sd`/`ld` to these at sp offsets are +# prologue/epilogue frame save/restore, not spills. +from .bench_utils import _CALLEE_SAVED + + +def _llvm_spill_stats(asm: str) -> dict: + """Approximate LLVM spill/frame stats from RISC-V assembly. + + libLLVM codegen does not expose regalloc pass statistics, so spill + counts are inferred from sp-based memory accesses: + - ``sd``/``ld`` to callee-saved regs → ABI frame save/restore + - 4-byte ``sw``/``lw``/``fsw``/``flw`` → spilled values + (each spill site emits a store + reload pair) + + Returns ``{llvm_spill_slots, llvm_frame_save, llvm_frame_restore}``. + """ + frame_save = frame_restore = spill4 = 0 + for line in asm.splitlines(): + line = line.strip() + if not line: + continue + m = re.match(r"^(sd|ld|sw|lw|fsw|flw)\s+([^,]+),\s*(-?\d+)\(sp\)", line) + if not m: + continue + op, reg, _ = m.group(1), m.group(2).strip(), int(m.group(3)) + if op in ("sd", "ld") and reg in _CALLEE_SAVED: + if op == "sd": + frame_save += 1 + else: + frame_restore += 1 + elif op in ("sw", "lw", "fsw", "flw"): + spill4 += 1 + return { + "llvm_spill_slots": spill4 // 2, + "llvm_frame_save": frame_save, + "llvm_frame_restore": frame_restore, + } + + +def _llvm_compare(cnn_path: str) -> dict: + """Compile *cnn_path* via LLVM (O2) and return comparison stats. + + Reuses the libLLVM pipeline from ``compare_codegen.py``: ONNX → + LLVM IR → RISC-V assembly at both RV64IM and RV64FD feature sets. + """ + from scratchv.standalone.compare_codegen import _load_llvm, llvm_ir_to_riscv + from scratchv.standalone.onnx_to_llvm_standalone import convert_onnx_to_llvm + from .bench_utils import llvmlite_ir_to_riscv + + # lib = _load_llvm() + ir = convert_onnx_to_llvm(cnn_path) + im_cnt, im_asm, _ = llvmlite_ir_to_riscv(ir, "+m", 2) + fd_cnt, fd_asm, fd_cats = llvmlite_ir_to_riscv(ir, "+m,+f,+d", 2) + + result = { + "llvm_im_instrs": im_cnt, + "llvm_fd_instrs": fd_cnt, + "llvm_fd_cats": fd_cats, + "llvm_fd_cat_buckets": _op_categories(fd_cats), + "_llvm_fd_asm": fd_asm, + "_llvm_im_asm": im_asm, + } + result.update(_llvm_spill_stats(fd_asm)) + return result + + +# --------------------------------------------------------------------------- +# Benchmark +# --------------------------------------------------------------------------- + + +def bench_allocate(cnn_path: str, phys_regs: list[str], repeats: int = 30) -> dict: + """Full CNN compilation pipeline with linear scan allocator.""" + machine, ir_count, vreg_total = _compile_onnx(cnn_path) + block = block_from_machine_instrs(machine) + + times = [] + spill_counts = [] + + # Warm up + for _ in range(repeats): + alloc = LinearScanAllocator(phys_regs=phys_regs) + t0 = time.perf_counter() + alloc.allocate(alloc.compute_live_intervals(block)) + t1 = time.perf_counter() + times.append(t1 - t0) + spill_counts.append(len(alloc._spill_slots)) + + # Final run for stable stats + assembly validation + alloc = LinearScanAllocator(phys_regs=phys_regs) + alloc.allocate(alloc.compute_live_intervals(block)) + code = alloc.get_allocated_code(block) + asm_errors = _validate_asm(code) + sv_cnt, sv_cats = count_riscv_instrs(code) + + # Greedy allocator baseline + t0 = time.perf_counter() + greedy = RegisterAllocator(machine, mode="greedy") + greedy_out = greedy.run() + greedy_time = time.perf_counter() - t0 + + return { + "mean_s": statistics.mean(times), + "stdev_s": statistics.stdev(times) if len(times) > 1 else 0, + "vreg_total": vreg_total, + "ir_inst_count": ir_count, + "machine_instrs": len(machine), + "vreg_count": len(alloc.alloc_map), + "reg_spill_count": spill_counts[-1], + "peak_active": alloc.peak_active, + "asm_lines": len(code.splitlines()), + "sv_static_instrs": sv_cnt, + "sv_cats": sv_cats, + "sv_cat_buckets": _op_categories(sv_cats), + "asm_errors": asm_errors, + "asm_valid": len(asm_errors) == 0, + "greedy_time_s": greedy_time, + "greedy_out_instrs": len(greedy_out), + "_report": alloc.report(), + "_alloc": alloc, + } + + +def run_bench( + cnn_path: str, phys_regs: list[str] | None = None, repeats: int = 30 +) -> dict: + """Entry point for the test suite runner.""" + if phys_regs is None: + phys_regs = list(_INT_REGS) + stats = bench_allocate(cnn_path, phys_regs, repeats=repeats) + + # Emulator verification (non-fatal) + emu = _run_emulator(cnn_path) + stats["emu_passed"] = emu["passed"] + stats["emu_error"] = emu.get("error", "") + stats["valid"] = stats["asm_valid"] + + # LLVM comparison (non-fatal) + try: + stats.update(_llvm_compare(cnn_path)) + stats["llvm_available"] = True + except Exception as exc: + stats["llvm_available"] = False + stats["llvm_error"] = str(exc)[:120] + if stats["llvm_available"]: + stats["instr_ratio_fd"] = round( + stats["llvm_fd_instrs"] / max(stats["sv_static_instrs"], 1), 2 + ) + return stats + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark 3 — CNN Model Integration") + parser.add_argument( + "--repeats", type=int, default=30, help="Number of repeat measurements" + ) + parser.add_argument("--cnn-path", default="", help="Path to ONNX model") + + args = parser.parse_args() + + if not args.cnn_path: + args.cnn_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "models", + "graph", + "cnn.onnx", + ) + + phys_regs = list(_INT_REGS) + + print("=" * 60) + print("Benchmark 3 — CNN Model Integration And Comparation With LLVM Backend") + print(f" Model: {os.path.basename(args.cnn_path)}") + print("=" * 60) + + stats = run_bench(args.cnn_path, phys_regs=phys_regs, repeats=args.repeats) + + print( + f"\n{'':>8} {'Mean(ms)':>10} {'Stdev(ms)':>10} " + f"{'Vregs':>6} {'Spills':>7} {'Peak':>6} {'Asm':>5}" + ) + print("-" * 55) + print( + f"{'cnn':>8} {stats['mean_s'] * 1000:>10.3f} " + f"{stats['stdev_s'] * 1000:>10.3f} " + f"{stats['vreg_count']:>6} {stats['reg_spill_count']:>7} " + f"{stats['peak_active']:>6} {stats['asm_lines']:>5}" + ) + + print() + print(stats["_report"]) + print( + f" Greedy baseline: {stats['greedy_time_s'] * 1000:.3f}ms, " + f"{stats['greedy_out_instrs']} instrs" + ) + + if not stats["asm_valid"]: + for e in stats["asm_errors"][:3]: + print(f" ✗ {e}") + if not stats["emu_passed"]: + print(f" Emulator: ✗ {stats['emu_error']}") + else: + print(f" Emulator: ✓ passed") + + # LLVM comparison output + print() + print("-" * 55) + print(" LLVM comparison (O2, same ONNX model)") + print("-" * 55) + + print( + f" ScratchV LinearScan: {stats['sv_static_instrs']} instrs " + f"{stats['sv_cat_buckets']}" + ) + print(f" LLVM RV64IM: {stats['llvm_im_instrs']} instrs") + print( + f" LLVM RV64FD: {stats['llvm_fd_instrs']} instrs " + f"({stats['instr_ratio_fd']}x vs ScratchV) " + f"{stats['llvm_fd_cat_buckets']}" + ) + print( + f" Spill (LLVM approx): {stats['llvm_spill_slots']} slots " + f"(frame save/restore {stats['llvm_frame_save']}/" + f"{stats['llvm_frame_restore']}); " + f"ScratchV (exact): reg_spill_count={stats['reg_spill_count']}" + ) + + asm_ok = "PASS" if stats["asm_valid"] else "FAIL" + print( + f"\n asm_valid={stats['asm_valid']}, " + f"reg_spill_count={stats['reg_spill_count']} [{asm_ok}]" + ) + return 0 if stats["asm_valid"] else 1 + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/benchmarks/test_regalloc/bench_dense.py b/benchmarks/test_regalloc/bench_dense.py new file mode 100644 index 0000000..0be539c --- /dev/null +++ b/benchmarks/test_regalloc/bench_dense.py @@ -0,0 +1,146 @@ +# flake8: noqa +"""Benchmark 2 — Dense computation (30 vregs, triggers spilling). + +Forces the linear scan allocator to spill by providing more virtual +registers than physical registers. +""" + +import argparse +import os +import random +import statistics +import sys +import time + +from scratchv.backend.regalloc_linear import LinearScanAllocator, LsInstruction + + +def _gen_block( + num_insts: int = 80, num_vregs: int = 30, seed: int = 42 +) -> list[LsInstruction]: + """Generate a high-register-pressure block.""" + random.seed(seed) + ops = ["add", "sub", "mul", "and", "or", "xor", "sll", "srl"] + vreg_names = [f"v{i}" for i in range(num_vregs)] + insts = [] + + # Phase 1: define each vreg — creates long live ranges + for i in range(num_vregs): + insts.append( + LsInstruction( + id=i, + opcode="addi", + operands=[vreg_names[i], "zero", str(random.randint(1, 100))], + defines={vreg_names[i]}, + uses=set(), + comment=f"def {vreg_names[i]}", + ) + ) + + # Phase 2: cross-reference dense ops — keeps many vregs live + for i in range(num_vregs, num_insts): + src1 = random.choice(vreg_names) + src2 = random.choice(vreg_names) + dst = random.choice(vreg_names) + insts.append( + LsInstruction( + id=i, + opcode=random.choice(ops), + operands=[dst, src1, src2], + defines={dst}, + uses={src1, src2}, + comment=f"dense op {i}", + ) + ) + return insts + + +def bench_allocate( + block: list[LsInstruction], phys_regs: list[str], repeats: int = 30 +) -> dict: + """Benchmark the full allocation pipeline under register pressure.""" + times = [] + spill_counts = [] + + for _ in range(repeats): + alloc = LinearScanAllocator(phys_regs=phys_regs) + t0 = time.perf_counter() + alloc.allocate(alloc.compute_live_intervals(block)) + t1 = time.perf_counter() + times.append(t1 - t0) + spill_counts.append(len(alloc._spill_slots)) + + # Final run for stable stats + alloc = LinearScanAllocator(phys_regs=phys_regs) + alloc.allocate(alloc.compute_live_intervals(block)) + code = alloc.get_allocated_code(block) + reloads = sum( + 1 for ln in code.splitlines() if ln.strip().startswith("lw ") and "reload" in ln + ) + + return { + "mean_s": statistics.mean(times), + "stdev_s": statistics.stdev(times) if len(times) > 1 else 0, + "vreg_count": len(alloc.alloc_map), + "spills": spill_counts[-1], + "reg_spill_count": spill_counts[-1], + "peak_active": alloc.peak_active, + "asm_lines": len(code.splitlines()), + "reloads": reloads, + "_report": alloc.report(), + "_alloc": alloc, + } + + +def run_bench(phys_regs: list[str] | None = None, repeats: int = 30) -> dict: + """Entry point for the test suite runner.""" + if phys_regs is None: + phys_regs = [f"r{i}" for i in range(5)] + block = _gen_block(num_insts=80, num_vregs=30) + stats = bench_allocate(block, phys_regs, repeats=repeats) + stats["valid"] = stats["spills"] > 0 + return stats + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark 2 — Dense Computation (spills)" + ) + parser.add_argument( + "--repeats", type=int, default=30, help="Number of repeat measurements" + ) + args = parser.parse_args() + + phys_regs = [f"r{i}" for i in range(5)] + + print("=" * 60) + print("Benchmark 2 — Dense Computation (30 vregs / 5 phys regs)") + print("=" * 60) + + stats = run_bench(phys_regs=phys_regs, repeats=args.repeats) + + print( + f"\n{'':>8} {'Mean(ms)':>10} {'Stdev(ms)':>10} " + f"{'Vregs':>6} {'Spills':>7} {'Peak':>6} {'Reloads':>8} " + f"{'Asm':>5}" + ) + print("-" * 65) + print( + f"{'dense':>8} {stats['mean_s'] * 1000:>10.3f} " + f"{stats['stdev_s'] * 1000:>10.3f} " + f"{stats['vreg_count']:>6} {stats['spills']:>7} " + f"{stats['peak_active']:>6} {stats['reloads']:>8} " + f"{stats['asm_lines']:>5}" + ) + + print() + print(stats["_report"]) + + spills = stats["spills"] + ok = "PASS" if spills > 0 else "FAIL (expected spills)" + print(f"\n reg_spill_count={spills} [{ok}]") + return 0 if spills > 0 else 1 + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/benchmarks/test_regalloc/bench_regalloc_linear.py b/benchmarks/test_regalloc/bench_regalloc_linear.py new file mode 100644 index 0000000..003f21f --- /dev/null +++ b/benchmarks/test_regalloc/bench_regalloc_linear.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Run all 3 register allocation benchmarks and produce a report.""" + +import argparse +import datetime +import json +import os +import sys +import time + + +from benchmarks.test_regalloc import bench_simple, bench_dense, bench_cnn + + +# --------------------------------------------------------------------------- +# Report helpers +# --------------------------------------------------------------------------- + + +def _make_html(results: dict, total_time: float) -> str: + """Generate an HTML report.""" + rows = "" + for name, r in results.items(): + if not isinstance(r, dict): + continue + v = "✓" if r.get("valid", True) else "✗" + c = "#22863a" if r.get("valid", True) else "#cb2431" + ms = f"{r.get('mean_s', 0) * 1000:.3f}" + sd = f"{r.get('stdev_s', 0) * 1000:.3f}" + rows += ( + f"{name}{ms}{sd}" + f"{r.get('vreg_count', '-')}" + f"{r.get('reg_spill_count', r.get('spills', '-'))}" + f"{r.get('peak_active', '-')}" + f"{r.get('reloads', '-')}" + f"{r.get('asm_lines', '-')}" + f"{v}\n" + ) + + return f""" + + + +Register Allocation Benchmark Report + + + +

Register Allocation Benchmark Report

+

Generated: {datetime.datetime.now().isoformat()} | Total: {total_time * 1000:.1f}ms

+ + + +{rows} +
BenchmarkMean(ms)Std(ms)VregsSpillsPeakReloadsAsmValid
+ +""" + + +def _make_markdown(results: dict) -> str: + """Generate a Markdown report.""" + lines = [ + "# Register Allocation Benchmark Report", + "", + f"**Generated**: {datetime.datetime.now().isoformat()}", + "", + "| Benchmark | Mean(ms) | Std(ms) | Vregs | Spills | Peak | " + "Reloads | Asm | Valid |", + "|-----------|----------|---------|-------|--------|------|" + "---------|-----|-------|", + ] + for name, r in results.items(): + if not isinstance(r, dict): + continue + ms = f"{r.get('mean_s', 0) * 1000:.3f}" + sd = f"{r.get('stdev_s', 0) * 1000:.3f}" + v = "✓" if r.get("valid", True) else "✗" + lines.append( + f"| {name} | {ms} | {sd} | {r.get('vreg_count', '-')} | " + f"{r.get('reg_spill_count', r.get('spills', '-'))} | " + f"{r.get('peak_active', '-')} | " + f"{r.get('reloads', '-')} | {r.get('asm_lines', '-')} | " + f"{v} |" + ) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser(description="Register Allocation Benchmark Suite") + parser.add_argument("--repeats", type=int, default=30) + parser.add_argument("--output-json", default="") + parser.add_argument("--output-html", default="") + parser.add_argument("--output-md", default="") + args = parser.parse_args() + + print("=" * 60) + print(" ScratchV — Register Allocation Benchmark Suite") + print("=" * 60) + + t0 = time.perf_counter() + results: dict = {} + + # Benchmark 1 — Simple (no-spill) + r1 = bench_simple.run_bench(repeats=args.repeats) + results["1. Simple Arithmetic"] = r1 + print( + f" 1. Simple: reg_spill_count={r1['reg_spill_count']}, " + f"mean={r1['mean_s'] * 1000:.3f}ms " + f"{'✓' if r1.get('valid') else '✗'}" + ) + + # Benchmark 2 — Dense (spill) + r2 = bench_dense.run_bench(repeats=args.repeats) + results["2. Dense Computation"] = r2 + print( + f" 2. Dense: reg_spill_count={r2['reg_spill_count']}, " + f"mean={r2['mean_s'] * 1000:.3f}ms " + f"{'✓' if r2.get('valid') else '✗'}" + ) + + # Benchmark 3 — CNN Integration And Comparation With LLVM + cnn_default = "models/graph/cnn.onnx" + r3 = bench_cnn.run_bench(cnn_path=cnn_default, repeats=args.repeats) + results["3. CNN Integration"] = r3 + print( + f" 3. CNN: reg_spill_count={r3['reg_spill_count']}, " + f"mean={r3['mean_s'] * 1000:.3f}ms " + f"{'✓' if r3.get('valid') else '✗'}" + ) + + total_time = time.perf_counter() - t0 + + # Summary + all_ok = all(r.get("valid", True) for r in results.values() if isinstance(r, dict)) + print(f"\n Total: {total_time * 1000:.1f}ms {'PASS' if all_ok else 'HAD ERRORS'}") + + # Reports + if args.output_json: + report = { + "timestamp": datetime.datetime.now().isoformat(), + "total_time_s": total_time, + "repeats": args.repeats, + "results": { + name: { + k: v + for k, v in r.items() + if not k.startswith("_") and k != "asm_errors" + } + for name, r in results.items() + if isinstance(r, dict) + }, + } + with open(args.output_json, "w") as f: + json.dump(report, f, indent=2) + print(f"\n JSON report: {args.output_json}") + + if args.output_html: + with open(args.output_html, "w") as f: + f.write(_make_html(results, total_time)) + print(f" HTML report: {args.output_html}") + + if args.output_md: + with open(args.output_md, "w") as f: + f.write(_make_markdown(results)) + print(f" Markdown: {args.output_md}") + + return 0 if all_ok else 1 + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/benchmarks/test_regalloc/bench_simple.py b/benchmarks/test_regalloc/bench_simple.py new file mode 100644 index 0000000..a3e02e8 --- /dev/null +++ b/benchmarks/test_regalloc/bench_simple.py @@ -0,0 +1,136 @@ +# flake8: noqa +"""Benchmark 1 — Simple arithmetic (3-5 vregs, no spilling). + +Verifies that the linear scan allocator produces zero spills when +physical registers are plentiful. +""" + +import argparse +import os +import random +import statistics +import sys +import time + +from scratchv.backend.regalloc_linear import LinearScanAllocator, LsInstruction + + +def _gen_block( + num_insts: int = 10, num_vregs: int = 5, seed: int = 42 +) -> list[LsInstruction]: + """Generate a basic block with simple arithmetic using few vregs.""" + random.seed(seed) + ops = ["add", "sub", "mul", "and", "or"] + vreg_names = [f"v{i}" for i in range(num_vregs)] + insts = [] + + for i in range(num_insts): + if i < num_vregs: + dst = vreg_names[i] + pool = vreg_names[: max(i, 1)] + src1 = random.choice(pool) + src2 = random.choice(pool) + insts.append( + LsInstruction( + id=i, + opcode=random.choice(ops), + operands=[dst, src1, src2], + defines={dst}, + uses={src1, src2}, + ) + ) + else: + dst = random.choice(vreg_names) + src1 = random.choice(vreg_names) + src2 = random.choice(vreg_names) + insts.append( + LsInstruction( + id=i, + opcode=random.choice(ops), + operands=[dst, src1, src2], + defines={dst}, + uses={src1, src2} - {dst}, + ) + ) + return insts + + +def bench_allocate( + block: list[LsInstruction], phys_regs: list[str], repeats: int = 50 +) -> dict: + """Benchmark the full allocation pipeline.""" + times = [] + spill_counts = [] + + for _ in range(repeats): + alloc = LinearScanAllocator(phys_regs=phys_regs) + t0 = time.perf_counter() + alloc.allocate(alloc.compute_live_intervals(block)) + t1 = time.perf_counter() + times.append(t1 - t0) + spill_counts.append(len(alloc._spill_slots)) + + # One final run for stable stats + alloc = LinearScanAllocator(phys_regs=phys_regs) + alloc.allocate(alloc.compute_live_intervals(block)) + code = alloc.get_allocated_code(block) + + return { + "mean_s": statistics.mean(times), + "stdev_s": statistics.stdev(times) if len(times) > 1 else 0, + "vreg_count": len(alloc.alloc_map), + "spills": spill_counts[-1], + "reg_spill_count": spill_counts[-1], + "peak_active": alloc.peak_active, + "asm_lines": len(code.splitlines()), + "_report": alloc.report(), + "_alloc": alloc, + } + + +def run_bench(phys_regs: list[str] | None = None, repeats: int = 50) -> dict: + """Entry point for the test suite runner.""" + if phys_regs is None: + phys_regs = [f"r{i}" for i in range(8)] + block = _gen_block(num_insts=10, num_vregs=5) + stats = bench_allocate(block, phys_regs, repeats=repeats) + stats["valid"] = stats["spills"] == 0 + return stats + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark 1 — Simple Arithmetic (no-spill)" + ) + parser.add_argument( + "--repeats", type=int, default=50, help="Number of repeat measurements" + ) + args = parser.parse_args() + + phys_regs = [f"r{i}" for i in range(8)] + + print("=" * 60) + print("Benchmark 1 — Simple Arithmetic (5 vregs / 8 phys regs)") + print("=" * 60) + + stats = run_bench(phys_regs=phys_regs, repeats=args.repeats) + + print(f"\n{'':>8} {'Mean(ms)':>10} {'Stdev(ms)':>10} {'Vregs':>6} {'Spills':>7}") + print("-" * 50) + print( + f"{'simple':>8} {stats['mean_s'] * 1000:>10.3f} " + f"{stats['stdev_s'] * 1000:>10.3f} " + f"{stats['vreg_count']:>6} {stats['spills']:>7}" + ) + + print() + print(stats["_report"]) + + spills = stats["spills"] + ok = "PASS" if spills == 0 else "FAIL" + print(f"\n reg_spill_count={spills} [{ok}]") + return 0 if spills == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/benchmarks/test_regalloc/bench_utils.py b/benchmarks/test_regalloc/bench_utils.py new file mode 100644 index 0000000..da7e631 --- /dev/null +++ b/benchmarks/test_regalloc/bench_utils.py @@ -0,0 +1,263 @@ +# flake8: noqa +"""Common used benchmark utils for all benchmarks""" + +from tabulate import tabulate + + +# Common used constants +_CALLEE_SAVED = { + "ra", + "fp", + "s0", + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + "s9", + "s10", + "s11", + "fs0", + "fs1", + "fs2", + "fs3", + "fs4", + "fs5", + "fs6", + "fs7", + "fs8", + "fs9", + "fs10", + "fs11", +} + +_KNOWN_OPS = { + "add", + "addi", + "sub", + "mul", + "div", + "rem", + "and", + "andi", + "or", + "ori", + "xor", + "xori", + "sll", + "slli", + "srl", + "srli", + "sra", + "srai", + "slt", + "slti", + "sltu", + "sltiu", + "lw", + "lh", + "lb", + "lbu", + "lhu", + "sw", + "sh", + "sb", + "beq", + "bne", + "blt", + "bge", + "bltu", + "bgeu", + "jal", + "jalr", + "auipc", + "lui", + "li", + "mv", + "nop", + "ret", + "bnez", + "j", + "max", + ".label", + ".text", + ".data", + ".global", + ".type", + "flw", + "fsw", + "fadd.s", + "fsub.s", + "fmul.s", + "fdiv.s", + "feq.s", + "flt.s", + "fle.s", + "fcvt.w.s", + "fcvt.s.w", +} + +# Instruction category buckets used to compare opcode mixes between the +# ScratchV backend and the LLVM backend. `sd`/`ld` are ABI stack +# save/restore pairs — the classic LLVM frame-management cost. +_CAT_ALU = { + "add", + "addi", + "sub", + "sll", + "slli", + "srl", + "srli", + "sra", + "srai", + "and", + "andi", + "or", + "ori", + "xor", + "xori", + "slt", + "slti", + "sltu", + "sltiu", + "lui", + "auipc", +} +_CAT_LOAD = {"lw", "lh", "lb", "lbu", "lhu", "flw"} +_CAT_STORE = {"sw", "sh", "sb", "fsw"} +_CAT_BRANCH = {"beq", "bne", "blt", "bge", "bltu", "bgeu", "jal", "jalr", "j"} +_CAT_MUL = {"mul", "mulh", "mulhu", "mulhsu"} +_CAT_STACK = {"sd", "ld"} + + +def _op_categories(cats: dict[str, int]) -> dict[str, int]: + """Group raw opcode counts into instruction categories.""" + out = { + "ALU": 0, + "Load": 0, + "Store": 0, + "Branch": 0, + "Mul": 0, + "Stack": 0, + "Other": 0, + } + for op, n in cats.items(): + if op in _CAT_STACK: + out["Stack"] += n + elif op in _CAT_ALU: + out["ALU"] += n + elif op in _CAT_LOAD: + out["Load"] += n + elif op in _CAT_STORE: + out["Store"] += n + elif op in _CAT_BRANCH: + out["Branch"] += n + elif op in _CAT_MUL: + out["Mul"] += n + else: + out["Other"] += n + return out + + +# RV64 ABI callee-saved registers — `sd`/`ld` to these at sp offsets are +# prologue/epilogue frame save/restore, not spills. +_CALLEE_SAVED = { + "ra", + "fp", + "s0", + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + "s9", + "s10", + "s11", + "fs0", + "fs1", + "fs2", + "fs3", + "fs4", + "fs5", + "fs6", + "fs7", + "fs8", + "fs9", + "fs10", + "fs11", +} + + +_llvmlite_ready = False + + +def llvmlite_ir_to_riscv( + ir_text: str, + features: str = "", + opt_level: int = 2, + output_path: str = "", +) -> tuple[int, str, dict[str, int]]: + """Compile LLVM IR → RISC-V assembly via llvmlite. + + Drop-in replacement for ``compare_codegen.llvm_ir_to_riscv``, using + llvmlite's Python bindings instead of ctypes. No ``lib`` parameter is + needed — llvmlite manages the LLVM library internally. + + Attempts to load the system ``libLLVM-20.so`` so that codegen output is + byte-identical to the existing ctypes path. Falls back to llvmlite's + bundled LLVM if the system library is not available. + + Args: + ir_text: LLVM IR source text. + features: Target features, e.g. ``""`` or ``"+f,+d"``. + opt_level: 0=None, 1=Less, 2=Default, 3=Aggressive. + output_path: If set, save assembly to this file. + + Returns: + ``(instruction_count, assembly_text, opcode_breakdown)``. + """ + global _llvmlite_ready + if not _llvmlite_ready: + from llvmlite import binding as llvm + + # Attempt to load the system libLLVM so that load_library_permanently + # can resolve any additional LLVM plugins from the system install. + # Note: llvmlite's codegen functions are pre-linked against its own + # bundled LLVM, so minor output differences (+2 instrs) from LLVM + # version evolution are expected. + try: + llvm.load_library_permanently("libLLVM-20.so") + except Exception: + pass + llvm.initialize_all_targets() + llvm.initialize_all_asmprinters() + _llvmlite_ready = True + + from llvmlite import binding as llvm + from scratchv.standalone.compare_codegen import count_riscv_instrs + + mod = llvm.parse_assembly(ir_text) + + target = llvm.Target.from_triple("riscv64-unknown-elf") + tm = target.create_target_machine( + cpu="generic-rv64", + features=features, + opt=opt_level, + reloc="default", + codemodel="default", + ) + + asm = tm.emit_assembly(mod) + + if output_path: + with open(output_path, "w") as f: + f.write(asm) + + count, cats = count_riscv_instrs(asm) + return count, asm, cats diff --git a/benchmarks/test_regalloc/regalloc.md b/benchmarks/test_regalloc/regalloc.md new file mode 100644 index 0000000..730e4dc --- /dev/null +++ b/benchmarks/test_regalloc/regalloc.md @@ -0,0 +1,295 @@ +# 寄存器分配 Benchmark 设计文档 + +## 1. 概述 + +对 ScratchV 的**线性扫描寄存器分配器**（`LinearScanAllocator`）进行正确性与性能评估，并在 CNN 路径上与 LLVM 后端做指令数及寄存器溢出的交叉对比。 + +**设计目标**： + +- **正确性**：验证无溢出 / 有溢出两种场景的分配结果合法（无未解析 vreg、有效 opcode） +- **性能**：测量分配耗时（均值 / 标准差）、活跃区间峰值压力 +- **可对比**：每项输出统一的 `reg_spill_count` 指标，支持回归对比和筛选 +- **跨后端对比**：同一 ONNX 模型经 ScratchV 和 LLVM 两条路径编译，对比静态指令数、opcode 类别分布、溢出/帧操作数量 + +入口：`benchmarks/test_regalloc/run_benchmark.py` +单文件运行：直接执行 `bench_simple.py` / `bench_dense.py` / `bench_cnn.py` + +--- + +## 2. 架构 + +``` +benchmarks/test_regalloc/ +├── __init__.py +├── bench_utils.py 共享常量（opcode 分类、callee-saved 集合）、llvmlite IR→RISC-V +├── bench_simple.py Benchmark 1 — 无溢出正确性 +├── bench_dense.py Benchmark 2 — 溢出正确性 +├── bench_cnn.py Benchmark 3 — CNN 集成 + LLVM 对比 +└── run_benchmark.py 运行器：汇总 3 路输出 → JSON / HTML / Markdown 报告 +``` + +### 2.1 统一接口 + +每个 benchmark 文件导出： + +```python +def run_bench(...) -> dict: + """返回统一结构的统计 dict，必需键见 §4。""" +``` + +运行器遍历三个 `run_bench()`，收集 dict 生成报告。 + +### 2.2 数据流 + +``` + ┌──────────────────┐ + │ run_benchmark.py │ + └──────┬───────────┘ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ + bench_simple bench_dense bench_cnn + run_bench() run_bench() run_bench() + │ │ │ + ▼ ▼ ▼ + LinearScanAllocator ... LinearScanAllocator + .allocate() .allocate() + .report() / to_dict() .report() / to_dict() + │ │ │ │ + ▼ ▼ ▼ ▼ + {dict} {dict} {dict} {LLVM dict} + │ │ │ │ + └──────────────┴─────────┴─────────┘ + │ + ▼ + JSON / HTML / MD 报告 +``` + +--- + +## 3. 三项 Benchmark + +### 3.1 Benchmark 1 — 简单算术（无溢出） + +**文件**：`bench_simple.py` +**目的**：验证物理寄存器充足时分配器产生零溢出。 + +| 参数 | 值 | +|------|-----| +| 虚拟寄存器 | 5 个（`v0`–`v4`） | +| 物理寄存器池 | 8 个（`r0`–`r7`） | +| 指令数 | 10 条（随机 add/sub/mul/and/or） | +| 块生成 | `_gen_block(num_insts=10, num_vregs=5)` | +| 断言 | `reg_spill_count == 0` → `valid = True` | + + + +### 3.2 Benchmark 2 — 密集计算（触发溢出） + +**文件**：`bench_dense.py` +**目的**：人为制造高寄存器压力，迫使分配器溢出，验证溢出代码生成正确。 + +| 参数 | 值 | +|------|-----| +| 虚拟寄存器 | 30 个（`v0`–`v29`） | +| 物理寄存器池 | 5 个（`r0`–`r4`） | +| 指令数 | 80 条（30 条 define + 50 条交叉引用） | +| 块生成 | 阶段 1: 逐个定义 vreg → 创建长 live range；阶段 2: 随机交叉引用保持活跃 | +| 断言 | `reg_spill_count > 0` → `valid = True` | + + + +### 3.3 Benchmark 3 — CNN 集成 + LLVM 对比 + +**文件**：`bench_cnn.py` +**目的**：通过真实 ONNX 模型编译流水线验证分配器，并与 LLVM 后端做指令数和溢出对比。 + +**ScratchV 编译流水线**： + +``` +ONNXParser → IR Program (17 ops) + → ConstantFolder → DeadCodeEliminator + → InstructionSelector → MachineInstr (57, with vregs) + → block_from_machine_instrs → [LsInstruction] + → LinearScanAllocator.allocate() + → get_allocated_code() → RISC-V 伪指令 + → _validate_asm() # 检查无未解析 vreg、合法 opcode +``` + +**LLVM 对比流水线**： + +``` +convert_onnx_to_llvm(model) → LLVM IR (866K lines, 183MB) + → llvmlite IR → RISC-V asm → 静态指令数 + opcode 分布 + → 汇编启发式 → 溢出 slot / 帧操作 近似统计 +``` + +| 参数 | 值 | +|------|-----| +| 模型 | `models/graph/cnn.onnx`（可 CLI 覆盖） | +| IR 指令 | 17 条（3×conv + 3×relu + 3×maxpool + 2×gemm + sigmoid + 2×reshape） | +| 物理寄存器 | `_INT_REGS`（28 个） | +| ScratchV 输出 | ~57 条伪指令（mv/mul/add/slt/bnez…） | +| LLVM 输出 | ~1099 条（RV64FD O2，真实循环展开） | +| 断言 | `asm_valid == True` | + +> **注意**：ScratchV 侧输出 57 条**伪指令**（conv/maxpool 等语义级操作由仿真器实现），LLVM 侧输出 1099 条**自包含机器指令**（每个 conv 展开为 5 重嵌套循环的完整 RISC-V 指令序列）。`instr_ratio_fd ≈ 23.89x` 反映的是抽象层级差异而非优化能力差距，因此还提供 opcode **类别分布**作为跨层级可比指标。 + +--- + +## 4. 指标规范 + +### 4.1 所有 benchmark 通用键 + +| 键 | 类型 | 来源 | 说明 | +|----|------|------|------| +| `mean_s` | `float` | `perf_counter` 均值 | 单次分配耗时（秒） | +| `stdev_s` | `float` | `stdev` | 耗时标准差 | +| `vreg_count` | `int` | `len(alloc.alloc_map)` | 已分配的虚拟寄存器数 | +| `spills` | `int` | `len(alloc._spill_slots)` | 溢出 slot 数（别名） | +| `reg_spill_count` | `int` | 同上 | **统一溢出指标键**（接口规范） | +| `peak_active` | `int` | `alloc.peak_active` | 峰值同时活跃的物理寄存器数 | +| `asm_lines` | `int` | `len(code.splitlines())` | 汇编输出行数 | +| `valid` | `bool` | 由 `run_bench()` 设置 | 该项是否通过断言 | + +### 4.2 Benchmark 特有键 + +**bench_dense**： + +| 键 | 说明 | +|----|------| +| `reloads` | `lw ... # reload` 注释行数 | + +**bench_cnn (ScratchV 侧)**： + +| 键 | 来源 | 说明 | +|----|------|------| +| `vreg_total` | ONNXParser→isel | 编译流水线中出现的 vreg 总数 | +| `ir_inst_count` | Program 指令计数 | IR 层操作数（=17） | +| `machine_instrs` | `len(machine)` | MachineInstr 数量（=57） | +| `sv_static_instrs` | `count_riscv_instrs()` | 汇编指令数（~46，不含标签/注释） | +| `sv_cats` | `count_riscv_instrs()` | opcode 原始计数 | +| `sv_cat_buckets` | `_op_categories()` | 6 类汇总（ALU/Load/Store/Branch/Mul/Other） | +| `asm_errors` | `_validate_asm()` | 未解析 vreg / 未知 opcode 列表 | +| `asm_valid` | `len(asm_errors) == 0` | 汇编合法性 | +| `greedy_time_s` | Greedy allocator | Greedy分配器耗时（baseline） | +| `greedy_out_instrs` | — | Greedy分配器输出指令数 | + +**bench_cnn (LLVM 对比侧)**： + +| 键 | 说明 | +|----|------| +| `llvm_im_instrs` | LLVM RV64IM 静态指令数 | +| `llvm_fd_instrs` | LLVM RV64FD 静态指令数 | +| `instr_ratio_fd` | `llvm_fd_instrs / max(sv_static_instrs, 1)` | +| `llvm_fd_cats` | opcode 原始计数 | +| `llvm_fd_cat_buckets` | 7 类汇总（+ Stack） | +| `llvm_spill_slots` | 4 字节 sp 访问 ÷ 2（**近似**） | +| `llvm_frame_save` | `sd` 到 callee-saved 的计数 | +| `llvm_frame_restore` | `ld` 到 callee-saved 的计数 | + +### 4.3 `reg_spill_count` 规范 + +- **经过 regalloc 的路径**：直接取自 `alloc._spill_slots` 长度 → 精确值 +- **不经过 regalloc 的路径**：LLVM 侧 `reg_spill_count` 是本路径的 ScratchV 精确值（0）；LLVM 近似溢出独立为 `llvm_spill_slots`，不污染统一键 +- **降级路径**：当 libLLVM 不可用时，`llvm_fd_instrs`/`llvm_spill_slots` 等键不存在于 dict 中，报告渲染 fallback 到 `"-"` + +--- + +## 5. LLVM 溢出统计 + +由于 libLLVM 缺少 regalloc pass 统计入口，LLVM 侧无法获取精确的寄存器溢出计数，通过汇编层面识别特定模式统计溢出： + +| 形态 | 正则 / 判定 | 含义 | +|------|------------|------| +| 帧保存 | `sd , N(sp)` | prologue 保存 callee-saved 寄存器 | +| 帧恢复 | `ld , N(sp)` | epilogue 恢复 callee-saved 寄存器 | +| 溢出 | `sw/lw/fsw/flw , N(sp)` | 寄存器值被 spill→reload（4 字节） | + +**callee-saved 集合**（RV64 ABI）：`ra, fp, s0–s11, fs0–fs11` + +--- + +## 6. 输出格式 + +### 6.1 终端 + +``` +============================================================ + ScratchV — Register Allocation Benchmark Suite +============================================================ + 1. Simple: reg_spill_count=0, mean=0.019ms ✓ + 2. Dense: reg_spill_count=15, mean=0.084ms ✓ + 3. CNN: reg_spill_count=0, mean=0.052ms ✓ + LLVM: RV64FD=1099 instrs (23.89x vs ScratchV 46) + + Total: 25173.6ms PASS + + JSON report: /tmp/rg_spill.json + HTML report: /tmp/rg_spill.html + Markdown: /tmp/rg_spill.md +``` + +### 6.2 JSON + +标准化结构，`_` 前缀字段（如 asm 文本、allocator 实例）被排除： + +```json +{ + "timestamp": "2026-08-09T...", + "total_time_s": 25.17, + "repeats": 3, + "results": { + "1. Simple Arithmetic": { + "mean_s": 1.9e-05, "reg_spill_count": 0, "peak_active": 5, + "asm_lines": 10, "valid": true + }, + "2. Dense Computation": { + "mean_s": 8.4e-05, "reg_spill_count": 15, "peak_active": 14, + "asm_lines": 179, "reloads": 61, "valid": true + }, + "3. CNN Integration": { + "mean_s": 5.2e-05, "reg_spill_count": 0, + "sv_static_instrs": 46, "llvm_fd_instrs": 1099, + "instr_ratio_fd": 23.89, "llvm_spill_slots": 87, + "llvm_frame_save": 70, "llvm_frame_restore": 70, + "llvm_fd_cat_buckets": {"ALU": 346, "Load": 98, ...}, + "asm_valid": true, "valid": true + } + } +} +``` + +### 6.3 HTML / Markdown 汇总表 + +| Benchmark | Mean(ms) | Std(ms) | Vregs | Spills | Peak | Reloads | Asm | LLVM-FD | Ratio | LLVM-Spill | Valid | +|-----------|----------|---------|-------|--------|------|---------|-----|---------|-------|------------|-------| +| 1. Simple Arithmetic | 0.019 | 0.011 | 5 | 0 | 5 | - | 10 | - | - | - | ✓ | +| 2. Dense Computation | 0.084 | 0.016 | 30 | 15 | 14 | 61 | 179 | - | - | - | ✓ | +| 3. CNN Integration | 0.052 | 0.015 | 30 | 0 | 11 | - | 57 | 1099 | 23.89 | 87 | ✓ | + +--- + +## 7. 使用方式 + +```bash +# 运行全部 3 项 benchmark，生成三格式报告 +python benchmarks/test_regalloc/run_benchmark.py \ + --repeats 30 \ + --output-json report.json \ + --output-html report.html \ + --output-md report.md + +# 单独运行某项 +python benchmarks/test_regalloc/bench_simple.py --repeats 100 +python benchmarks/test_regalloc/bench_dense.py --repeats 50 +python benchmarks/test_regalloc/bench_cnn.py \ + --cnn-path models/graph/cnn.onnx --repeats 30 +``` + +--- + +## 8. TODO + +1. **优化报告输出格式** +2. **将通用的Benchmark组件进一步抽象到`bench_utils.py`当中** +3. **目前ONNX模型的编译路径指令选择方面无法完全正确生成算子的汇编指令，与LLVM后端对比不公平**