diff --git a/.github/workflows/benchmark-full.yml b/.github/workflows/benchmark-full.yml new file mode 100644 index 0000000..51f6f7e --- /dev/null +++ b/.github/workflows/benchmark-full.yml @@ -0,0 +1,151 @@ +name: Benchmark (full sweep) + +# Pull requests only benchmark the modules they touch, which keeps Actions time +# down but means no single PR ever sees the whole picture. This sweep runs +# every workload on master and compares against the previous sweep, so drift +# that accumulates across many small PRs still shows up somewhere. + +on: + schedule: + # 03:00 UTC every Monday. + - cron: '0 3 * * 1' + workflow_dispatch: + inputs: + scale: + description: 'ACL_BENCH_SCALE (a fraction, or "max" for the judge limits)' + required: false + default: '1.0' + passes: + description: 'Interleaved measurement passes' + required: false + default: '3' + +permissions: + contents: read + actions: read + +concurrency: + group: benchmark-full + cancel-in-progress: false + +jobs: + full-sweep: + runs-on: ubuntu-latest + timeout-minutes: 300 + env: + ACL_BENCH_SCALE: ${{ github.event.inputs.scale || '1.0' }} + ACL_BENCH_ROUNDS: '1' + PASSES: ${{ github.event.inputs.passes || '3' }} + ARTIFACT_NAME: benchmark-full-results + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-benchmark + + - name: Run full suite + run: | + set -x + for pass in $(seq 1 "$PASSES"); do + pytest benchmarks/test_time.py --benchmark-only \ + --benchmark-json="current-time-$pass.json" + done + python benchmarks/memory_bench.py --output current-memory.json + + # Fetch the previous sweep so the summary shows deltas rather than bare + # numbers. Absent (first run, expired retention), the report still + # renders with the current times and n/a deltas. + - name: Download previous sweep + id: previous + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const runs = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'benchmark-full.yml', + status: 'success', + per_page: 10, + }); + for (const run of runs.data.workflow_runs) { + if (run.id === context.runId) continue; + const arts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + }); + const art = arts.data.artifacts.find( + a => a.name === process.env.ARTIFACT_NAME && !a.expired + ); + if (!art) continue; + const zip = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: art.id, + archive_format: 'zip', + }); + fs.writeFileSync('previous.zip', Buffer.from(zip.data)); + core.setOutput('found', 'true'); + core.setOutput('sha', run.head_sha); + core.info(`Comparing against run ${run.id} (${run.head_sha})`); + return; + } + core.setOutput('found', 'false'); + core.info('No previous sweep artifact available.'); + + - name: Unpack previous sweep + run: | + mkdir -p previous + if [ "${{ steps.previous.outputs.found }}" = "true" ] && [ -f previous.zip ]; then + unzip -o -q previous.zip -d previous + fi + # compare.py needs a base to read even when there is no history yet. + if ! ls previous/current-time-*.json >/dev/null 2>&1; then + echo '{"benchmarks": []}' > previous/current-time-1.json + fi + if [ ! -f previous/current-memory.json ]; then + echo '{"benchmarks": []}' > previous/current-memory.json + fi + + - name: Compare against previous sweep + run: | + python benchmarks/compare.py \ + --base-time previous/current-time-*.json \ + --pr-time current-time-*.json \ + --base-memory previous/current-memory.json \ + --pr-memory current-memory.json \ + --base-label 'Previous' --pr-label 'Now' \ + --title '๐Ÿ“Š Full benchmark sweep' \ + --output report.md + + - name: Publish to job summary + run: | + { + echo "Commit: \`${{ github.sha }}\`" + if [ "${{ steps.previous.outputs.found }}" = "true" ]; then + echo "Previous sweep: \`${{ steps.previous.outputs.sha }}\`" + else + echo "No previous sweep to compare against." + fi + echo + cat report.md + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: benchmark-full-results + retention-days: 90 + path: | + current-time-*.json + current-memory.json + report.md diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f49640f..31d86fb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,9 +8,24 @@ permissions: contents: read pull-requests: write +concurrency: + group: benchmark-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + # Fraction of the Library Checker maximum constraints to run at. The query + # mix and operand types are scale-invariant, so a scaled run keeps the shape + # that makes these benchmarks predictive while fitting in a CI budget. + ACL_BENCH_SCALE: '0.2' + # One timed round per pass; the passes below provide the repetitions, and + # interleaving them is what makes base and PR comparable. + ACL_BENCH_ROUNDS: '1' + PASSES: '3' + jobs: benchmark: runs-on: ubuntu-latest + timeout-minutes: 60 steps: - name: Checkout PR branch uses: actions/checkout@v4 @@ -33,46 +48,92 @@ jobs: python -m pip install --upgrade pip pip install pytest pytest-benchmark - - name: Run PR benchmarks (time) - working-directory: pr - run: pytest benchmarks/test_time.py --benchmark-only --benchmark-json=../pr-time.json + - name: List changed files + uses: actions/github-script@v7 + with: + script: | + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100, + }); + require('fs').writeFileSync( + 'changed-files.txt', + files.map(f => f.filename).join('\n') + '\n' + ); - - name: Run PR benchmarks (memory) - working-directory: pr - run: python benchmarks/memory_bench.py --output ../pr-memory.json + # A change to segtree.py cannot make factorize faster, so measuring + # factorize would only burn Actions minutes and give noise another chance + # to invent a row. Everything runs when the workloads or the harness + # themselves change, because then the comparison is what is in question. + - name: Select affected benchmarks + id: select + run: | + names=$(python pr/benchmarks/select_workloads.py \ + --changed-files changed-files.txt --format csv) + echo "names=$names" >> "$GITHUB_OUTPUT" + echo "Selected: ${names:-}" - - name: Run base benchmarks (time) - working-directory: base + - name: Run benchmarks (interleaved passes) + if: steps.select.outputs.names != '' + env: + ACL_BENCH_ONLY: ${{ steps.select.outputs.names }} run: | - if [ -f benchmarks/test_time.py ]; then - pytest benchmarks/test_time.py --benchmark-only --benchmark-json=../base-time.json - else - echo '{"benchmarks": []}' > ../base-time.json - fi + set -x + for pass in $(seq 1 "$PASSES"); do + (cd pr && pytest benchmarks/test_time.py --benchmark-only \ + --benchmark-json="../pr-time-$pass.json") + if [ -f base/benchmarks/test_time.py ]; then + (cd base && pytest benchmarks/test_time.py --benchmark-only \ + --benchmark-json="../base-time-$pass.json") + else + echo '{"benchmarks": []}' > "base-time-$pass.json" + fi + done - - name: Run base benchmarks (memory) - working-directory: base + - name: Run benchmarks (memory) + if: steps.select.outputs.names != '' + env: + ACL_BENCH_ONLY: ${{ steps.select.outputs.names }} run: | - if [ -f benchmarks/memory_bench.py ]; then - python benchmarks/memory_bench.py --output ../base-memory.json + (cd pr && python benchmarks/memory_bench.py --output ../pr-memory.json) + if [ -f base/benchmarks/memory_bench.py ]; then + (cd base && python benchmarks/memory_bench.py --output ../base-memory.json) else - echo '{"benchmarks": []}' > ../base-memory.json + echo '{"benchmarks": []}' > base-memory.json fi - name: Compare results + if: steps.select.outputs.names != '' run: | python pr/benchmarks/compare.py \ - --base-time base-time.json --pr-time pr-time.json \ + --base-time base-time-*.json --pr-time pr-time-*.json \ --base-memory base-memory.json --pr-memory pr-memory.json \ + --only "${{ steps.select.outputs.names }}" \ --output report.md + - name: Report that nothing needed benchmarking + if: steps.select.outputs.names == '' + run: | + { + echo '## ๐Ÿ“Š Performance Benchmark Results' + echo + echo 'No benchmarked module changed, so no workload was run.' + echo + echo 'Benchmarks run only for the modules a pull request' + echo 'touches. A full sweep of every workload runs on a schedule' + echo '(see `.github/workflows/benchmark-full.yml`).' + } > report.md + - name: Upload raw results + if: steps.select.outputs.names != '' uses: actions/upload-artifact@v4 with: name: benchmark-results path: | - pr-time.json - base-time.json + pr-time-*.json + base-time-*.json pr-memory.json base-memory.json report.md diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..7311908 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,161 @@ +# Benchmarks + +These benchmarks exist to answer one question: **will this change make a real +submission faster?** + +Every primary workload is therefore a transcription of an actual problem from +[yosupo06/library-checker-problems](https://github.com/yosupo06/library-checker-problems). +If a change is faster here, it should be faster on +[Library Checker](https://judge.yosupo.jp/) too. + +## Why the old workloads disagreed with the judge + +The previous suite generated its own query mixes ("exercise every public +method"), which made its numbers systematically misleading: + +| Problem | Effect | +|---|---| +| Invented query mix | 20% of the segtree benchmark was `max_right`/`min_left`, which Point Set Range Composite never calls. A 30% win in `prod` showed up as 8%; a win in a method nobody uses showed up as a headline number. | +| Unrealistic operators | `segtree(values, max, -1)` uses a C builtin as `op`, so tree-internal Python overhead dominated. Real submissions pass a Python lambda that costs an order of magnitude more per call, which dilutes the same change to a few percent. | +| Wrong range distribution | `l = randrange(n); r = randrange(l, n + 1)` is not the judge's `uniform_pair`, so query widths โ€” and therefore segment-tree descent depth โ€” were off. | +| Sizes below the constraints | segtree at 200k vs the judge's 500k, convolution at 2^16 vs 2^19, FPS at 8k vs 500k. | +| Results thrown away | `seg.prod(l, r)` with the return value discarded is not what a submission does. | +| `mean` of 3 rounds, compared across two sequential jobs | On a shared runner that is worth several percent on its own, and the report flagged โœ… at 1%. | + +## What runs now + +| Benchmark | Module | Library Checker problem | Max constraints | +|---|---|---|---| +| `point_add_range_sum` | `fenwicktree` | [Point Add Range Sum](https://judge.yosupo.jp/problem/point_add_range_sum) | N = Q = 500,000 | +| `staticrmq` | `segtree` | [Static RMQ](https://judge.yosupo.jp/problem/staticrmq) | N = Q = 500,000 | +| `point_set_range_composite` | `segtree` | [Point Set Range Composite](https://judge.yosupo.jp/problem/point_set_range_composite) | N = Q = 500,000 | +| `range_affine_range_sum` | `lazysegtree` | [Range Affine Range Sum](https://judge.yosupo.jp/problem/range_affine_range_sum) | N = Q = 500,000 | +| `unionfind` | `dsu` | [Unionfind](https://judge.yosupo.jp/problem/unionfind) | N = Q = 200,000 | +| `scc` | `scc` | [Strongly Connected Components](https://judge.yosupo.jp/problem/scc) | N = M = 500,000 | +| `two_sat` | `two_sat` | [2 SAT](https://judge.yosupo.jp/problem/two_sat) | N = M = 500,000 | +| `bipartitematching` | `maxflow` | [Matching on Bipartite Graph](https://judge.yosupo.jp/problem/bipartitematching) | L = R = 100,000, M = 200,000 | +| `assignment` | `mincostflow` | [Assignment Problem](https://judge.yosupo.jp/problem/assignment) | N = 500 | +| `convolution_mod` | `convolution` | [Convolution](https://judge.yosupo.jp/problem/convolution_mod) | N = M = 524,288 | +| `inv_of_formal_power_series` | `fps` | [Inv of FPS](https://judge.yosupo.jp/problem/inv_of_formal_power_series) | N = 500,000 | +| `log_of_formal_power_series` | `fps` | [Log of FPS](https://judge.yosupo.jp/problem/log_of_formal_power_series) | N = 500,000 | +| `exp_of_formal_power_series` | `fps` | [Exp of FPS](https://judge.yosupo.jp/problem/exp_of_formal_power_series) | N = 500,000 | +| `suffixarray` | `acl_string` | [Suffix Array](https://judge.yosupo.jp/problem/suffixarray) | N = 500,000 | +| `zalgorithm` | `acl_string` | [Z Algorithm](https://judge.yosupo.jp/problem/zalgorithm) | N = 500,000 | +| `number_of_substrings` | `acl_string` | [Number of Substrings](https://judge.yosupo.jp/problem/number_of_substrings) | N = 500,000 | +| `sum_of_floor_of_linear` | `acl_math` | [Sum of Floor of Linear](https://judge.yosupo.jp/problem/sum_of_floor_of_linear) | T = 100,000 | +| `primality_test` | `prime_fact` | [Primality Test](https://judge.yosupo.jp/problem/primality_test) | Q = 100,000, N โ‰ค 10^18 | +| `factorize` | `prime_fact` | [Factorize](https://judge.yosupo.jp/problem/factorize) | Q = 100, A โ‰ค 10^18 | + +Inputs are generated the way each problem's `gen/max_random.cpp` does: same +query mix, same value ranges, and a faithful port of the judge's `uniform_pair` +for range queries. The `run_*` functions are shaped like accepted submissions โ€” +same monoid, same operators, and they accumulate the answers rather than +discarding them. `tests/test_benchmark_workloads.py` checks each one against a +naive reference so a benchmark can never get "fast" by being wrong. + +### API coverage workloads + +`extra_*` workloads cover public methods no Library Checker problem exercises +(`max_right`/`min_left`, `dsu.groups`, `min_cut`, `change_edge`, +`mcf_graph.slope`, `crt`, `divisors`/`totient`/`lcm`). They are a regression +net only. **Their timings are not calibrated against anything โ€” do not quote +them as speedups.** The report keeps them in a separate collapsed table for +that reason. + +## Sizing + +`ACL_BENCH_SCALE` scales N/Q/M. The query mix, operand types and range +distribution are scale-invariant, so a scaled run keeps the shape that makes +the benchmark predictive. + +The modules differ by orders of magnitude in per-element cost, so each workload +also carries a `COST_FACTOR` in `workloads.py`. The effective size is +`min(1, ACL_BENCH_SCALE * COST_FACTOR[name])` of the judge's maximum โ€” capped, +so a workload never runs past constraints the judge does not enforce. + +```sh +# CI default: every workload lands in a one-to-four-second band +pytest benchmarks/test_time.py + +# every workload at its exact Library Checker maximum (well over an hour) +ACL_BENCH_SCALE=max pytest benchmarks/test_time.py + +# a single workload, at full size +ACL_BENCH_SCALE=max pytest benchmarks/test_time.py -k point_set_range_composite +``` + +Two workloads run well below the judge's constraints even at the default, +because the current implementations cannot reach them in CPython: + +* `bipartitematching` (3% of L/R/M). `mf_graph.flow` runs a full BFS per + augmenting path instead of per phase, so unit-capacity matching costs + O(VยทEยทflow) rather than Dinic's O(EโˆšV) โ€” 196 seconds at 20% of the judge's + size. The low factor is a CI budget decision, not a claim that the size is + representative. +* `range_affine_range_sum`, `exp/log_of_formal_power_series` (10%). + +## What runs, and when + +Running all 25 workloads on both branches of every pull request costs far more +Actions time than it earns. So: + +**On a pull request**, only the workloads whose module the diff touches are +measured. `select_workloads.py` maps changed files to workload names, and the +workflow passes them to both runners via `ACL_BENCH_ONLY`; `compare.py --only` +keeps the report to the same set rather than padding it with `n/a` rows. The +mapping is exact rather than heuristic, because none of the library modules +imports another โ€” they are meant to be pasted into a submission โ€” so a change +to `scc.py` reaches exactly the workloads whose module is `scc`. + +Note that `two_sat.py` and `fps.py` carry their own inlined copies of the SCC +and NTT routines. Editing `scc.py` or `convolution.py` genuinely does not +affect them, and the selection reflects that; if you port an optimisation +between the copies, touch both files. + +Changing `benchmarks/workloads.py`, `test_time.py`, `memory_bench.py`, +`select_workloads.py` or the workflow runs everything, because then the +comparison itself is what is in question. Changing anything else โ€” docs, tests, +`compare.py` โ€” runs nothing, and the PR comment says so. + +```sh +git diff --name-only origin/master... | python benchmarks/select_workloads.py +ACL_BENCH_ONLY=staticrmq,unionfind pytest benchmarks/test_time.py +``` + +**On a schedule** (`benchmark-full.yml`, Mondays 03:00 UTC, or on demand via +*Run workflow*), the whole suite runs on master at `ACL_BENCH_SCALE=1.0` and is +compared against the previous sweep's artifact. That is the safety net for +drift that accumulates across many small PRs, and for anything the per-PR +selection cannot see. Results go to the job summary and are kept as artifacts +for 90 days. + +## Reading the report + +`compare.py` renders the PR comment. Two things about it are deliberate: + +* **The statistic is the minimum, not the mean.** Interference from + neighbouring jobs can only make a run slower, so the fastest observation is + the closest estimate of the code's real cost. The mean mostly measures how + busy the runner was. +* **A delta is only called a win or a regression when it exceeds the noise we + actually measured** between passes, and never below 3%. Anything smaller is + reported as `โ‰ˆ`. The base and PR branches are measured in *interleaved* + passes for the same reason โ€” running one to completion before the other turns + the runner's drift into a fake delta. + +Memory is measured with `tracemalloc` and is deterministic, so it needs only +one pass and a 1% allowance. + +## Local use + +```sh +pip install pytest pytest-benchmark + +pytest benchmarks/test_time.py --benchmark-json=result.json +python benchmarks/memory_bench.py --output memory.json + +python benchmarks/compare.py \ + --base-time base-time-*.json --pr-time pr-time-*.json \ + --base-memory base-memory.json --pr-memory pr-memory.json \ + --output report.md +``` diff --git a/benchmarks/compare.py b/benchmarks/compare.py index 0d783d6..b21176c 100644 --- a/benchmarks/compare.py +++ b/benchmarks/compare.py @@ -4,9 +4,22 @@ Usage: python benchmarks/compare.py \\ - --base-time base-time.json --pr-time pr-time.json \\ + --base-time base-time-*.json --pr-time pr-time-*.json \\ --base-memory base-memory.json --pr-memory pr-memory.json \\ --output report.md + +Several time files may be given per side: the workflow runs the two branches in +interleaved passes so that drift on the runner hits both sides equally. For +each workload we keep the *minimum* observed time across all rounds of all +passes. The minimum is the right statistic here -- interference from +neighbouring jobs can only ever make a run slower, so the fastest observation +is the closest estimate of the code's actual cost, while the mean mostly +measures how busy the runner was. + +A delta is only called a win or a regression when it is larger than the noise +we actually observed between passes (and never below ``MIN_SIGNIFICANT``). +Reporting "1% faster" from a single noisy pair of runs is how a benchmark ends +up disagreeing with the judge. """ import argparse @@ -15,6 +28,15 @@ NAME_RE = re.compile(r"\[(.+)\]$") +# A delta smaller than this is never reported as a change, however quiet the +# runner looked. GitHub-hosted runners do not do better than a few percent. +MIN_SIGNIFICANT = 0.03 + +# tracemalloc peaks are deterministic, so memory needs only a token allowance. +MEMORY_SIGNIFICANT = 0.01 + +LIBRARY_CHECKER = "https://judge.yosupo.jp/problem/" + def _benchmark_name(raw_name): m = NAME_RE.search(raw_name) @@ -25,19 +47,35 @@ def _benchmark_name(raw_name): return raw_name -def load_time_results(path): - with open(path) as f: - data = json.load(f) - return { - _benchmark_name(b["name"]): b["stats"]["mean"] - for b in data.get("benchmarks", []) - } +def load_time_results(paths): + """Return {name: {"samples": [per-pass minimum, ...], "info": {...}}}.""" + results = {} + for path in paths: + try: + with open(path) as f: + data = json.load(f) + except (OSError, ValueError): + continue + for b in data.get("benchmarks", []): + name = _benchmark_name(b["name"]) + entry = results.setdefault(name, {"samples": [], "info": {}}) + # "min" is best-of-rounds within this pass. + entry["samples"].append(b["stats"]["min"]) + entry["info"].update(b.get("extra_info") or {}) + return results -def load_memory_results(path): - with open(path) as f: - data = json.load(f) - return {b["name"]: b["peak_bytes"] for b in data.get("benchmarks", [])} +def load_memory_results(paths): + results = {} + for path in paths: + try: + with open(path) as f: + data = json.load(f) + except (OSError, ValueError): + continue + for b in data.get("benchmarks", []): + results[b["name"]] = b["peak_bytes"] + return results def fmt_time(seconds): @@ -57,44 +95,157 @@ def fmt_bytes(n): return f"{value:.1f}TiB" -def fmt_delta(base, pr): +def spread(samples): + """Relative gap between the slowest and fastest pass: our noise estimate.""" + if len(samples) < 2: + return 0.0 + best = min(samples) + if best <= 0: + return 0.0 + return (max(samples) - best) / best + + +def fmt_delta(base, pr, threshold): if base == 0: return "n/a" + if base == pr: + return "0.0%" pct = (pr - base) / base * 100 - if pct <= -1: - mark = "โœ…" - elif pct > 5: - mark = "โš ๏ธ" - else: - mark = "" sign = "+" if pct >= 0 else "" - return f"{sign}{pct:.1f}% {mark}".strip() + if abs(pct) < threshold * 100: + return f"{sign}{pct:.1f}% โ‰ˆ" + mark = "โœ…" if pct < 0 else "โš ๏ธ" + return f"{sign}{pct:.1f}% {mark}" -def build_table(names, base_map, pr_map, fmt_value): - lines = ["| Benchmark | Base | PR | Delta |", "|---|---|---|---|"] +def fmt_pct(fraction): + """Percentage that stays informative below 1% (sizes can be tiny in CI).""" + pct = fraction * 100 + if pct >= 10: + return f"{pct:.0f}%" + if pct >= 1: + return f"{pct:.1f}%" + return f"{pct:.2f}%" + + +def name_cell(name, info): + problem = (info or {}).get("problem") + if problem: + return f"[{name}]({LIBRARY_CHECKER}{problem})" + return name + + +def build_time_table(names, base, pr, base_label="Base", pr_label="PR"): + lines = [ + f"| Benchmark | {base_label} | {pr_label} | Delta | Noise |", + "|---|---|---|---|---|", + ] for name in names: - base_v = base_map.get(name) - pr_v = pr_map.get(name) + base_entry = base.get(name) + pr_entry = pr.get(name) + info = (pr_entry or base_entry or {}).get("info", {}) + label = name_cell(name, info) + if not base_entry or not pr_entry: + base_cell = fmt_time(min(base_entry["samples"])) if base_entry else "n/a" + pr_cell = fmt_time(min(pr_entry["samples"])) if pr_entry else "n/a" + lines.append(f"| {label} | {base_cell} | {pr_cell} | n/a | n/a |") + continue + base_v = min(base_entry["samples"]) + pr_v = min(pr_entry["samples"]) + noise = max(spread(base_entry["samples"]), spread(pr_entry["samples"])) + threshold = max(MIN_SIGNIFICANT, noise) + lines.append( + f"| {label} | {fmt_time(base_v)} | {fmt_time(pr_v)} | " + f"{fmt_delta(base_v, pr_v, threshold)} | ยฑ{noise * 100:.1f}% |" + ) + return "\n".join(lines) + + +def build_memory_table(names, base, pr, base_label="Base", pr_label="PR"): + lines = [f"| Benchmark | {base_label} | {pr_label} | Delta |", "|---|---|---|---|"] + for name in names: + base_v = base.get(name) + pr_v = pr.get(name) if base_v is None or pr_v is None: lines.append( - f"| {name} | {'n/a' if base_v is None else fmt_value(base_v)} | " - f"{'n/a' if pr_v is None else fmt_value(pr_v)} | n/a |" + f"| {name} | {'n/a' if base_v is None else fmt_bytes(base_v)} | " + f"{'n/a' if pr_v is None else fmt_bytes(pr_v)} | n/a |" ) continue lines.append( - f"| {name} | {fmt_value(base_v)} | {fmt_value(pr_v)} | {fmt_delta(base_v, pr_v)} |" + f"| {name} | {fmt_bytes(base_v)} | {fmt_bytes(pr_v)} | " + f"{fmt_delta(base_v, pr_v, MEMORY_SIGNIFICANT)} |" ) return "\n".join(lines) +def split_names(names, base, pr): + """Library-Checker-calibrated names first, uncalibrated extras second.""" + calibrated, extras = [], [] + for name in names: + info = (pr.get(name) or base.get(name) or {}).get("info", {}) + if info.get("problem") or not name.startswith("extra_"): + calibrated.append(name) + else: + extras.append(name) + return calibrated, extras + + +def scale_note(base, pr): + scales = { + entry["info"].get("global_scale") + for entry in list(base.values()) + list(pr.values()) + if entry.get("info", {}).get("global_scale") + } + if len(scales) == 1: + return f"Workloads ran at `ACL_BENCH_SCALE={scales.pop()}`." + if len(scales) > 1: + return ( + "โš ๏ธ Base and PR ran at different `ACL_BENCH_SCALE` values " + f"({sorted(scales)}); the times are not comparable." + ) + return "" + + +def build_size_table(names, base, pr): + """What each workload actually ran, relative to the judge's constraints.""" + rows = [] + for name in names: + info = (pr.get(name) or base.get(name) or {}).get("info", {}) + full_size = info.get("full_size") + if not full_size: + continue + scale = info.get("scale") + pct = fmt_pct(scale) if isinstance(scale, (int, float)) else "?" + note = info.get("note") or "" + rows.append(f"| {name_cell(name, info)} | {full_size} | {pct} | {note} |") + if not rows: + return "" + return "\n".join( + ["| Benchmark | Library Checker max | Ran at | Note |", "|---|---|---|---|"] + + rows + ) + + def main(): parser = argparse.ArgumentParser() - parser.add_argument("--base-time", required=True) - parser.add_argument("--pr-time", required=True) - parser.add_argument("--base-memory", required=True) - parser.add_argument("--pr-memory", required=True) + parser.add_argument("--base-time", required=True, nargs="+") + parser.add_argument("--pr-time", required=True, nargs="+") + parser.add_argument("--base-memory", required=True, nargs="+") + parser.add_argument("--pr-memory", required=True, nargs="+") parser.add_argument("--output", required=True) + parser.add_argument( + "--only", + default="", + help=( + "comma-separated workload names to report. Pull requests benchmark " + "only the modules they touch, so without this the table would pad " + "itself with n/a rows for workloads that were deliberately skipped." + ), + ) + parser.add_argument("--base-label", default="Base") + parser.add_argument("--pr-label", default="PR") + parser.add_argument("--title", default="\U0001f4ca Performance Benchmark Results") args = parser.parse_args() base_time = load_time_results(args.base_time) @@ -104,29 +255,77 @@ def main(): time_names = sorted(set(base_time) | set(pr_time)) memory_names = sorted(set(base_memory) | set(pr_memory)) + if args.only: + wanted = {n.strip() for n in args.only.split(",") if n.strip()} + time_names = [n for n in time_names if n in wanted] + memory_names = [n for n in memory_names if n in wanted] + calibrated, extras = split_names(time_names, base_time, pr_time) - report = ["## \U0001f4ca Performance Benchmark Results", ""] + report = [f"## {args.title}", ""] if not time_names and not memory_names: report.append("No benchmark results were produced.") else: - report.append("### โฑ๏ธ Execution time (mean, lower is better)") + note = scale_note(base_time, pr_time) + if note: + report.append(note) + report.append("") + report.append("### โฑ๏ธ Library Checker workloads (best of N, lower is better)") report.append( - build_table(time_names, base_time, pr_time, fmt_time) - if time_names + build_time_table( + calibrated, base_time, pr_time, args.base_label, args.pr_label + ) + if calibrated else "n/a" ) report.append("") - report.append("### \U0001f4be Peak memory (lower is better)") + if extras: + report.append( + "
\U0001f9ea API coverage workloads (not judge-calibrated)" + ) + report.append("") + report.append( + build_time_table( + extras, base_time, pr_time, args.base_label, args.pr_label + ) + ) + report.append("") + report.append("
") + report.append("") + size_table = build_size_table(calibrated + extras, base_time, pr_time) + if size_table: + report.append("
\U0001f4cf Workload sizes") + report.append("") + report.append(size_table) + report.append("") + report.append("
") + report.append("") + report.append( + "
\U0001f4be Peak memory (lower is better)" + ) + report.append("") report.append( - build_table(memory_names, base_memory, pr_memory, fmt_bytes) + build_memory_table( + memory_names, base_memory, pr_memory, args.base_label, args.pr_label + ) if memory_names else "n/a" ) report.append("") + report.append("
") + report.append("") report.append( - "Base = target branch, PR = this branch. Negative delta = improvement. " - "โœ… improvement โ‰ฅ 1%, โš ๏ธ regression > 5%. " - '"n/a" base entries mean the target branch has no benchmark suite yet.' + "Only the workloads whose module the change touches are measured; " + "a scheduled sweep covers the rest. Each workload mirrors the " + "linked Library Checker problem: same query mix, same operators, same " + "value ranges as that problem's max_random generator. " + "Times are the best observation across interleaved passes; " + "Noise is the spread between passes. " + "โœ… improvement / โš ๏ธ regression are only shown when the delta " + "exceeds both 3% and the measured noise; โ‰ˆ means the difference is " + "not distinguishable from noise. " + "API-coverage workloads exist to catch regressions in methods no judge " + "problem exercises โ€” do not quote their timings as speedups. " + '"n/a" rows mean one side has no such benchmark.' ) text = "\n".join(report) diff --git a/benchmarks/memory_bench.py b/benchmarks/memory_bench.py index bbfae88..da083bf 100644 --- a/benchmarks/memory_bench.py +++ b/benchmarks/memory_bench.py @@ -1,9 +1,15 @@ #!/usr/bin/env python3 -"""Measure peak memory usage (via tracemalloc) for ACL-python's core data -structures, using the same workloads as benchmarks/test_time.py. +"""Measure peak memory usage (via tracemalloc) for ACL-python, using the same +Library-Checker-calibrated workloads as benchmarks/test_time.py. Usage: python benchmarks/memory_bench.py --output result.json + +Unlike wall-clock time, tracemalloc's peak is deterministic for a fixed input, +so a single pass is enough and the comparison needs no noise allowance. + +Honours ACL_BENCH_SCALE and ACL_BENCH_ONLY exactly as benchmarks/test_time.py +does, so a pull request measures the same subset for time and for memory. """ import argparse @@ -17,11 +23,11 @@ import workloads # noqa: E402 -def measure(build_fn, run_fn): - args = build_fn() +def measure(workload): + args = workload.build() tracemalloc.start() before, _ = tracemalloc.get_traced_memory() - result = run_fn(*args) + result = workload.run(*args) _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() del result @@ -34,13 +40,24 @@ def main(): args = parser.parse_args() results = [] - for name, build_fn, run_fn in workloads.BENCHMARKS: - peak_bytes = measure(build_fn, run_fn) - results.append({"name": name, "peak_bytes": peak_bytes}) - print(f"{name}: {peak_bytes / 1024:.1f} KiB") + for workload in workloads.SELECTED: + peak_bytes = measure(workload) + results.append( + { + "name": workload.name, + "peak_bytes": peak_bytes, + "module": workload.module, + "problem": workload.problem or "", + } + ) + print(f"{workload.name}: {peak_bytes / 1024:.1f} KiB", flush=True) with open(args.output, "w") as f: - json.dump({"benchmarks": results}, f, indent=2) + json.dump( + {"scale": workloads.SCALE, "benchmarks": results}, + f, + indent=2, + ) if __name__ == "__main__": diff --git a/benchmarks/select_workloads.py b/benchmarks/select_workloads.py new file mode 100644 index 0000000..1064ae2 --- /dev/null +++ b/benchmarks/select_workloads.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Pick the benchmarks a change can actually affect. + +Running all 25 workloads on both branches of every pull request costs far more +Actions time than it earns: a change to ``segtree.py`` cannot make ``factorize`` +faster, so measuring ``factorize`` only adds runtime and another chance for +noise to produce a spurious row. + +The mapping is exact rather than heuristic because the library modules are +standalone -- none of them imports another (they are meant to be pasted into a +submission), so a change to ``scc.py`` reaches exactly the workloads whose +module is ``scc``. Note that ``two_sat.py`` and ``fps.py`` carry their own +inlined copies of the SCC and NTT routines; editing ``scc.py`` or +``convolution.py`` genuinely does not affect them, and the selection reflects +that. + +Usage: + python benchmarks/select_workloads.py --changed-files changed.txt + git diff --name-only origin/master... | python benchmarks/select_workloads.py +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import workloads # noqa: E402 + +# Touching any of these invalidates the comparison itself (the workloads or the +# harness changed), so everything has to run. +RUN_EVERYTHING = frozenset( + { + "benchmarks/workloads.py", + "benchmarks/test_time.py", + "benchmarks/memory_bench.py", + "benchmarks/select_workloads.py", + ".github/workflows/benchmark.yml", + } +) + +# Every benchmarked module is a single top-level file named after itself. +MODULE_FILE = {f"{w.module}.py": w.module for w in workloads.BENCHMARKS} + + +def _normalise(path): + path = path.strip() + # Not lstrip("./"): that strips every leading "." and "/", turning + # ".github/workflows/benchmark.yml" into "github/workflows/benchmark.yml". + if path.startswith("./"): + path = path[2:] + return path + + +def select(changed_files): + """Workload names to benchmark, in registry order.""" + changed = {_normalise(path) for path in changed_files if path.strip()} + if changed & RUN_EVERYTHING: + return [w.name for w in workloads.BENCHMARKS] + modules = {MODULE_FILE[path] for path in changed if path in MODULE_FILE} + if not modules: + return [] + return [w.name for w in workloads.BENCHMARKS if w.module in modules] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--changed-files", + help="file listing changed paths, one per line (default: stdin)", + ) + parser.add_argument( + "--format", + choices=("names", "csv"), + default="names", + help="'names' prints one per line; 'csv' is the ACL_BENCH_ONLY format", + ) + args = parser.parse_args() + + if args.changed_files: + with open(args.changed_files) as f: + changed = f.read().splitlines() + else: + changed = sys.stdin.read().splitlines() + + names = select(changed) + if args.format == "csv": + print(",".join(names)) + else: + for name in names: + print(name) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/test_time.py b/benchmarks/test_time.py index 82e6f6f..65b98e1 100644 --- a/benchmarks/test_time.py +++ b/benchmarks/test_time.py @@ -2,6 +2,18 @@ Usage: pytest benchmarks/test_time.py --benchmark-json=result.json + +Environment: + ACL_BENCH_SCALE size of the workloads relative to the Library Checker + maximum constraints (see workloads.py). Default 0.2. + ACL_BENCH_ROUNDS timed rounds per workload per pass. Default 3. + ACL_BENCH_ONLY comma-separated workload names to run. Pull requests set + this to the modules they touch (see select_workloads.py). + +The reported statistic that ``compare.py`` uses is the *minimum* across rounds +and across passes, not the mean: on a shared CI runner the mean is dominated by +interference from neighbouring jobs, and a 1-2% "improvement" in the mean is +almost always noise rather than a real change. """ import os @@ -13,12 +25,23 @@ import workloads # noqa: E402 +ROUNDS = int(os.environ.get("ACL_BENCH_ROUNDS", "3")) + @pytest.mark.parametrize( - "name,build_fn,run_fn", - workloads.BENCHMARKS, - ids=[b[0] for b in workloads.BENCHMARKS], + "workload", + workloads.SELECTED, + ids=[w.name for w in workloads.SELECTED], ) -def test_benchmark(benchmark, name, build_fn, run_fn): - args = build_fn() - benchmark.pedantic(run_fn, args=args, rounds=3, warmup_rounds=1) +def test_benchmark(benchmark, workload): + args = workload.build() + benchmark.extra_info["module"] = workload.module + benchmark.extra_info["problem"] = workload.problem or "" + benchmark.extra_info["full_size"] = workload.full_size + benchmark.extra_info["scale"] = workload.effective_scale + # float("inf") is not portable through JSON, so send "max" as a string. + benchmark.extra_info["global_scale"] = ( + "max" if workloads.SCALE == float("inf") else str(workloads.SCALE) + ) + benchmark.extra_info["note"] = workload.note + benchmark.pedantic(workload.run, args=args, rounds=ROUNDS, warmup_rounds=1) diff --git a/benchmarks/workloads.py b/benchmarks/workloads.py index 37488eb..800e140 100644 --- a/benchmarks/workloads.py +++ b/benchmarks/workloads.py @@ -1,12 +1,57 @@ -"""Deterministic workloads shared by the time (pytest-benchmark) and memory -(tracemalloc) benchmarks, so both measure exactly the same operations. - -Data sizes are intentionally large (and randomly generated) because these -benchmarks only care about wall-clock/memory cost, not about producing a -particular result. Each workload also drives every public method the -corresponding module exposes (e.g. dsu exercises leader/merge/same/size/ -groups) so a regression in any single method is caught, not just the ones -that happened to be exercised before. +"""Benchmark workloads, calibrated against Library Checker problems. + +Why this file looks the way it does +----------------------------------- +The point of these benchmarks is to predict whether a change makes a real +submission faster. A synthetic workload that exercises "every public method +with a made-up query mix" does not do that: it can report a large win for an +optimisation that a real judge run barely notices (or that is actually a +regression), because the operation mix, the operand types and the input sizes +are all different from what the judge runs. + +So every primary workload here is a transcription of an actual problem from +https://github.com/yosupo06/library-checker-problems : + +* the input is generated the way that problem's ``gen/max_random.cpp`` does + (same query mix, same value ranges, same ``uniform_pair`` range + distribution), +* the sizes are that problem's maximum constraints (``full_size`` on each + workload), scaled by ``ACL_BENCH_SCALE`` so CI stays affordable, +* the ``run`` function is shaped like an accepted submission: it uses the same + monoid / operator the submission would use, and it accumulates the answers + instead of throwing them away. + +That last point matters more than it looks. ``segtree(values, max, -1)`` +measures the segment tree with the cheapest possible ``op`` (a C builtin), so +Python-level overhead inside the tree dominates and any change to it looks +enormous. Point Set Range Composite uses a Python lambda that composes affine +maps, which costs an order of magnitude more per call -- the same change is +then worth a few percent. Benchmarking the first and shipping for the second +is exactly how a benchmark ends up disagreeing with the judge. + +A handful of secondary workloads (``extra_*``, ``Workload.problem is None``) +cover public methods that no Library Checker problem exercises. They exist as +a regression net only; their timings are *not* calibrated against anything and +should not be used to justify a performance claim. + +Sizing +------ +``ACL_BENCH_SCALE`` scales N/Q/M. The query mix, operand types and range +distribution are scale-invariant, so a scaled run keeps the shape that makes +the benchmark predictive while running in a fraction of the time. + +A single global scale is not enough on its own: at the same fraction of their +respective maxima, ``zalgorithm`` finishes in 20ms while ``bipartitematching`` +takes three minutes, and a workload that runs for 20ms measures the timer more +than it measures the library. So each workload also carries a ``COST_FACTOR`` +that normalises for its per-element cost. The effective fraction is +``min(1, ACL_BENCH_SCALE * COST_FACTOR[name])`` -- capped, so no workload ever +runs past the constraints the judge actually enforces. + +* ``ACL_BENCH_SCALE`` unset (default 0.2): every workload takes roughly one to + four seconds per round. +* ``ACL_BENCH_SCALE=max``: every workload at its exact Library Checker maximum. + Expect this to take well over an hour in CPython. """ import os @@ -30,394 +75,919 @@ import two_sat as two_sat_mod MOD = 998244353 +INF = 1 << 62 + +LIBRARY_CHECKER = "https://judge.yosupo.jp/problem/" + + +DEFAULT_SCALE = 0.2 + +# Per-workload cost normalisation; see "Sizing" above. A factor of 5 means the +# workload reaches its Library Checker maximum already at the default scale +# (it is cheap enough that there is no reason to shrink it); a factor below 1 +# means the module is too slow for CPython to reach the judge's constraints in +# a CI-sized budget. Values were picked so every workload lands in the same +# one-to-four-second band at the default scale -- re-measure before changing +# one, and change it only for time budget, never to flatter a result. +COST_FACTOR = { + "point_add_range_sum": 3.0, + "staticrmq": 3.0, + "point_set_range_composite": 2.0, + "range_affine_range_sum": 0.5, + "unionfind": 5.0, + "scc": 5.0, + "two_sat": 3.0, + # mf_graph.flow runs a full BFS per augmenting path rather than per phase, + # so unit-capacity matching costs O(V*E*flow): 196s at 20% of the judge's + # size. This factor keeps CI affordable; it is not a claim that the size is + # representative. + "bipartitematching": 0.15, + "assignment": 2.5, + "convolution_mod": 1.0, + "inv_of_formal_power_series": 1.0, + "log_of_formal_power_series": 0.5, + "exp_of_formal_power_series": 0.5, + "suffixarray": 5.0, + "zalgorithm": 5.0, + "number_of_substrings": 5.0, + "sum_of_floor_of_linear": 5.0, + "primality_test": 5.0, + "factorize": 5.0, + # The extras have no judge constraints to cap them against, so their + # "maximum" is just a size chosen to keep each one measurable (a workload + # that finishes in 20ms measures the timer, not the library). + "extra_segtree_binary_search": 1.0, + "extra_dsu_groups": 5.0, + "extra_maxflow_edges": 5.0, + "extra_mincostflow_slope": 5.0, + "extra_acl_math_crt": 5.0, + "extra_prime_fact_divisors": 1.0, +} + + +def _env_scale(): + raw = os.environ.get("ACL_BENCH_SCALE", "").strip().lower() + if not raw: + return DEFAULT_SCALE + if raw in ("max", "full"): + # Every factor is <= 5, so this pins every workload to its cap. + return float("inf") + try: + value = float(raw) + except ValueError: + raise ValueError( + f"ACL_BENCH_SCALE must be a positive float or 'max', got {raw!r}" + ) + if value <= 0: + raise ValueError(f"ACL_BENCH_SCALE must be positive, got {value}") + return value + + +SCALE = _env_scale() + + +def effective_scale(name): + """Fraction of the Library Checker maximum this workload actually runs at.""" + return min(1.0, SCALE * COST_FACTOR[name]) + + +def _scaled(n, name, minimum=1): + """Scale a Library Checker constraint down for CI.""" + return max(minimum, int(n * effective_scale(name))) def _rng(seed): return random.Random(seed) -# --------------------------------------------------------------------------- -# segtree: set / get / prod / all_prod / max_right / min_left on a large array -# --------------------------------------------------------------------------- -def build_segtree_workload(): +def _uniform_pair(rnd, lower, upper): + """Port of library-checker-problems' ``Random::uniform_pair``. + + Picks two distinct values from the *inclusive* range [lower, upper] and + returns them sorted, i.e. a non-empty subinterval [l, r) of [lower, upper). + This is not the same distribution as ``l = randrange(n); r = randrange(l, + n + 1)``: uniform_pair is symmetric in l and produces a mean width of + roughly n/3, which is what the judge's segment-tree queries actually cost. + """ + while True: + a = rnd.randint(lower, upper) + b = rnd.randint(lower, upper) + if a != b: + return (a, b) if a < b else (b, a) + + +class Workload: + """One benchmark: a Library Checker problem, or an uncalibrated extra.""" + + def __init__(self, name, module, build, run, full_size, problem=None, note=""): + self.name = name + self.module = module + self.build = build + self.run = run + self.full_size = full_size + self.problem = problem # Library Checker slug, or None for extras + self.note = note + + @property + def url(self): + return LIBRARY_CHECKER + self.problem if self.problem else None + + @property + def effective_scale(self): + """Fraction of ``full_size`` this run actually uses.""" + return effective_scale(self.name) + + def __iter__(self): + # Kept so existing `for name, build_fn, run_fn in BENCHMARKS` consumers + # keep working. + return iter((self.name, self.build, self.run)) + + +# =========================================================================== +# data_structure/point_add_range_sum -- fenwicktree +# =========================================================================== +def build_point_add_range_sum(): r = _rng(1) - n, q = 200_000, 200_000 - values = [r.randint(1, 10**9) for _ in range(n)] + n = q = _scaled(500_000, "point_add_range_sum") + a_max = 1_000_000_000 + a = [r.randint(0, a_max) for _ in range(n)] queries = [] for _ in range(q): - choice = r.random() - if choice < 0.3: - queries.append(("set", r.randrange(n), r.randint(1, 10**9))) - elif choice < 0.5: - queries.append(("get", r.randrange(n))) - elif choice < 0.75: - l = r.randrange(n) - queries.append(("prod", l, r.randrange(l, n + 1))) - elif choice < 0.8: - queries.append(("all_prod",)) - elif choice < 0.9: - queries.append(("max_right", r.randrange(n + 1), r.randint(1, 10**9))) + if r.randint(0, 1) == 0: + queries.append((0, r.randint(0, n - 1), r.randint(0, a_max))) else: - queries.append(("min_left", r.randrange(n + 1), r.randint(1, 10**9))) - return values, queries - - -def run_segtree(values, queries): - seg = segtree_mod.segtree(list(values), max, -1) - for q in queries: - op = q[0] - if op == "set": - seg.set(q[1], q[2]) - elif op == "get": - seg.get(q[1]) - elif op == "prod": - seg.prod(q[1], q[2]) - elif op == "all_prod": - seg.all_prod() - elif op == "max_right": - threshold = q[2] - seg.max_right(q[1], lambda x, th=threshold: x <= th) + queries.append((1,) + _uniform_pair(r, 0, n)) + return n, a, queries + + +def run_point_add_range_sum(n, a, queries): + ft = fenwicktree_mod.fenwick_tree(n) + for i in range(n): + ft.add(i, a[i]) + answer = 0 + for t, x, y in queries: + if t == 0: + ft.add(x, y) else: - threshold = q[2] - seg.min_left(q[1], lambda x, th=threshold: x <= th) - return seg + answer += ft.sum(x, y) + return answer -# --------------------------------------------------------------------------- -# lazysegtree: set / get / prod / all_prod / apply_point / apply / max_right / -# min_left on a large array (range-add, range-min) -# --------------------------------------------------------------------------- -def build_lazysegtree_workload(): +# =========================================================================== +# data_structure/staticrmq -- segtree with a C-level op +# =========================================================================== +def build_staticrmq(): r = _rng(2) - n, q = 100_000, 100_000 - values = [0] * n + n = q = _scaled(500_000, "staticrmq") + a = [r.randint(0, 1_000_000_000) for _ in range(n)] + queries = [_uniform_pair(r, 0, n) for _ in range(q)] + return a, queries + + +def run_staticrmq(a, queries): + seg = segtree_mod.segtree(a, min, INF) + answer = 0 + for l, rr in queries: + answer ^= seg.prod(l, rr) + return answer + + +# =========================================================================== +# data_structure/point_set_range_composite -- segtree with a Python op +# =========================================================================== +def build_point_set_range_composite(): + r = _rng(3) + n = q = _scaled(500_000, "point_set_range_composite") + f = [(r.randint(1, MOD - 1), r.randint(0, MOD - 1)) for _ in range(n)] queries = [] for _ in range(q): - choice = r.random() - if choice < 0.25: - l = r.randrange(n) - rr = r.randrange(l, n + 1) - queries.append(("apply", l, rr, r.randint(-1000, 1000))) - elif choice < 0.45: - l = r.randrange(n) - rr = r.randrange(l, n + 1) - queries.append(("prod", l, rr)) - elif choice < 0.55: - queries.append(("set", r.randrange(n), r.randint(-1000, 1000))) - elif choice < 0.65: - queries.append(("get", r.randrange(n))) - elif choice < 0.7: - queries.append(("all_prod",)) - elif choice < 0.8: - queries.append(("apply_point", r.randrange(n), r.randint(-1000, 1000))) - elif choice < 0.9: - queries.append(("max_right", r.randrange(n + 1), r.randint(-2000, 2000))) + if r.randint(0, 1) == 0: + queries.append( + (0, r.randint(0, n - 1), r.randint(1, MOD - 1), r.randint(0, MOD - 1)) + ) else: - queries.append(("min_left", r.randrange(n + 1), r.randint(-2000, 2000))) - return values, queries + l, rr = _uniform_pair(r, 0, n) + queries.append((1, l, rr, r.randint(0, MOD - 1))) + return f, queries -def run_lazysegtree(values, queries): - inf = float("inf") +def _compose(f, g): + # g after f: (g.a * f.a) x + (g.a * f.b + g.b) + return (f[0] * g[0] % MOD, (g[0] * f[1] + g[1]) % MOD) + + +def run_point_set_range_composite(f, queries): + seg = segtree_mod.segtree(list(f), _compose, (1, 0)) + answer = 0 + for query in queries: + if query[0] == 0: + _, p, c, d = query + seg.set(p, (c, d)) + else: + _, l, rr, x = query + a, b = seg.prod(l, rr) + answer += (a * x + b) % MOD + return answer + + +# =========================================================================== +# data_structure/range_affine_range_sum -- lazysegtree +# =========================================================================== +def build_range_affine_range_sum(): + r = _rng(4) + n = q = _scaled(500_000, "range_affine_range_sum") + a = [r.randint(0, MOD - 1) for _ in range(n)] + queries = [] + for _ in range(q): + l, rr = _uniform_pair(r, 0, n) + if r.randint(0, 1) == 0: + queries.append((0, l, rr, r.randint(1, MOD - 1), r.randint(0, MOD - 1))) + else: + queries.append((1, l, rr, 0, 0)) + return a, queries + + +def _raras_op(x, y): + return ((x[0] + y[0]) % MOD, x[1] + y[1]) + + +def _raras_mapping(f, x): + return ((f[0] * x[0] + f[1] * x[1]) % MOD, x[1]) + + +def _raras_composition(f, g): + # f after g + return (f[0] * g[0] % MOD, (f[0] * g[1] + f[1]) % MOD) + + +def run_range_affine_range_sum(a, queries): seg = lazysegtree_mod.lazy_segtree( - list(values), min, inf, lambda f, x: f + x, lambda f, g: f + g, 0 + [(v, 1) for v in a], + _raras_op, + (0, 0), + _raras_mapping, + _raras_composition, + (1, 0), ) - for q in queries: - op = q[0] - if op == "apply": - seg.apply(q[1], q[2], q[3]) - elif op == "prod": - seg.prod(q[1], q[2]) - elif op == "set": - seg.set(q[1], q[2]) - elif op == "get": - seg.get(q[1]) - elif op == "all_prod": - seg.all_prod() - elif op == "apply_point": - seg.apply_point(q[1], q[2]) - elif op == "max_right": - threshold = q[2] - # min is non-increasing as the range grows, so "value >= threshold" - # is a valid (monotonically falling) predicate for max_right/min_left. - seg.max_right(q[1], lambda x, th=threshold: x >= th) + answer = 0 + for t, l, rr, b, c in queries: + if t == 0: + seg.apply(l, rr, (b, c)) else: - threshold = q[2] - seg.min_left(q[1], lambda x, th=threshold: x >= th) - return seg + answer += seg.prod(l, rr)[0] + return answer -# --------------------------------------------------------------------------- -# dsu: leader / merge / same / size / groups on a large number of elements -# --------------------------------------------------------------------------- -def build_dsu_workload(): - r = _rng(3) - n = 300_000 - merge_ops = [(r.randrange(n), r.randrange(n)) for _ in range(n)] - leader_queries = [r.randrange(n) for _ in range(n)] - same_queries = [(r.randrange(n), r.randrange(n)) for _ in range(n)] - size_queries = [r.randrange(n) for _ in range(n)] - return n, merge_ops, leader_queries, same_queries, size_queries +# =========================================================================== +# data_structure/unionfind -- dsu +# =========================================================================== +def build_unionfind(): + r = _rng(5) + n = _scaled(200_000, "unionfind") + q = _scaled(200_000, "unionfind") + queries = [ + (r.randint(0, 1), r.randint(0, n - 1), r.randint(0, n - 1)) for _ in range(q) + ] + return n, queries -def run_dsu(n, merge_ops, leader_queries, same_queries, size_queries): +def run_unionfind(n, queries): d = dsu_mod.dsu(n) - for a, b in merge_ops: - d.merge(a, b) - for a in leader_queries: - d.leader(a) - for a, b in same_queries: - d.same(a, b) - for a in size_queries: - d.size(a) - d.groups() - return d - - -# --------------------------------------------------------------------------- -# maxflow: add_edge / get_edge / edges / change_edge / flow / min_cut on a -# random DAG (Dinic's algorithm) -# --------------------------------------------------------------------------- -def build_maxflow_workload(): - r = _rng(4) - n, m = 50_000, 200_000 - edges = [] - for _ in range(m): - u = r.randrange(n - 1) - v = r.randrange(u + 1, n) - edges.append((u, v, r.randint(1, 1000))) + answer = 0 + for t, u, v in queries: + if t == 0: + d.merge(u, v) + else: + answer += d.same(u, v) + return answer + + +# =========================================================================== +# graph/scc -- scc +# =========================================================================== +def build_scc(): + r = _rng(6) + n = _scaled(500_000, "scc") + m = _scaled(500_000, "scc") + edges = [(r.randint(0, n - 1), r.randint(0, n - 1)) for _ in range(m)] return n, edges -def run_maxflow(n, edges): - g = maxflow_mod.mf_graph(n) - for u, v, cap in edges: - g.add_edge(u, v, cap) - g.get_edge(0) - g.edges() - g.change_edge(0, 2000, 500) - g.flow(0, n - 1) - g.min_cut(0) - return g +def run_scc(n, edges): + groups = scc_mod.scc(n, edges) + return len(groups) -# --------------------------------------------------------------------------- -# mincostflow: add_edge / get_edge / edges / slope / flow on a random DAG -# (successive shortest paths) -# --------------------------------------------------------------------------- -def build_mincostflow_workload(): - r = _rng(5) - n, m = 5_000, 25_000 - edges = [] +# =========================================================================== +# other/two_sat -- two_sat +# =========================================================================== +def build_two_sat(): + r = _rng(7) + n = _scaled(500_000, "two_sat") + m = _scaled(500_000, "two_sat") + clause = [] for _ in range(m): - u = r.randrange(n - 1) - v = r.randrange(u + 1, n) - edges.append((u, v, r.randint(1, 20), r.randint(1, 100))) - return n, edges + i = r.randint(0, n - 1) + j = r.randint(0, n - 1) + clause.append((i, r.randint(0, 1) == 1, j, r.randint(0, 1) == 1)) + return n, clause -def run_mincostflow(n, edges): - g = mincostflow_mod.mcf_graph(n) - for u, v, cap, cost in edges: - g.add_edge(u, v, cap, cost) - g.slope(0, n - 1) - g.get_edge(0) - g.edges() - g.flow(0, n - 1) - return g +def run_two_sat(n, clause): + answer = two_sat_mod.two_sat(n, clause) + if answer is None: # UNSATISFIABLE + return -1 + return sum(answer) -# --------------------------------------------------------------------------- -# convolution: NTT-based convolution of two large arrays -# --------------------------------------------------------------------------- -def build_convolution_workload(): - r = _rng(6) - n = 1 << 16 - a = [r.randrange(MOD) for _ in range(n)] - b = [r.randrange(MOD) for _ in range(n)] +# =========================================================================== +# graph/bipartitematching -- maxflow (Dinic) +# =========================================================================== +def build_bipartitematching(): + r = _rng(8) + left = _scaled(100_000, "bipartitematching") + right = _scaled(100_000, "bipartitematching") + m = _scaled(200_000, "bipartitematching") + # The generator inserts into a std::set, so duplicate edges collapse. + edges = {(r.randint(0, left - 1), r.randint(0, right - 1)) for _ in range(m)} + return left, right, sorted(edges) + + +def run_bipartitematching(left, right, edges): + s = left + right + t = s + 1 + g = maxflow_mod.mf_graph(t + 1) + for i in range(left): + g.add_edge(s, i, 1) + for j in range(right): + g.add_edge(left + j, t, 1) + for u, v in edges: + g.add_edge(u, left + v, 1) + return g.flow(s, t) + + +# =========================================================================== +# graph/assignment -- mincostflow (successive shortest paths) +# =========================================================================== +def build_assignment(): + r = _rng(9) + n = _scaled(500, "assignment") + a_abs_max = 1_000_000_000 + a = [[r.randint(-a_abs_max, a_abs_max) for _ in range(n)] for _ in range(n)] + return n, a + + +def run_assignment(n, a): + # mcf_graph needs non-negative costs, so shift by the constraint bound the + # way a real submission does and subtract the shift back out at the end. + shift = 1_000_000_000 + s = 2 * n + t = s + 1 + g = mincostflow_mod.mcf_graph(t + 1) + for i in range(n): + g.add_edge(s, i, 1, 0) + g.add_edge(n + i, t, 1, 0) + for i in range(n): + row = a[i] + for j in range(n): + g.add_edge(i, n + j, 1, row[j] + shift) + flow, cost = g.flow(s, t) + return cost - flow * shift + + +# =========================================================================== +# convolution/convolution_mod -- convolution (NTT) +# =========================================================================== +def build_convolution_mod(): + r = _rng(10) + n = _scaled(524_288, "convolution_mod") + m = _scaled(524_288, "convolution_mod") + a = [r.randint(0, MOD - 1) for _ in range(n)] + b = [r.randint(0, MOD - 1) for _ in range(m)] return a, b -def run_convolution(a, b): +def run_convolution_mod(a, b): fft = convolution_mod.FFT(MOD) - return fft.convolution(a, b) + c = fft.convolution(a, b) + return c[0] ^ c[-1] -# --------------------------------------------------------------------------- -# scc: Tarjan's algorithm on a large random directed graph -# --------------------------------------------------------------------------- -def build_scc_workload(): - r = _rng(7) - n, m = 120_000, 360_000 - edges = [(r.randrange(n), r.randrange(n)) for _ in range(m)] - return n, edges +# =========================================================================== +# polynomial/{inv,log,exp}_of_formal_power_series -- fps +# =========================================================================== +def build_inv_of_formal_power_series(): + r = _rng(11) + n = _scaled(500_000, "inv_of_formal_power_series") + return ([r.randint(1, MOD - 1)] + [r.randint(0, MOD - 1) for _ in range(n - 1)],) -def run_scc(n, edges): - return scc_mod.scc(n, edges) +def run_inv_of_formal_power_series(a): + return fps_mod.FPS(a).inv(len(a)).Func[-1] -# --------------------------------------------------------------------------- -# fps: formal power series ops (exp/log/diff/integral/add/sub) on a large -# random series -# --------------------------------------------------------------------------- -def build_fps_workload(): - r = _rng(8) - n = 8_192 - seq = [0] + [r.randrange(MOD) for _ in range(n - 1)] - return (seq,) +def build_log_of_formal_power_series(): + r = _rng(12) + n = _scaled(500_000, "log_of_formal_power_series") + return ([1] + [r.randint(0, MOD - 1) for _ in range(n - 1)],) -def run_fps(seq): - a = fps_mod.FPS(seq) - b = a.exp() - c = b.log() - d = c.diff() - e = d.integral() - g = (b + a) - a - return c.resize(len(seq)), e, g +def run_log_of_formal_power_series(a): + return fps_mod.FPS(a).log().resize(len(a)).Func[-1] -# --------------------------------------------------------------------------- -# fenwicktree: add / sum on a large array -# --------------------------------------------------------------------------- -def build_fenwicktree_workload(): - r = _rng(9) - n, q = 300_000, 300_000 - add_ops = [(r.randrange(n), r.randint(-1000, 1000)) for _ in range(q)] - sum_ops = [] - for _ in range(q): - l = r.randrange(n) - rr = r.randrange(l, n + 1) - sum_ops.append((l, rr)) - return n, add_ops, sum_ops +def build_exp_of_formal_power_series(): + r = _rng(13) + n = _scaled(500_000, "exp_of_formal_power_series") + return ([0] + [r.randint(0, MOD - 1) for _ in range(n - 1)],) -def run_fenwicktree(n, add_ops, sum_ops): - ft = fenwicktree_mod.fenwick_tree(n) - for p, x in add_ops: - ft.add(p, x) - for l, rr in sum_ops: - ft.sum(l, rr) - return ft - - -# --------------------------------------------------------------------------- -# acl_math: inv_gcd / inv_mod / crt / floor_sum on a large batch of random -# queries -# --------------------------------------------------------------------------- -_CRT_PRIMES = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37) -_MOD_BIG = (1 << 61) - 1 +def run_exp_of_formal_power_series(a): + return fps_mod.FPS(a).exp(len(a)).Func[-1] -def build_acl_math_workload(): - r = _rng(10) - q = 100_000 - inv_mod_queries = [r.randrange(1, _MOD_BIG) for _ in range(q)] - inv_gcd_queries = [(r.randrange(1, 10**9), r.randint(1, 10**9)) for _ in range(q)] - crt_queries = [[r.randrange(p) for p in _CRT_PRIMES] for _ in range(q)] - floor_sum_queries = [ - ( - r.randint(1, 10**6), - r.randint(1, 10**6), - r.randint(0, 10**6), - r.randint(0, 10**6), - ) - for _ in range(q) +# =========================================================================== +# string/{suffixarray,zalgorithm,number_of_substrings} -- acl_string +# =========================================================================== +_LOWER = "abcdefghijklmnopqrstuvwxyz" + + +def build_suffixarray(): + r = _rng(14) + n = _scaled(500_000, "suffixarray") + return ("".join(r.choice(_LOWER) for _ in range(n)),) + + +def run_suffixarray(s): + sa = acl_string_mod.String(s).suffix_array() + return sa[0] ^ sa[-1] + + +def build_zalgorithm(): + r = _rng(15) + n = _scaled(500_000, "zalgorithm") + return ("".join(r.choice(_LOWER) for _ in range(n)),) + + +def run_zalgorithm(s): + return sum(acl_string_mod.String(s).z_algorithm()) + + +def build_number_of_substrings(): + r = _rng(16) + n = _scaled(500_000, "number_of_substrings") + return ("".join(r.choice(_LOWER) for _ in range(n)),) + + +def run_number_of_substrings(s): + st = acl_string_mod.String(s) + n = len(s) + sa = st.suffix_array() + return n * (n + 1) // 2 - sum(st.lcp_array(sa)) + + +# =========================================================================== +# number_theory/sum_of_floor_of_linear -- acl_math.floor_sum +# =========================================================================== +def build_sum_of_floor_of_linear(): + r = _rng(17) + t = _scaled(100_000, "sum_of_floor_of_linear") + cases = [] + for _ in range(t): + n = r.randint(1, 10**9) + m = r.randint(1, 10**9) + cases.append((n, m, r.randint(0, m - 1), r.randint(0, m - 1))) + return (cases,) + + +def run_sum_of_floor_of_linear(cases): + answer = 0 + for n, m, a, b in cases: + answer += acl_math_mod.floor_sum(n, m, a, b) + return answer + + +# =========================================================================== +# number_theory/primality_test -- prime_fact.is_probable_prime +# =========================================================================== +def build_primality_test(): + r = _rng(18) + q = _scaled(100_000, "primality_test") + return ([r.randint(1, 10**18) for _ in range(q)],) + + +def run_primality_test(values): + is_probable_prime = prime_fact_mod.is_probable_prime + return sum(1 for n in values if is_probable_prime(n)) + + +# =========================================================================== +# number_theory/factorize -- prime_fact.prime_fact (Pollard's rho) +# =========================================================================== +def build_factorize(): + # gen/max.cpp emits consecutive integers just below 10^18; those are the + # timing-relevant cases, far harder than uniformly random ones. + q = _scaled(100, "factorize") + return ([10**18 - i for i in range(q)],) + + +def run_factorize(values): + answer = 0 + for n in values: + answer += sum(prime_fact_mod.prime_fact(n).values()) + return answer + + +# =========================================================================== +# Extras: public methods no Library Checker problem exercises. +# Regression net only -- these timings predict nothing about judge runtime. +# =========================================================================== +def build_extra_segtree_binary_search(): + r = _rng(101) + n = _scaled(200_000, "extra_segtree_binary_search") + q = _scaled(200_000, "extra_segtree_binary_search") + # A prefix-sum monoid with a threshold drawn from the whole range of + # attainable sums: "longest prefix summing to at most x" is the standard + # use of max_right, and it forces a full O(log n) descent. A max-monoid + # with an independently random threshold would fail the predicate on the + # first element almost every time and measure nothing. + a = [r.randint(1, 1000) for _ in range(n)] + total = sum(a) + queries = [ + (r.randint(0, 1), r.randint(0, n), r.randint(0, total)) for _ in range(q) ] - return inv_mod_queries, inv_gcd_queries, crt_queries, floor_sum_queries + return a, queries -def run_acl_math(inv_mod_queries, inv_gcd_queries, crt_queries, floor_sum_queries): +def _add(x, y): + return x + y + + +def run_extra_segtree_binary_search(a, queries): + seg = segtree_mod.segtree(a, _add, 0) + answer = 0 + for t, p, threshold in queries: + if t == 0: + answer += seg.max_right(p, lambda x, th=threshold: x <= th) + else: + answer += seg.min_left(p, lambda x, th=threshold: x <= th) + return answer + + +def build_extra_dsu_groups(): + r = _rng(102) + n = _scaled(200_000, "extra_dsu_groups") + merges = [(r.randint(0, n - 1), r.randint(0, n - 1)) for _ in range(n // 2)] + sizes = [r.randint(0, n - 1) for _ in range(n)] + return n, merges, sizes + + +def run_extra_dsu_groups(n, merges, sizes): + d = dsu_mod.dsu(n) + for a, b in merges: + d.merge(a, b) + answer = 0 + for a in sizes: + answer += d.size(a) + for a in sizes: + answer ^= d.leader(a) + return answer + len(d.groups()) + + +def build_extra_maxflow_edges(): + r = _rng(103) + n = _scaled(20_000, "extra_maxflow_edges") + m = _scaled(60_000, "extra_maxflow_edges") + edges = [] + for _ in range(m): + u = r.randint(0, n - 2) + v = r.randint(u + 1, n - 1) + edges.append((u, v, r.randint(1, 1000))) + return n, edges + + +def run_extra_maxflow_edges(n, edges): + g = maxflow_mod.mf_graph(n) + for u, v, cap in edges: + g.add_edge(u, v, cap) + g.change_edge(0, 2000, 500) + g.get_edge(0) + flow = g.flow(0, n - 1) + return flow + len(g.edges()) + sum(g.min_cut(0)) + + +def build_extra_mincostflow_slope(): + r = _rng(104) + n = _scaled(2_000, "extra_mincostflow_slope") + m = _scaled(8_000, "extra_mincostflow_slope") + # In a purely random DAG the source has a handful of outgoing edges, so + # slope() finishes after two augmentations and measures nothing. Fan out of + # the source and into the sink so the successive-shortest-path loop runs. + width = max(1, n // 10) + edges = [] + for v in range(1, width + 1): + edges.append((0, v, r.randint(1, 20), r.randint(1, 100))) + edges.append((n - 1 - v, n - 1, r.randint(1, 20), r.randint(1, 100))) + for _ in range(m): + u = r.randint(1, n - 3) + v = r.randint(u + 1, n - 2) + edges.append((u, v, r.randint(1, 20), r.randint(1, 100))) + return n, edges + + +def run_extra_mincostflow_slope(n, edges): + g = mincostflow_mod.mcf_graph(n) + for u, v, cap, cost in edges: + g.add_edge(u, v, cap, cost) + slope = g.slope(0, n - 1) + g.get_edge(0) + return len(slope) + len(g.edges()) + + +_CRT_MODS = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37) +_MOD_BIG = (1 << 61) - 1 + + +def build_extra_acl_math_crt(): + r = _rng(105) + q = _scaled(100_000, "extra_acl_math_crt") + inv_mod_queries = [r.randint(1, _MOD_BIG) for _ in range(q)] + inv_gcd_queries = [(r.randint(1, 10**9), r.randint(1, 10**9)) for _ in range(q)] + crt_queries = [[r.randrange(p) for p in _CRT_MODS] for _ in range(q // 10)] + return inv_mod_queries, inv_gcd_queries, crt_queries + + +def run_extra_acl_math_crt(inv_mod_queries, inv_gcd_queries, crt_queries): + answer = 0 for x in inv_mod_queries: - acl_math_mod.inv_mod(x, _MOD_BIG) + answer ^= acl_math_mod.inv_mod(x, _MOD_BIG) for a, b in inv_gcd_queries: - acl_math_mod.inv_gcd(a, b) + answer ^= acl_math_mod.inv_gcd(a, b)[0] + mods = list(_CRT_MODS) for rems in crt_queries: - acl_math_mod.crt(rems, list(_CRT_PRIMES)) - for n, m, a, b in floor_sum_queries: - acl_math_mod.floor_sum(n, m, a, b) + answer ^= acl_math_mod.crt(rems, mods)[0] + return answer -# --------------------------------------------------------------------------- -# acl_string: suffix_array / lcp_array / z_algorithm on a large random string -# --------------------------------------------------------------------------- -def build_acl_string_workload(): - r = _rng(11) - n = 100_000 - s = "".join(r.choice("abcd") for _ in range(n)) - return (s,) +def build_extra_prime_fact_divisors(): + r = _rng(106) + values = [ + r.randint(2, 10**7) + for _ in range(_scaled(10_000, "extra_prime_fact_divisors", minimum=100)) + ] + lcm_queries = [ + (r.randint(1, 10**6), r.randint(1, 10**6)) + for _ in range(_scaled(10_000, "extra_prime_fact_divisors", minimum=100)) + ] + return values, lcm_queries -def run_acl_string(s): - st = acl_string_mod.String(s) - sa = st.suffix_array() - lcp = st.lcp_array(sa) - z = st.z_algorithm() - len(st) - st[0] - str(st) - repr(st) - return sa, lcp, z - - -# --------------------------------------------------------------------------- -# prime_fact: is_probable_prime / prime_fact / divisors / totient / lcm on a -# large batch of random numbers -# --------------------------------------------------------------------------- -def build_prime_fact_workload(): - r = _rng(12) - is_prime_queries = [r.randrange(2, 10**7) for _ in range(20_000)] - fact_queries = [r.randrange(2, 10**7) for _ in range(2_000)] - lcm_queries = [(r.randrange(1, 10**6), r.randrange(1, 10**6)) for _ in range(2_000)] - return is_prime_queries, fact_queries, lcm_queries - - -def run_prime_fact(is_prime_queries, fact_queries, lcm_queries): - for n in is_prime_queries: - prime_fact_mod.is_probable_prime(n) - for n in fact_queries: - prime_fact_mod.prime_fact(n) - prime_fact_mod.divisors(n) - prime_fact_mod.totient(n) +def run_extra_prime_fact_divisors(values, lcm_queries): + answer = 0 + for n in values: + answer += len(prime_fact_mod.divisors(n)) + answer += prime_fact_mod.totient(n) for x, y in lcm_queries: - prime_fact_mod.lcm(x, y) + answer ^= prime_fact_mod.lcm(x, y) + return answer + + +# =========================================================================== +# Registry +# =========================================================================== +LIBRARY_CHECKER_BENCHMARKS = [ + Workload( + "point_add_range_sum", + "fenwicktree", + build_point_add_range_sum, + run_point_add_range_sum, + "N = Q = 500,000", + problem="point_add_range_sum", + ), + Workload( + "staticrmq", + "segtree", + build_staticrmq, + run_staticrmq, + "N = Q = 500,000", + problem="staticrmq", + note="segtree with a C-level op (min); measures tree overhead", + ), + Workload( + "point_set_range_composite", + "segtree", + build_point_set_range_composite, + run_point_set_range_composite, + "N = Q = 500,000", + problem="point_set_range_composite", + note="segtree with a Python-level op; measures op-dominated cost", + ), + Workload( + "range_affine_range_sum", + "lazysegtree", + build_range_affine_range_sum, + run_range_affine_range_sum, + "N = Q = 500,000", + problem="range_affine_range_sum", + ), + Workload( + "unionfind", + "dsu", + build_unionfind, + run_unionfind, + "N = Q = 200,000", + problem="unionfind", + ), + Workload( + "scc", + "scc", + build_scc, + run_scc, + "N = M = 500,000", + problem="scc", + ), + Workload( + "two_sat", + "two_sat", + build_two_sat, + run_two_sat, + "N = M = 500,000", + problem="two_sat", + ), + Workload( + "bipartitematching", + "maxflow", + build_bipartitematching, + run_bipartitematching, + "L = R = 100,000, M = 200,000", + problem="bipartitematching", + ), + Workload( + "assignment", + "mincostflow", + build_assignment, + run_assignment, + "N = 500", + problem="assignment", + note="O(N^3)-ish in CPython; ACL_BENCH_SCALE=1 is far past the judge's TL", + ), + Workload( + "convolution_mod", + "convolution", + build_convolution_mod, + run_convolution_mod, + "N = M = 524,288", + problem="convolution_mod", + ), + Workload( + "inv_of_formal_power_series", + "fps", + build_inv_of_formal_power_series, + run_inv_of_formal_power_series, + "N = 500,000", + problem="inv_of_formal_power_series", + ), + Workload( + "log_of_formal_power_series", + "fps", + build_log_of_formal_power_series, + run_log_of_formal_power_series, + "N = 500,000", + problem="log_of_formal_power_series", + ), + Workload( + "exp_of_formal_power_series", + "fps", + build_exp_of_formal_power_series, + run_exp_of_formal_power_series, + "N = 500,000", + problem="exp_of_formal_power_series", + ), + Workload( + "suffixarray", + "acl_string", + build_suffixarray, + run_suffixarray, + "N = 500,000", + problem="suffixarray", + ), + Workload( + "zalgorithm", + "acl_string", + build_zalgorithm, + run_zalgorithm, + "N = 500,000", + problem="zalgorithm", + ), + Workload( + "number_of_substrings", + "acl_string", + build_number_of_substrings, + run_number_of_substrings, + "N = 500,000", + problem="number_of_substrings", + ), + Workload( + "sum_of_floor_of_linear", + "acl_math", + build_sum_of_floor_of_linear, + run_sum_of_floor_of_linear, + "T = 100,000", + problem="sum_of_floor_of_linear", + ), + Workload( + "primality_test", + "prime_fact", + build_primality_test, + run_primality_test, + "Q = 100,000, N <= 10^18", + problem="primality_test", + ), + Workload( + "factorize", + "prime_fact", + build_factorize, + run_factorize, + "Q = 100, A <= 10^18", + problem="factorize", + ), +] + +EXTRA_BENCHMARKS = [ + Workload( + "extra_segtree_binary_search", + "segtree", + build_extra_segtree_binary_search, + run_extra_segtree_binary_search, + "N = Q = 200,000", + note="max_right / min_left", + ), + Workload( + "extra_dsu_groups", + "dsu", + build_extra_dsu_groups, + run_extra_dsu_groups, + "N = 200,000", + note="size / leader / groups", + ), + Workload( + "extra_maxflow_edges", + "maxflow", + build_extra_maxflow_edges, + run_extra_maxflow_edges, + "N = 20,000, M = 60,000", + note="get_edge / change_edge / edges / min_cut", + ), + Workload( + "extra_mincostflow_slope", + "mincostflow", + build_extra_mincostflow_slope, + run_extra_mincostflow_slope, + "N = 2,000, M = 8,000", + note="slope / get_edge / edges", + ), + Workload( + "extra_acl_math_crt", + "acl_math", + build_extra_acl_math_crt, + run_extra_acl_math_crt, + "Q = 100,000", + note="inv_mod / inv_gcd / crt", + ), + Workload( + "extra_prime_fact_divisors", + "prime_fact", + build_extra_prime_fact_divisors, + run_extra_prime_fact_divisors, + "Q = 10,000", + note="divisors / totient / lcm", + ), +] +BENCHMARKS = LIBRARY_CHECKER_BENCHMARKS + EXTRA_BENCHMARKS -# --------------------------------------------------------------------------- -# two_sat: a large random 2-SAT instance -# --------------------------------------------------------------------------- -def build_two_sat_workload(): - r = _rng(13) - n, m = 200_000, 400_000 - clause = [ - ( - r.randrange(n), - r.choice([True, False]), - r.randrange(n), - r.choice([True, False]), +BY_NAME = {w.name: w for w in BENCHMARKS} + + +def _env_only(): + """``ACL_BENCH_ONLY``: run just these workloads, comma separated. + + Pull requests only benchmark the modules they touch (see select_workloads.py), so + both the time and the memory runner need the same filter -- otherwise the + report would compare a subset against a full run. + """ + raw = os.environ.get("ACL_BENCH_ONLY", "").strip() + if not raw: + return list(BENCHMARKS) + names = [n.strip() for n in raw.split(",") if n.strip()] + unknown = [n for n in names if n not in BY_NAME] + if unknown: + raise ValueError( + f"ACL_BENCH_ONLY names no such workload: {', '.join(sorted(unknown))}" ) - for _ in range(m) - ] - return n, clause + wanted = set(names) + # Registry order, so the report reads the same however the list was written. + return [w for w in BENCHMARKS if w.name in wanted] -def run_two_sat(n, clause): - return two_sat_mod.two_sat(n, clause) - - -BENCHMARKS = [ - ("segtree", build_segtree_workload, run_segtree), - ("lazysegtree", build_lazysegtree_workload, run_lazysegtree), - ("dsu", build_dsu_workload, run_dsu), - ("maxflow", build_maxflow_workload, run_maxflow), - ("mincostflow", build_mincostflow_workload, run_mincostflow), - ("convolution", build_convolution_workload, run_convolution), - ("scc", build_scc_workload, run_scc), - ("fps", build_fps_workload, run_fps), - ("fenwicktree", build_fenwicktree_workload, run_fenwicktree), - ("acl_math", build_acl_math_workload, run_acl_math), - ("acl_string", build_acl_string_workload, run_acl_string), - ("prime_fact", build_prime_fact_workload, run_prime_fact), - ("two_sat", build_two_sat_workload, run_two_sat), -] +SELECTED = _env_only() diff --git a/tests/test_benchmark_compare.py b/tests/test_benchmark_compare.py new file mode 100644 index 0000000..5012847 --- /dev/null +++ b/tests/test_benchmark_compare.py @@ -0,0 +1,215 @@ +"""Tests for the benchmark comparison report. + +The report is what people act on, so its thresholds need to hold: a real win +must be reported, and run-to-run noise must not be. +""" + +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert( + 0, + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "benchmarks" + ), +) + +import compare # noqa: E402 + + +def write_time_json(path, samples, name="staticrmq", problem="staticrmq"): + """One pytest-benchmark JSON file; ``samples`` maps name -> stats.min.""" + payload = { + "benchmarks": [ + { + "name": f"test_benchmark[{n}]", + "stats": {"min": v, "mean": v * 1.2}, + "extra_info": { + "problem": problem if n == name else "", + "full_size": "N = Q = 500,000", + "scale": 0.6, + "global_scale": "0.2", + "note": "", + }, + } + for n, v in samples.items() + ] + } + with open(path, "w") as f: + json.dump(payload, f) + + +class TestNameParsing(unittest.TestCase): + def test_parametrised_id(self): + self.assertEqual(compare._benchmark_name("test_benchmark[scc]"), "scc") + + def test_plain_test_name(self): + self.assertEqual(compare._benchmark_name("test_scc"), "scc") + + def test_unrecognised_name_passes_through(self): + self.assertEqual(compare._benchmark_name("scc"), "scc") + + +class TestSpread(unittest.TestCase): + def test_single_sample_has_no_measurable_noise(self): + self.assertEqual(compare.spread([1.0]), 0.0) + + def test_relative_gap_between_best_and_worst(self): + self.assertAlmostEqual(compare.spread([1.0, 1.5, 1.2]), 0.5) + + +class TestDeltaThresholds(unittest.TestCase): + def test_real_improvement_is_flagged(self): + self.assertIn("โœ…", compare.fmt_delta(1.0, 0.7, 0.03)) + + def test_real_regression_is_flagged(self): + self.assertIn("โš ๏ธ", compare.fmt_delta(1.0, 1.4, 0.03)) + + def test_small_delta_is_not_flagged(self): + out = compare.fmt_delta(1.0, 0.98, 0.03) + self.assertIn("โ‰ˆ", out) + self.assertNotIn("โœ…", out) + + def test_delta_below_measured_noise_is_not_flagged(self): + # 10% faster, but the passes themselves varied by 20%. + out = compare.fmt_delta(1.0, 0.90, 0.20) + self.assertIn("โ‰ˆ", out) + self.assertNotIn("โœ…", out) + + def test_identical_values_report_no_change(self): + self.assertEqual(compare.fmt_delta(1.0, 1.0, 0.03), "0.0%") + + +class TestReport(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + + def path(self, name): + return os.path.join(self.dir, name) + + def render(self, base_passes, pr_passes): + base_files, pr_files = [], [] + for i, samples in enumerate(base_passes): + p = self.path(f"base-{i}.json") + write_time_json(p, samples) + base_files.append(p) + for i, samples in enumerate(pr_passes): + p = self.path(f"pr-{i}.json") + write_time_json(p, samples) + pr_files.append(p) + for p in ("base-mem.json", "pr-mem.json"): + with open(self.path(p), "w") as f: + json.dump({"benchmarks": []}, f) + out = self.path("report.md") + sys.argv = [ + "compare.py", + "--base-time", + *base_files, + "--pr-time", + *pr_files, + "--base-memory", + self.path("base-mem.json"), + "--pr-memory", + self.path("pr-mem.json"), + "--output", + out, + ] + with contextlib.redirect_stdout(io.StringIO()): + compare.main() + with open(out) as f: + return f.read() + + @staticmethod + def row(report, name): + """The table row for one benchmark, so assertions never match the + legend text (which itself mentions โœ… and โš ๏ธ).""" + for line in report.splitlines(): + if line.startswith("| ") and f"| {name} " in line.replace("[", "").replace( + "]", " " + ).replace("(https://judge.yosupo.jp/problem/", " ("): + return line + raise AssertionError(f"no row for {name!r} in:\n{report}") + + def test_minimum_is_taken_across_passes(self): + # A single slow pass must not drag the reported time up. + report = self.render( + [{"staticrmq": 1.0}, {"staticrmq": 5.0}], + [{"staticrmq": 1.0}, {"staticrmq": 1.0}], + ) + self.assertIn("1.000s", report) + self.assertNotIn("5.000s", report) + + def test_consistent_win_across_quiet_passes_is_flagged(self): + report = self.render( + [{"staticrmq": 1.00}, {"staticrmq": 1.01}], + [{"staticrmq": 0.70}, {"staticrmq": 0.71}], + ) + self.assertIn("โœ…", self.row(report, "staticrmq")) + + def test_win_smaller_than_the_noise_is_not_flagged(self): + report = self.render( + [{"staticrmq": 1.00}, {"staticrmq": 1.40}], + [{"staticrmq": 0.92}, {"staticrmq": 1.30}], + ) + row = self.row(report, "staticrmq") + self.assertNotIn("โœ…", row) + self.assertIn("โ‰ˆ", row) + + def test_extras_are_separated_from_calibrated_workloads(self): + report = self.render( + [{"staticrmq": 1.0, "extra_dsu_groups": 1.0}], + [{"staticrmq": 1.0, "extra_dsu_groups": 1.0}], + ) + self.assertIn("API coverage workloads", report) + head, tail = report.split("API coverage workloads", 1) + self.assertIn("staticrmq", head) + self.assertIn("extra_dsu_groups", tail) + + def test_missing_side_renders_as_na(self): + report = self.render([{}], [{"staticrmq": 1.0}]) + self.assertIn("| n/a |", report) + + def test_mismatched_scales_are_called_out(self): + base = self.path("base-0.json") + pr = self.path("pr-0.json") + write_time_json(base, {"staticrmq": 1.0}) + write_time_json(pr, {"staticrmq": 1.0}) + with open(pr) as f: + data = json.load(f) + data["benchmarks"][0]["extra_info"]["global_scale"] = "1.0" + with open(pr, "w") as f: + json.dump(data, f) + for p in ("base-mem.json", "pr-mem.json"): + with open(self.path(p), "w") as f: + json.dump({"benchmarks": []}, f) + out = self.path("report.md") + sys.argv = [ + "compare.py", + "--base-time", + base, + "--pr-time", + pr, + "--base-memory", + self.path("base-mem.json"), + "--pr-memory", + self.path("pr-mem.json"), + "--output", + out, + ] + with contextlib.redirect_stdout(io.StringIO()): + compare.main() + with open(out) as f: + self.assertIn("different `ACL_BENCH_SCALE`", f.read()) + + def test_problem_links_are_rendered(self): + report = self.render([{"staticrmq": 1.0}], [{"staticrmq": 1.0}]) + self.assertIn("https://judge.yosupo.jp/problem/staticrmq", report) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_benchmark_select.py b/tests/test_benchmark_select.py new file mode 100644 index 0000000..baa597e --- /dev/null +++ b/tests/test_benchmark_select.py @@ -0,0 +1,128 @@ +"""Tests for diff-based benchmark selection. + +Selecting too little silently stops measuring a module; selecting too much +burns the Actions time this exists to save. Both directions are checked. +""" + +import importlib +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert( + 0, + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "benchmarks" + ), +) + +import select_workloads as select_mod # noqa: E402 +import workloads as W # noqa: E402 + + +class TestSelect(unittest.TestCase): + def test_unrelated_change_selects_nothing(self): + self.assertEqual(select_mod.select(["README.md", "pyproject.toml"]), []) + + def test_empty_diff_selects_nothing(self): + self.assertEqual(select_mod.select([]), []) + self.assertEqual(select_mod.select(["", " "]), []) + + def test_module_change_selects_only_its_workloads(self): + names = select_mod.select(["segtree.py"]) + self.assertEqual( + names, + ["staticrmq", "point_set_range_composite", "extra_segtree_binary_search"], + ) + + def test_selection_covers_every_workload_of_the_module(self): + for module in {w.module for w in W.BENCHMARKS}: + names = set(select_mod.select([f"{module}.py"])) + expected = {w.name for w in W.BENCHMARKS if w.module == module} + self.assertEqual(names, expected, module) + + def test_multiple_modules_are_unioned(self): + names = select_mod.select(["dsu.py", "scc.py"]) + self.assertEqual(names, ["unionfind", "scc", "extra_dsu_groups"]) + + def test_result_is_in_registry_order(self): + order = [w.name for w in W.BENCHMARKS] + names = select_mod.select(["prime_fact.py", "segtree.py", "fps.py"]) + self.assertEqual(names, [n for n in order if n in set(names)]) + + def test_workload_change_selects_everything(self): + self.assertEqual( + select_mod.select(["benchmarks/workloads.py"]), + [w.name for w in W.BENCHMARKS], + ) + + def test_harness_change_selects_everything(self): + for path in ( + "benchmarks/test_time.py", + "benchmarks/memory_bench.py", + "benchmarks/select_workloads.py", + ".github/workflows/benchmark.yml", + ): + self.assertEqual(len(select_mod.select([path])), len(W.BENCHMARKS), path) + + def test_report_only_change_selects_nothing(self): + # compare.py renders the report; it cannot change a measurement. + self.assertEqual(select_mod.select(["benchmarks/compare.py"]), []) + + def test_leading_dot_slash_is_tolerated(self): + self.assertEqual(select_mod.select(["./dsu.py"]), select_mod.select(["dsu.py"])) + + def test_nested_file_with_a_module_name_is_not_matched(self): + # tests/segtree.py is not the library module. + self.assertEqual(select_mod.select(["tests/segtree.py"]), []) + + def test_every_module_file_is_mapped(self): + for w in W.BENCHMARKS: + self.assertIn(f"{w.module}.py", select_mod.MODULE_FILE) + + +class TestOnlyFilter(unittest.TestCase): + """``ACL_BENCH_ONLY`` is how the selection reaches both runners.""" + + def reload_with(self, value): + if value is None: + os.environ.pop("ACL_BENCH_ONLY", None) + else: + os.environ["ACL_BENCH_ONLY"] = value + importlib.reload(W) + + def tearDown(self): + self.reload_with(None) + + def test_unset_runs_everything(self): + self.reload_with(None) + self.assertEqual(len(W.SELECTED), len(W.BENCHMARKS)) + + def test_blank_runs_everything(self): + self.reload_with(" ") + self.assertEqual(len(W.SELECTED), len(W.BENCHMARKS)) + + def test_filters_to_the_named_workloads(self): + self.reload_with("scc,unionfind") + self.assertEqual([w.name for w in W.SELECTED], ["unionfind", "scc"]) + + def test_whitespace_is_tolerated(self): + self.reload_with(" scc , unionfind ") + self.assertEqual([w.name for w in W.SELECTED], ["unionfind", "scc"]) + + def test_unknown_name_is_rejected(self): + # A typo would silently benchmark nothing, and the report would look + # like a clean run. + with self.assertRaises(ValueError) as ctx: + self.reload_with("segtree") + self.assertIn("segtree", str(ctx.exception)) + + def test_selection_output_is_accepted_by_the_filter(self): + names = select_mod.select(["segtree.py"]) + self.reload_with(",".join(names)) + self.assertEqual([w.name for w in W.SELECTED], names) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_benchmark_workloads.py b/tests/test_benchmark_workloads.py new file mode 100644 index 0000000..26400e7 --- /dev/null +++ b/tests/test_benchmark_workloads.py @@ -0,0 +1,386 @@ +"""Correctness tests for the benchmark workloads. + +A benchmark is only meaningful if the code it times actually solves the +problem it claims to. Without these tests an "optimisation" that quietly +breaks the library would show up as a large, celebrated speedup. + +Each test drives a ``benchmarks.workloads.run_*`` function with hand-built +inputs in the same shape its ``build_*`` counterpart produces, and checks the +answer against a naive reference. +""" + +import copy +import importlib +import itertools +import os +import random +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert( + 0, + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "benchmarks" + ), +) + +import workloads as W # noqa: E402 + +MOD = W.MOD + + +class TestUniformPair(unittest.TestCase): + def test_returns_non_empty_subinterval(self): + r = random.Random(0) + for n in (1, 2, 5, 50): + for _ in range(200): + l, rr = W._uniform_pair(r, 0, n) + self.assertTrue(0 <= l < rr <= n, (l, rr, n)) + + def test_covers_both_endpoints(self): + r = random.Random(1) + seen = {W._uniform_pair(r, 0, 3) for _ in range(500)} + self.assertEqual(len(seen), 6) # all C(4, 2) pairs + + +class TestDataStructureWorkloads(unittest.TestCase): + def test_point_add_range_sum(self): + r = random.Random(2) + n = 40 + a = [r.randint(0, 100) for _ in range(n)] + queries = [] + for _ in range(200): + if r.randint(0, 1) == 0: + queries.append((0, r.randrange(n), r.randint(0, 100))) + else: + queries.append((1,) + W._uniform_pair(r, 0, n)) + + naive = list(a) + expected = 0 + for t, x, y in queries: + if t == 0: + naive[x] += y + else: + expected += sum(naive[x:y]) + + self.assertEqual(W.run_point_add_range_sum(n, a, queries), expected) + + def test_staticrmq(self): + r = random.Random(3) + n = 37 + a = [r.randint(0, 10**9) for _ in range(n)] + queries = [W._uniform_pair(r, 0, n) for _ in range(200)] + + expected = 0 + for l, rr in queries: + expected ^= min(a[l:rr]) + + self.assertEqual(W.run_staticrmq(a, queries), expected) + + def test_point_set_range_composite(self): + r = random.Random(4) + n = 33 + f = [(r.randint(1, MOD - 1), r.randint(0, MOD - 1)) for _ in range(n)] + queries = [] + for _ in range(150): + if r.randint(0, 1) == 0: + queries.append( + (0, r.randrange(n), r.randint(1, MOD - 1), r.randint(0, MOD - 1)) + ) + else: + l, rr = W._uniform_pair(r, 0, n) + queries.append((1, l, rr, r.randint(0, MOD - 1))) + + naive = list(f) + expected = 0 + for query in queries: + if query[0] == 0: + _, p, c, d = query + naive[p] = (c, d) + else: + _, l, rr, x = query + value = x + # f_{r-1}(f_{r-2}(...f_l(x))) + for i in range(l, rr): + value = (naive[i][0] * value + naive[i][1]) % MOD + expected += value + + self.assertEqual(W.run_point_set_range_composite(f, queries), expected) + + def test_range_affine_range_sum(self): + r = random.Random(5) + n = 29 + a = [r.randint(0, MOD - 1) for _ in range(n)] + queries = [] + for _ in range(150): + l, rr = W._uniform_pair(r, 0, n) + if r.randint(0, 1) == 0: + queries.append((0, l, rr, r.randint(1, MOD - 1), r.randint(0, MOD - 1))) + else: + queries.append((1, l, rr, 0, 0)) + + naive = list(a) + expected = 0 + for t, l, rr, b, c in queries: + if t == 0: + for i in range(l, rr): + naive[i] = (b * naive[i] + c) % MOD + else: + expected += sum(naive[l:rr]) % MOD + + self.assertEqual(W.run_range_affine_range_sum(a, queries), expected) + + def test_unionfind(self): + r = random.Random(6) + n = 30 + queries = [ + (r.randint(0, 1), r.randrange(n), r.randrange(n)) for _ in range(200) + ] + + parent = list(range(n)) + + def find(x): + while parent[x] != x: + x = parent[x] + return x + + expected = 0 + for t, u, v in queries: + if t == 0: + parent[find(u)] = find(v) + else: + expected += find(u) == find(v) + + self.assertEqual(W.run_unionfind(n, queries), expected) + + +class TestGraphWorkloads(unittest.TestCase): + def test_scc_component_count(self): + # two 3-cycles joined by a one-way bridge, plus an isolated vertex + edges = [(0, 1), (1, 2), (2, 0), (3, 4), (4, 5), (5, 3), (2, 3)] + self.assertEqual(W.run_scc(7, edges), 3) + + def test_two_sat_satisfiable(self): + # (x0 or x1) and (not x0 or x1) -> satisfiable + clause = [(0, True, 1, True), (0, False, 1, True)] + self.assertGreaterEqual(W.run_two_sat(2, clause), 0) + + def test_two_sat_unsatisfiable(self): + clause = [ + (0, True, 0, True), + (0, False, 0, False), + ] + self.assertEqual(W.run_two_sat(1, clause), -1) + + def test_bipartitematching(self): + r = random.Random(7) + left = right = 6 + edges = sorted({(r.randrange(left), r.randrange(right)) for _ in range(18)}) + + # naive Kuhn + adj = [[] for _ in range(left)] + for u, v in edges: + adj[u].append(v) + match = [-1] * right + + def try_kuhn(u, seen): + for v in adj[u]: + if seen[v]: + continue + seen[v] = True + if match[v] == -1 or try_kuhn(match[v], seen): + match[v] = u + return True + return False + + expected = sum(try_kuhn(u, [False] * right) for u in range(left)) + + self.assertEqual(W.run_bipartitematching(left, right, edges), expected) + + def test_assignment(self): + r = random.Random(8) + n = 5 + a = [[r.randint(-1000, 1000) for _ in range(n)] for _ in range(n)] + expected = min( + sum(a[i][p[i]] for i in range(n)) for p in itertools.permutations(range(n)) + ) + self.assertEqual(W.run_assignment(n, a), expected) + + +class TestMathWorkloads(unittest.TestCase): + def test_convolution_mod(self): + r = random.Random(9) + # Above 40 elements so this exercises the NTT path, not the naive + # fallback that FFT.convolution takes for small inputs. + a = [r.randint(0, MOD - 1) for _ in range(70)] + b = [r.randint(0, MOD - 1) for _ in range(53)] + c = [0] * (len(a) + len(b) - 1) + for i, x in enumerate(a): + for j, y in enumerate(b): + c[i + j] = (c[i + j] + x * y) % MOD + self.assertEqual(W.run_convolution_mod(a, b), c[0] ^ c[-1]) + + def test_inv_of_formal_power_series(self): + r = random.Random(10) + n = 32 + a = [r.randint(1, MOD - 1)] + [r.randint(0, MOD - 1) for _ in range(n - 1)] + inv = W.fps_mod.FPS(a).inv(n) + product = (W.fps_mod.FPS(a) * inv).resize(n).Func + self.assertEqual(product, [1] + [0] * (n - 1)) + self.assertEqual(W.run_inv_of_formal_power_series(a), inv.Func[-1]) + + def test_exp_log_round_trip(self): + r = random.Random(11) + n = 32 + a = [0] + [r.randint(0, MOD - 1) for _ in range(n - 1)] + exp = W.fps_mod.FPS(a).exp(n) + self.assertEqual(exp.Func[0], 1) + self.assertEqual(exp.log().resize(n).Func, a) + self.assertEqual(W.run_exp_of_formal_power_series(a), exp.Func[-1]) + + def test_log_of_formal_power_series(self): + r = random.Random(12) + n = 32 + a = [1] + [r.randint(0, MOD - 1) for _ in range(n - 1)] + expected = W.fps_mod.FPS(a).log().resize(n) + self.assertEqual(expected.Func[0], 0) + self.assertEqual(W.run_log_of_formal_power_series(a), expected.Func[-1]) + + def test_sum_of_floor_of_linear(self): + r = random.Random(13) + cases = [] + for _ in range(30): + n = r.randint(1, 60) + m = r.randint(1, 60) + cases.append((n, m, r.randrange(m), r.randrange(m))) + expected = sum(sum((a * i + b) // m for i in range(n)) for n, m, a, b in cases) + self.assertEqual(W.run_sum_of_floor_of_linear(cases), expected) + + def test_primality_test(self): + values = list(range(1, 300)) + + def naive(n): + if n < 2: + return False + return all(n % d for d in range(2, int(n**0.5) + 1)) + + self.assertEqual( + W.run_primality_test(values), sum(1 for n in values if naive(n)) + ) + + def test_factorize(self): + values = [10**18 - i for i in range(3)] + self.assertEqual( + W.run_factorize(values), + sum(sum(W.prime_fact_mod.prime_fact(n).values()) for n in values), + ) + for n in values: + product = 1 + for p, e in W.prime_fact_mod.prime_fact(n).items(): + self.assertTrue(W.prime_fact_mod.is_probable_prime(p)) + product *= p**e + self.assertEqual(product, n) + + +class TestStringWorkloads(unittest.TestCase): + def test_suffixarray(self): + s = "mississippi" + expected = sorted(range(len(s)), key=lambda i: s[i:]) + sa = W.acl_string_mod.String(s).suffix_array() + self.assertEqual(sa, expected) + self.assertEqual(W.run_suffixarray(s), expected[0] ^ expected[-1]) + + def test_zalgorithm(self): + s = "abacabadabacaba" + z = [] + for i in range(len(s)): + k = 0 + while i + k < len(s) and s[k] == s[i + k]: + k += 1 + z.append(k) + self.assertEqual(W.run_zalgorithm(s), sum(z)) + + def test_number_of_substrings(self): + r = random.Random(14) + s = "".join(r.choice("ab") for _ in range(40)) + expected = len( + {s[i:j] for i in range(len(s)) for j in range(i + 1, len(s) + 1)} + ) + self.assertEqual(W.run_number_of_substrings(s), expected) + + +class TestRegistry(unittest.TestCase): + def test_every_module_is_covered(self): + modules = { + "segtree", + "lazysegtree", + "dsu", + "maxflow", + "mincostflow", + "convolution", + "scc", + "fps", + "fenwicktree", + "acl_math", + "acl_string", + "prime_fact", + "two_sat", + } + self.assertEqual(modules - {w.module for w in W.BENCHMARKS}, set()) + + def test_library_checker_workloads_declare_a_problem(self): + for w in W.LIBRARY_CHECKER_BENCHMARKS: + self.assertTrue(w.problem, w.name) + self.assertEqual(w.name, w.problem) + self.assertTrue(w.url.startswith("https://judge.yosupo.jp/problem/")) + + def test_extra_workloads_declare_no_problem(self): + for w in W.EXTRA_BENCHMARKS: + self.assertIsNone(w.problem, w.name) + self.assertTrue(w.name.startswith("extra_")) + + def test_names_are_unique(self): + names = [w.name for w in W.BENCHMARKS] + self.assertEqual(len(names), len(set(names))) + + def test_cost_factor_matches_the_registry(self): + self.assertEqual(set(W.COST_FACTOR), {w.name for w in W.BENCHMARKS}) + + def test_effective_scale_never_exceeds_the_judge_constraints(self): + for w in W.BENCHMARKS: + self.assertTrue(0 < w.effective_scale <= 1, w.name) + + def test_max_scale_pins_every_workload_to_the_maximum(self): + original = W.SCALE + try: + W.SCALE = float("inf") + for w in W.BENCHMARKS: + self.assertEqual(w.effective_scale, 1.0, w.name) + finally: + W.SCALE = original + + def test_runners_do_not_mutate_their_inputs(self): + """pytest-benchmark reuses one ``build()`` result across every round + and warmup, so a runner that mutated its arguments would time a + different problem each round. Checked at a tiny scale.""" + os.environ["ACL_BENCH_SCALE"] = "0.0005" + try: + importlib.reload(W) + for w in W.BENCHMARKS: + args = w.build() + before = copy.deepcopy(args) + w.run(*args) + self.assertEqual(args, before, w.name) + finally: + del os.environ["ACL_BENCH_SCALE"] + importlib.reload(W) + + def test_unpacks_as_a_triple(self): + name, build, run = W.BENCHMARKS[0] + self.assertEqual(name, W.BENCHMARKS[0].name) + self.assertTrue(callable(build) and callable(run)) + + +if __name__ == "__main__": + unittest.main()