From 7547aaabc1d23c6b41fcdb12622b97154a12e549 Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Sun, 15 Nov 2026 23:26:15 +0800 Subject: [PATCH 1/9] fix(distributed): handle document-local KDA shards safely --- cula/distributed/kda.py | 22 +++++++++++++++++++--- cula/distributed/plan.py | 11 +++++++++++ docs/KIMI_CP_RUNTIME.md | 20 ++++++++++++++++++-- tests/test_kimi_cp_helpers.py | 28 +++++++++++++++++++--------- tests/test_kimi_cp_plan.py | 23 +++++++++++++++++++++++ 5 files changed, 90 insertions(+), 14 deletions(-) diff --git a/cula/distributed/kda.py b/cula/distributed/kda.py index 916583c5..603cd113 100644 --- a/cula/distributed/kda.py +++ b/cula/distributed/kda.py @@ -719,7 +719,12 @@ def backward(ctx, grad_output: torch.Tensor): # The reverse carry is determined only by the first local span. cuDNN # backward apply consumes the summary transition in its stored M^T domain. first_end = plan.first_boundary_end - g_local, transition_backward, backward_context = kimi_delta_attention_summary_bwd( + # Native context contains per-token intermediates, not just the boundary + # summary. A prefix summary's context cannot serve the full THD shard. + # Keep the prefix-only scan for multi-document shards and let apply + # prepare its own intermediates; reuse only for identical token ranges. + reuse_context = first_end == q.shape[0] + summary_outputs = kimi_delta_attention_summary_bwd( q[:first_end], k[:first_end], g[:first_end], @@ -728,11 +733,13 @@ def backward(ctx, grad_output: torch.Tensor): cu_seqlens[:2], scale=options.scale, output_transition=True, - output_context=True, + output_context=reuse_context, checkpoint_every_n_tokens=options.checkpoint_every_n_tokens, plan_name=options.summary_plan_name, **options.kwargs(a_log, dt_bias), ) + g_local, transition_backward = summary_outputs[:2] + backward_context = summary_outputs[2] if reuse_context else None following_h, following_transition = _validate_boundary_summaries( ctx.boundary_transport.boundary_summaries( g_local[0], @@ -868,6 +875,11 @@ def cudnn_kda_cp( layers. Inputs use cuDNN's canonical THD contract; framework-specific layout and gate conversions belong in the framework adapter. + If every shard boundary is a document boundary, the plan marks the whole + group as independent. Execution then uses native local KDA/autograd and + skips boundary summaries and transport. An isolated rank alone cannot + enable this path while other ranks still require an exchange. + ``boundary_transport`` defaults to exact ordered summary all-gather with lazy storage. Large-CP callers should prepare one :class:`AllGatherKimiCPTransport` per concurrent CUDA stream and reuse it @@ -916,7 +928,11 @@ def cudnn_kda_cp( plan_name=plan_name, summary_plan_name=summary_plan_name, ) - if plan.world_size == 1: + if plan.world_size == 1 or not plan.requires_boundary_exchange: + # Every shard boundary is a document boundary. This is the same + # independent native recurrence as CP1, including native autograd; + # no forward or backward state can cross ranks. The plan flag is + # identical on all peers, so no rank skips a required collective. kimi_delta_attention, _ = _load_cudnn_kda_ops() attention_outputs = kimi_delta_attention( q, diff --git a/cula/distributed/plan.py b/cula/distributed/plan.py index 3890e70d..9e8d6132 100644 --- a/cula/distributed/plan.py +++ b/cula/distributed/plan.py @@ -29,6 +29,10 @@ class KimiCPPlan: plan records their token spans and the ordered rank chains that contribute incoming forward state or outgoing backward state. Tensor shapes and process-group identity are checked again at execution time. + + ``requires_boundary_exchange`` is a group-wide property derived from all + shard boundaries, identical on every rank. Manual plans default to the + conservative exchanging path. """ rank: int @@ -42,6 +46,9 @@ class KimiCPPlan: last_boundary_start: int pre_num_ranks: int post_num_ranks: int + # Group-wide, not rank-local: one isolated rank must still participate if + # any other rank needs the boundary collective. Manual plans stay safe. + requires_boundary_exchange: bool = True @property def local_tokens(self) -> int: @@ -130,6 +137,9 @@ def from_global_cu_seqlens( if pre_num_ranks < 0 or post_num_ranks < 0: raise RuntimeError("internal error while deriving Kimi CP boundary rank ranges") + document_boundaries = set(boundaries) + requires_boundary_exchange = any(offset not in document_boundaries for offset in offsets[1:-1]) + return cls( rank=rank, world_size=world_size, @@ -142,6 +152,7 @@ def from_global_cu_seqlens( last_boundary_start=local_boundaries[-2], pre_num_ranks=pre_num_ranks, post_num_ranks=post_num_ranks, + requires_boundary_exchange=requires_boundary_exchange, ) diff --git a/docs/KIMI_CP_RUNTIME.md b/docs/KIMI_CP_RUNTIME.md index 0f3f795c..52861a7b 100644 --- a/docs/KIMI_CP_RUNTIME.md +++ b/docs/KIMI_CP_RUNTIME.md @@ -50,11 +50,22 @@ balanced partition or explicit global shard offsets. It derives: - clipped local `cu_seqlens`; - first and last local sequence IDs; - the first and last boundary token spans; -- the number of preceding and following ranks in those sequence chains. +- the number of preceding and following ranks in those sequence chains; +- whether any document crosses a shard boundary anywhere in the group. Plan construction is CPU-only and must happen once per packed microbatch, not once per layer. The same plan is reusable by every KDA layer. +When every internal shard boundary is a document boundary, +`requires_boundary_exchange` is false on **every rank**. The runtime then uses +native local KDA/autograd without boundary summaries, state probes or transport. +The operands, dtypes and batch-invariance option are unchanged. A locally +isolated rank must still join the boundary exchanges if another document crosses +ranks elsewhere in the group. Manually constructed plans default to the +conservative exchanging path. The paired benchmark +`benchmarks/bench_kda_independent_shards.py` checks all outputs and five gradients, +including weak decay and an isolated-rank/crossing-peer control. + Only the first and last local spans can cross rank boundaries under the contiguous-shard contract. Forward therefore summarizes only the last local span; backward summarizes only the first. This is the boundary-only strategy @@ -73,12 +84,17 @@ Forward: Backward performs the symmetric reverse chain: -1. Produce `(G, M^T)` and reusable backward context for the first local span. +1. Produce `(G, M^T)` for the first local span. 2. All-gather the transition in its stored-domain orientation. 3. Select following ranks belonging to the last local sequence. 4. Pass them as `following_state_grad` and `following_transition`. 5. Let cuDNN compose the reverse chain and compute local tensor and gate gradients. +Native backward context contains per-token intermediates. It is reused only +when the summary covers the complete local shard. A multi-document shard's +prefix-only summary cannot supply context for the full apply operation; apply +prepares its own intermediates in that case. + The state payload and transition remain FP32. cuLA exposes no merge-precision switch; the cuDNN apply API owns the fixed high-precision state chain. The public cuDNN Frontend 1.32.2 wheel does not contain this API. Until a compatible diff --git a/tests/test_kimi_cp_helpers.py b/tests/test_kimi_cp_helpers.py index 932f48e4..c9e437bd 100644 --- a/tests/test_kimi_cp_helpers.py +++ b/tests/test_kimi_cp_helpers.py @@ -1,8 +1,10 @@ # Copyright 2025-2026 Ant Group Co., Ltd. # SPDX-License-Identifier: Apache-2.0 +from dataclasses import replace from types import SimpleNamespace +import pytest import torch from cula.distributed import kda as cp @@ -488,15 +490,18 @@ def test_boundary_transport_summary_contract_rejects_precision_or_shape_changes( raise AssertionError("invalid transport summaries must be rejected") -def test_distributed_autograd_delegates_summary_composition_to_cudnn(monkeypatch): +@pytest.mark.parametrize("multiple_documents", [False, True]) +def test_distributed_autograd_delegates_summary_composition_to_cudnn(monkeypatch, multiple_documents): plan = _transport_plan(rank=2, pre=2, post=1) + if multiple_documents: + plan = replace(plan, last_sequence=1, local_cu_seqlens=(0, 2, 4), first_boundary_end=2, last_boundary_start=2) heads, key_dim, value_dim = 2, 4, 3 q = torch.zeros(4, heads, key_dim) k = torch.zeros_like(q) v = torch.zeros(4, heads, value_dim) g = torch.zeros_like(q) beta = torch.zeros(4, heads) - cu = torch.tensor([0, 4], dtype=torch.int32) + cu = torch.tensor(plan.local_cu_seqlens, dtype=torch.int32) options = cp._KdaOptions( scale=None, use_qk_l2norm_in_kernel=False, @@ -538,19 +543,20 @@ def attention(q, k, v, g, beta, cu, **kwargs): ) def summary_bwd(q, k, g, beta, d_o, cu, **kwargs): - assert kwargs["output_context"] is True + assert q.shape[0] == plan.first_boundary_end + assert kwargs["output_context"] is (not multiple_documents) assert kwargs["checkpoint_every_n_tokens"] == 16 - return ( + result = ( torch.full((1, heads, value_dim, key_dim), 5.0), torch.full((1, heads, key_dim, key_dim), 6.0), - torch.tensor([7], dtype=torch.uint8), ) + return result if multiple_documents else (*result, torch.tensor([7], dtype=torch.uint8)) def backward(d_o, q, k, v, g, beta, cu, scale, **kwargs): observed["following_h"] = kwargs["following_state_grad"].clone() observed["following_transition"] = kwargs["following_transition"].clone() observed["composed"] = kwargs["composed_initial_state"].clone() - observed["context"] = kwargs["context"].clone() + observed["context"] = kwargs["context"] observed["checkpoints"] = kwargs["state_checkpoints"].clone() return ( torch.ones_like(q), @@ -587,14 +593,18 @@ def backward(d_o, q, k, v, g, beta, cu, scale, **kwargs): assert observed["following_h"].shape[0] == 1 assert torch.equal(observed["following_transition"], torch.full((1, heads, key_dim, key_dim), 6.0)) assert torch.equal(observed["composed"], torch.full((1, heads, value_dim, key_dim), 4.0)) - assert torch.equal(observed["context"], torch.tensor([7], dtype=torch.uint8)) + if multiple_documents: + assert observed["context"] is None + else: + assert torch.equal(observed["context"], torch.tensor([7], dtype=torch.uint8)) assert torch.equal(observed["checkpoints"], torch.full((1, heads, value_dim, key_dim), 3.0)) assert len(gradients) == 12 assert gradients[6:8] == (None, None) -def test_single_rank_fast_path_calls_attention_export(monkeypatch): - plan = KimiCPPlan.from_global_cu_seqlens([0, 2], rank=0, world_size=1) +@pytest.mark.parametrize("world", [1, 4]) +def test_independent_shards_call_native_attention_export(monkeypatch, world): + plan = KimiCPPlan.from_global_cu_seqlens(list(range(0, 2 * world + 1, 2)), rank=0, world_size=world) q = torch.zeros(2, 1, 4) v = torch.zeros(2, 1, 3) beta = torch.zeros(2, 1) diff --git a/tests/test_kimi_cp_plan.py b/tests/test_kimi_cp_plan.py index f6c6c791..f2f8eae1 100644 --- a/tests/test_kimi_cp_plan.py +++ b/tests/test_kimi_cp_plan.py @@ -103,3 +103,26 @@ def test_cpu_local_boundaries_must_match_plan(): torch.tensor([0, 3], dtype=torch.int32), device=torch.device("cpu"), ) + + +@pytest.mark.parametrize("world", [2, 4, 8, 16, 64]) +def test_document_local_shards_skip_boundary_exchange_globally(world): + cu = _global_cu([1024] * (world * 3)) + plans = [KimiCPPlan.from_global_cu_seqlens(cu, rank=rank, world_size=world) for rank in range(world)] + assert all(not p.requires_boundary_exchange for p in plans) + assert all(p.pre_num_ranks == p.post_num_ranks == 0 for p in plans) + + +def test_isolated_rank_cannot_skip_peers_boundary_collective(): + # Rank zero is independent, but the second document crosses ranks 1/2. + plans = [KimiCPPlan.from_global_cu_seqlens([0, 4, 12], rank=rank, world_size=3) for rank in range(3)] + assert plans[0].pre_num_ranks == plans[0].post_num_ranks == 0 + assert all(p.requires_boundary_exchange for p in plans) + + +def test_independent_detection_uses_supplied_uneven_shards(): + offsets = (0, 3, 11, 16) + plans = [ + KimiCPPlan.from_global_cu_seqlens([0, 1, 3, 11, 13, 16], rank=r, world_size=3, shard_offsets=offsets) for r in range(3) + ] + assert all(not p.requires_boundary_exchange for p in plans) From c981f9f501cd34c5d4ebe401fbdf9d48d435ccb9 Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Sun, 15 Nov 2026 23:26:22 +0800 Subject: [PATCH 2/9] perf(hybrid): coalesce DSA windows in linear time --- cula/hybrid/dsa.py | 4 +++- tests/test_hybrid_dsa_plan.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cula/hybrid/dsa.py b/cula/hybrid/dsa.py index f3f0cf74..4761bf91 100644 --- a/cula/hybrid/dsa.py +++ b/cula/hybrid/dsa.py @@ -265,6 +265,7 @@ def from_cu_seqlens( sequence, row_start, row_end, key_length, position, candidates, query_length = atoms[atom] whole = row_start == 0 and row_end == query_length group = [atoms[atom]] + group_candidates = candidates atom += 1 while whole and atom < len(atoms): next_atom = atoms[atom] @@ -273,10 +274,11 @@ def from_cu_seqlens( next_sequence != group[-1][0] + 1 or next_start != 0 or next_end != next_length - or sum(item[5] for item in group) + next_candidates > budget_floats + or group_candidates + next_candidates > budget_floats ): break group.append(next_atom) + group_candidates += next_candidates atom += 1 first_sequence = group[0][0] diff --git a/tests/test_hybrid_dsa_plan.py b/tests/test_hybrid_dsa_plan.py index f4c1c1c6..15022473 100644 --- a/tests/test_hybrid_dsa_plan.py +++ b/tests/test_hybrid_dsa_plan.py @@ -30,6 +30,26 @@ def test_selection_plan_coalesces_short_packed_sequences(): assert window.candidate_offsets == (0, 28, 73) +def test_thousands_of_short_sequences_keep_exact_window_budget_boundaries(): + # Each four-token document contributes exactly one complete-block logit. + count, capacity = 4097, 1000 + plan = DsaSelectionPlan.from_cu_seqlens( + tuple(4 * i for i in range(count + 1)), + tuple(range(count + 1)), + top_k=1, + candidate_budget_bytes=capacity * 4, + ) + assert [w.candidate_floats for w in plan.windows] == [1000, 1000, 1000, 1000, 97] + assert [(w.query_start, w.query_end) for w in plan.windows] == [ + (0, 4000), + (4000, 8000), + (8000, 12000), + (12000, 16000), + (16000, 16388), + ] + assert all(w.candidate_offsets == tuple(range(w.candidate_floats + 1)) for w in plan.windows) + + def test_selection_plan_splits_one_sequence_to_bound_peak_scratch(): plan = DsaSelectionPlan.from_cu_seqlens( [0, 32], From 5bbe9475e91f22a6feefc521c9379769ee62a25e Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Sun, 15 Nov 2026 23:29:49 +0800 Subject: [PATCH 3/9] feat(distributed): route THD selection with calibrated costs --- benchmarks/bench_csa_layout_matrix.py | 155 + benchmarks/bench_full_csa_cp.py | 828 +++ benchmarks/bench_kda_independent_shards.py | 148 + benchmarks/bench_thd_transport.py | 204 + benchmarks/calibrate_thd_selection.py | 188 + benchmarks/cp_layouts.py | 37 + benchmarks/fit_thd_transport.py | 108 + benchmarks/profiles/README.md | 23 + .../profiles/thd_selection_b300_cp8.json | 646 ++ .../profiles/thd_selection_h20_cp16.json | 1168 +++ benchmarks/summarize_csa_layout_matrix.py | 132 + benchmarks/thd_1m_cases.py | 259 + cula/distributed/__init__.py | 9 + cula/distributed/redistribute.py | 186 + cula/distributed/thd_planner.py | 872 ++- docs/THD_1M_STATISTICS_CASES.md | 114 + docs/THD_LAYOUT_PLANNER.md | 304 +- docs/THD_REDISTRIBUTION.md | 99 + docs/THD_SELECTION_RESULTS.md | 163 + docs/results/thd_selection_20261115.json | 6544 +++++++++++++++++ .../thd_selection_review_20261115.json | 1038 +++ tests/test_csa_matrix_report.py | 107 + tests/test_full_csa_cp_metadata.py | 59 + tests/test_thd_1m_cases.py | 91 + tests/test_thd_cost_fit.py | 136 + tests/test_thd_planner.py | 434 +- tests/test_thd_redistribute.py | 130 + 27 files changed, 14007 insertions(+), 175 deletions(-) create mode 100644 benchmarks/bench_csa_layout_matrix.py create mode 100644 benchmarks/bench_full_csa_cp.py create mode 100644 benchmarks/bench_kda_independent_shards.py create mode 100644 benchmarks/bench_thd_transport.py create mode 100644 benchmarks/calibrate_thd_selection.py create mode 100644 benchmarks/cp_layouts.py create mode 100644 benchmarks/fit_thd_transport.py create mode 100644 benchmarks/profiles/README.md create mode 100644 benchmarks/profiles/thd_selection_b300_cp8.json create mode 100644 benchmarks/profiles/thd_selection_h20_cp16.json create mode 100644 benchmarks/summarize_csa_layout_matrix.py create mode 100644 benchmarks/thd_1m_cases.py create mode 100644 cula/distributed/redistribute.py create mode 100644 docs/THD_1M_STATISTICS_CASES.md create mode 100644 docs/THD_REDISTRIBUTION.md create mode 100644 docs/THD_SELECTION_RESULTS.md create mode 100644 docs/results/thd_selection_20261115.json create mode 100644 docs/results/thd_selection_review_20261115.json create mode 100644 tests/test_csa_matrix_report.py create mode 100644 tests/test_full_csa_cp_metadata.py create mode 100644 tests/test_thd_1m_cases.py create mode 100644 tests/test_thd_cost_fit.py create mode 100644 tests/test_thd_redistribute.py diff --git a/benchmarks/bench_csa_layout_matrix.py b/benchmarks/bench_csa_layout_matrix.py new file mode 100644 index 00000000..18e97c65 --- /dev/null +++ b/benchmarks/bench_csa_layout_matrix.py @@ -0,0 +1,155 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Real-CP layout comparison with full output/gradient baselines in CPU RAM.""" + +import argparse +import gc +import sys +from pathlib import Path + +from benchmarks.thd_1m_cases import DEFAULT_SEED, counted_cases, heldout_cases, statistics_cases + + +def performance_cases(suite="baseline", seed=DEFAULT_SEED): + if suite == "validation": + if seed != DEFAULT_SEED: + raise ValueError("the validation suite fixes its seeds for reproducibility") + return [ + *performance_cases("baseline"), + *performance_cases("counted"), + *( + (f"{case.name}-seed{heldout_seed}", list(case.lengths), False) + for heldout_seed in (20261114, 20261115, 20261116) + for case in heldout_cases(heldout_seed) + ), + ] + cases = [] + if suite in ("baseline", "all"): + cases += [ + ("1m-single", [1048576], False), + ("1m-uniform", [16384] * 64, False), + ("1m-near-uniform", [15360, 17408] * 32, False), + ("1m-20s2l", [1024] * 20 + [514048] * 2, False), + ("1m-power-law", [1024, 4096, 16384, 65536, 131072, 307200, 523264], False), + ] + if suite in ("statistics", "all"): + cases += [(case.name, list(case.lengths), False) for case in statistics_cases(seed)] + if suite in ("counted", "all"): + cases += [(case.name, list(case.lengths), False) for case in counted_cases(seed)] + if suite in ("heldout", "all"): + cases += [(case.name, list(case.lengths), False) for case in heldout_cases(seed)] + if not cases: + raise ValueError(f"unknown suite: {suite}") + return cases + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--checks-only", action="store_true") + parser.add_argument( + "--suite", choices=("baseline", "statistics", "counted", "heldout", "all", "validation"), default="baseline" + ) + parser.add_argument("--cases", nargs="+", help="Exact performance case names from the selected suite") + parser.add_argument("--seed", type=int, default=DEFAULT_SEED, help="Synthetic THD generation seed") + parser.add_argument("--list-cases", action="store_true", help="List shapes without initializing CUDA/distributed") + parser.add_argument("--redistribution-scope", choices=("selection", "queries"), default="selection") + parser.add_argument("--skip-checks", action="store_true", help="Skip small FP32 checks after a separately validated run") + parser.add_argument( + "--layouts", nargs="+", choices=("contiguous", "zigzag", "planner"), default=["contiguous", "zigzag", "planner"] + ) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--heads", type=int, default=64, help="Attention heads; native SM100 D576 requires 64") + parser.add_argument("--planner-cost-model", help="JSON calibrated selection-routing coefficients") + parser.add_argument("--memory-stages", action="store_true") + parser.add_argument("--planner-strategy", choices=("balanced", "selection"), default="balanced") + args = parser.parse_args() + selected = performance_cases(args.suite, args.seed) + if args.cases: + unknown = set(args.cases) - {name for name, _, _ in selected} + if unknown: + parser.error(f"unknown cases for suite {args.suite}: {sorted(unknown)}") + selected = [case for case in selected if case[0] in args.cases] + if args.checks_only and args.cases: + parser.error("--cases cannot be combined with --checks-only") + if args.checks_only and args.skip_checks: + parser.error("--checks-only and --skip-checks are mutually exclusive") + if args.layouts[0] != "contiguous" or len(set(args.layouts)) != len(args.layouts): + parser.error("layouts must start with the contiguous numerical baseline and contain no duplicates") + if args.warmups < 0 or args.repeats < 1: + parser.error("warmups must be nonnegative and repeats positive") + if args.list_cases: + for name, lengths, _ in selected: + print(name, "documents", len(lengths), "tokens", sum(lengths)) + return + + import torch + import torch.distributed as dist + + from benchmarks.bench_full_csa_cp import main as run_case + + args.output_dir.mkdir(parents=True, exist_ok=True) + cases = [ + ("check-single", [32768], True), + ("check-mixed", [4] * 20 + [2008] * 2, True), + ("check-uniform", [256] * 16, True), + ("check-dense", [4] * 256 + [32] * 96, True), + ] + if args.skip_checks: + cases = [] + if not args.checks_only: + cases += selected + for case_index, (name, lengths, check) in enumerate(cases): + baseline = None + # Keep the numerical baseline first, alternate the remaining policies + # to avoid always giving one policy the same thermal/cache order. + layouts = args.layouts if case_index % 2 == 0 else [args.layouts[0], *reversed(args.layouts[1:])] + for layout in layouts: + print("CASE", name, layout, flush=True) + sys.argv = [ + "bench_full_csa_cp.py", + "--sequence-lengths", + *map(str, lengths), + "--heads", + str(args.heads), + "--top-k", + "512", + "--with-kda", + "--layout", + layout, + "--redistribution-scope", + args.redistribution_scope, + "--planner-strategy", + args.planner_strategy, + "--pool-backward-order", + "per-loss", + "--output", + str(args.output_dir / f"layout-{name}-{layout}.json"), + "--warmups", + "0" if check else str(args.warmups), + "--repeats", + "1" if check else str(args.repeats), + "--candidate-mib", + "64" if check else "0", + ] + if check: + sys.argv.append("--check") + if args.planner_cost_model: + sys.argv += ["--planner-cost-model", args.planner_cost_model] + if args.memory_stages: + sys.argv.append("--memory-stages") + saved = run_case(keep_group=True, baseline=baseline, save_baseline=layout == "contiguous") + if saved is not None: + baseline = saved + gc.collect() + torch.cuda.empty_cache() + dist.barrier(device_ids=[torch.cuda.current_device()]) + del baseline, saved + gc.collect() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_full_csa_cp.py b/benchmarks/bench_full_csa_cp.py new file mode 100644 index 00000000..885bc3b7 --- /dev/null +++ b/benchmarks/bench_full_csa_cp.py @@ -0,0 +1,828 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Eager, real-CP KDA + full MLA-absorb CSA operator benchmark. + +This experiment includes pooling, selection, attention, the detached MaxPool +teacher/KL indexer backward, and all CP activation/gradient collectives. It is +not a model benchmark: projection, residual, MLP and optimizer are excluded. +It deliberately reuses allocating cuDNN convenience APIs; no CUDA Graph or +bounded compilation-cache guarantee is made for this experimental harness. +Document lengths and CP cuts must be multiples of four. Per-query causal +partial blocks (including the first three queries of every doc) are supported. +""" + +import argparse +import datetime +import json +import math +import os +import statistics +import time +from dataclasses import asdict +from itertools import accumulate +from types import SimpleNamespace + +import torch +import torch.distributed as dist + +from benchmarks.cp_layouts import query_layouts +from cula._metadata import balanced_shards +from cula.distributed import ( + AllGatherKimiCPTransport, + THDRedistributionPlan, + THDRedistributor, + THDSelectionCostModel, + build_kimi_cp_plan, + cudnn_kda_cp, +) +from cula.hybrid import DsaSelectionPlan, DsaSelectionWorkspace, IndexPoolPlan, cudnn_dsa_select, cudnn_index_pool +from cula.hybrid.index_pool import _load_cudnn_index_pool_ops + + +def metrics(actual, expected): + actual, expected = actual.float(), expected.float() + delta = actual - expected + return { + "max_abs": delta.abs().max().item(), + "relative_rms": (delta.norm() / expected.norm().clamp_min(1e-12)).item(), + "finite": bool(torch.isfinite(actual).all()), + } + + +def expand_blocks(block_ids, scores, doc_start, visible): + """Compact valid block slots, expand members, append the causal tail. + + Sorting by validity preserves each valid (id, score) pairing. The returned + block ids are still document-local; token ids are globally addressed. + Invalid slots remain -1 even when topk_length is supplied to cuDNN. + """ + order = (block_ids < 0).to(torch.int32).argsort(dim=-1, stable=True) + ids = block_ids.gather(1, order) + scores = scores.gather(1, order) + valid = ids >= 0 + nblocks = valid.sum(-1, dtype=torch.int32) + members = doc_start[:, None, None] + 4 * ids[:, :, None] + torch.arange(4, device=ids.device) + members = members.masked_fill(~valid[:, :, None], -1).flatten(1).to(torch.int32) + # The padded row is friendly to both the attention and teacher kernels. + width = math.ceil((members.shape[1] + 3) / 128) * 128 + tokens = torch.full((ids.shape[0], width), -1, dtype=torch.int32, device=ids.device) + tokens[:, : members.shape[1]] = members + slots = nblocks[:, None] * 4 + torch.arange(3, device=ids.device) + tail = doc_start[:, None] + visible[:, None] // 4 * 4 + torch.arange(3, device=ids.device) + tail = tail.masked_fill(torch.arange(3, device=ids.device)[None] >= visible[:, None] % 4, -1) + tokens.scatter_(1, slots.long(), tail.to(torch.int32)) + lengths = nblocks * 4 + visible % 4 + return ids, scores, members, tokens, lengths.to(torch.int32) + + +def reduce_scatter_fp32(value, local_rows): + out = torch.empty((local_rows, *value.shape[1:]), device=value.device, dtype=torch.float32) + dist.reduce_scatter_tensor(out, value.float().contiguous()) + return out + + +def query_metadata(layout, device): + """Materialize all span addresses with one transfer, including empty ranks.""" + sizes = torch.tensor([span.tokens for span in layout.spans], dtype=torch.int64, device="cpu") + starts = torch.tensor([span.sequence_start for span in layout.spans], dtype=torch.int32, device="cpu") + shifts = torch.tensor( + [span.token_start - offset for span, offset in zip(layout.spans, (0, *accumulate(sizes.tolist())))], + dtype=torch.int32, + device="cpu", + ) + n = layout.local_tokens + origin = torch.repeat_interleave(starts, sizes, output_size=n) + rows = torch.arange(n, dtype=torch.int32, device="cpu") + torch.repeat_interleave(shifts, sizes, output_size=n) + return torch.stack((origin, rows - origin + 1, rows)).to(device).unbind(0) + + +class FullCSA: + def __init__( + self, + lengths, + heads, + top_k, + device, + candidate_mib, + pool_backward_order="per-loss", + layout="contiguous", + redistribution_scope="selection", + planner_cost_model=None, + planner_strategy="balanced", + ): + import cudnn + + self.ops = cudnn.DSA + self.top_k = top_k + self.pool_backward_order = pool_backward_order + self.redistribution_scope = redistribution_scope + _, self.pool_backward = _load_cudnn_index_pool_ops() + torch.cuda.synchronize(device) + common_prepare_start = time.perf_counter() + self.rank, self.world = dist.get_rank(), dist.get_world_size() + self.cu = (0, *accumulate(lengths)) + self.plan = build_kimi_cp_plan(self.cu, group=dist.group.WORLD) + self.source_n, self.total = self.plan.local_tokens, self.cu[-1] + self.scale, self.heads = 576**-0.5, heads + if self.source_n % 4 or any(n % 4 for n in lengths): + raise ValueError("source CP shards and documents must be block aligned") + n = self.source_n + pool_plan = IndexPoolPlan.from_cu_seqlens(self.plan.local_cu_seqlens, head_dim=128, ratio=4) + pool_cu_tokens, pool_cu_blocks = pool_plan.materialize(device) + torch.cuda.synchronize(device) + self.common_prepare_ms = (time.perf_counter() - common_prepare_start) * 1000 + # All random tensors are generated in SOURCE ownership, with identical + # RNG consumption across layouts. No target-layout synthetic inputs. + self.indexer = SimpleNamespace( + qi=torch.randn(n, 64, 128, dtype=torch.bfloat16, device=device), + local_kv=torch.randn(n, 128, dtype=torch.bfloat16, device=device), + pool_score=torch.randn(n, 128, dtype=torch.bfloat16, device=device), + ape=torch.randn(4, 128, dtype=torch.float32, device=device), + weights=torch.randn(n, 64, dtype=torch.float32, device=device), + global_kbar=torch.empty(self.total // 4, 128, dtype=torch.bfloat16, device=device), + pool_plan=pool_plan, + pool_cu_tokens=pool_cu_tokens, + pool_cu_blocks=pool_cu_blocks, + local_blocks=n // 4, + ) + b = self.indexer + # Match the native backward's BF16 weights contract from the outset; + # do not quantize an FP32 forward operand only for backward. + b.weights = (b.weights / (64 * math.sqrt(128))).bfloat16() + self.source_qi, self.source_weights = b.qi, b.weights + for x in (b.local_kv, b.pool_score, b.ape): + x.requires_grad_(True) + # APE is a replicated parameter, not an independent rank-local input. + dist.broadcast(b.ape.detach(), src=0) + self.source_q = torch.randn(n, heads, 576, device=device, dtype=torch.bfloat16) * 0.5 + self.kv = torch.randn(n, 576, device=device, dtype=torch.bfloat16) * 0.5 + self.source_dout = torch.randn(n, heads, 512, device=device, dtype=torch.bfloat16) + self.global_kv = torch.empty(self.total, 576, device=device, dtype=torch.bfloat16) + self.sink = torch.full((heads,), -torch.inf, device=device, dtype=torch.float32) + self.unit_grad = torch.ones((), device=device, dtype=torch.float32) + torch.cuda.synchronize(device) + prepare_start = time.perf_counter() + if planner_cost_model is not None and redistribution_scope != "selection": + raise ValueError("the selection cost model is valid only for selection-only redistribution") + if planner_strategy == "selection" and redistribution_scope != "selection": + raise ValueError("selection planning requires selection-only redistribution") + ranks, self.selected_policy = query_layouts( + self.cu, self.world, layout, cost_model=planner_cost_model, planner_strategy=planner_strategy + ) + if redistribution_scope == "queries" and any(r.local_tokens == 0 for r in ranks): + raise ValueError( + "query-wide redistribution requires nonempty attention ranks; use selection scope for empty ranks" + ) + self.selection_layout = ranks[self.rank] + source_offsets = balanced_shards(self.total, self.world) + exchange_plan = THDRedistributionPlan.build(ranks, source_offsets, self.rank) + self.selection_exchange = THDRedistributor(exchange_plan, device, group=dist.group.WORLD) + if redistribution_scope == "selection": + attention_ranks, _ = query_layouts(self.cu, self.world, "contiguous") + self.layout = attention_ranks[self.rank] + self.exchange = THDRedistributor( + THDRedistributionPlan.build(attention_ranks, source_offsets, self.rank), device, group=dist.group.WORLD + ) + else: + self.layout = self.selection_layout + self.exchange = self.selection_exchange + self.n = self.layout.local_tokens + self.doc_start, self.visible, self.query_global_rows = query_metadata(self.layout, device) + key_prefix, key_ranges = [0], [] + for span in self.selection_layout.spans: + key_prefix.append(key_prefix[-1] + (span.sequence_end - span.sequence_start) // 4) + key_ranges.append((span.sequence_start // 4, span.sequence_end // 4)) + b.select_plan = None + b.select_workspace = None + if self.selection_layout.local_tokens: + b.select_plan = DsaSelectionPlan.from_cu_seqlens( + self.selection_layout.local_cu_seqlens, + key_prefix, + top_k=top_k, + block_size=4, + q_causal_offsets=self.selection_layout.causal_offsets, + candidate_budget_bytes=None if candidate_mib == 0 else candidate_mib << 20, + ) + b.select_workspace = DsaSelectionWorkspace.from_plan(b.select_plan, device) + self.selection_key_ids = None + if not key_ranges: + self.selection_key_slice = slice(0, 0) + elif all(left[1] == right[0] for left, right in zip(key_ranges, key_ranges[1:])): + self.selection_key_slice = slice(key_ranges[0][0], key_ranges[-1][1]) + else: + self.selection_key_ids = torch.cat([torch.arange(a, z, device="cpu") for a, z in key_ranges]).to(device) + torch.cuda.synchronize(device) + self.layout_prepare_ms = (time.perf_counter() - prepare_start) * 1000 + self.events = [] + self.last = {} + + def mark(self, name): + event = torch.cuda.Event(enable_timing=True) + event.record() + self.events.append((name, event)) + if getattr(self, "record_memory", False): + self.memory_stages[name] = allocator_usage() + + def forward(self): + b = self.indexer + self.events = [] + self.memory_stages = {} + self.mark("start") + # Release prior iteration references before allocating the next layout. + self.last = {} + with torch.cuda.nvtx.range("csa.query_redistribute"): + self.q = self.exchange.distribute(self.source_q) + selection_qi = self.selection_exchange.distribute(self.source_qi) + selection_weights = self.selection_exchange.distribute(self.source_weights) + b.qi = self.source_qi if self.redistribution_scope == "selection" else selection_qi + b.weights = self.source_weights if self.redistribution_scope == "selection" else selection_weights + self.mark("query_redistribute") + for x in (b.local_kv, b.pool_score, b.ape): + x.grad = None + with torch.cuda.nvtx.range("csa.pool"): + pooled = cudnn_index_pool( + b.local_kv, b.pool_score, b.ape, b.pool_cu_tokens, b.pool_cu_blocks, plan=b.pool_plan, deterministic=False + ) + self.mark("pool_fwd") + with torch.cuda.nvtx.range("csa.pool_gather"): + dist.all_gather_into_tensor(b.global_kbar, pooled.detach()) + self.mark("pool_gather") + with torch.cuda.nvtx.range("csa.selection_key_layout"): + select_keys = ( + b.global_kbar[self.selection_key_slice] + if self.selection_key_ids is None + else b.global_kbar.index_select(0, self.selection_key_ids) + ) + self.mark("selection_key_layout") + with torch.cuda.nvtx.range("csa.selection"): + if b.select_plan is None: + selected = SimpleNamespace( + block_ids=torch.empty((0, self.top_k), device=selection_qi.device, dtype=torch.int32), + scores=torch.empty((0, self.top_k), device=selection_qi.device, dtype=torch.float32), + ) + else: + selected = cudnn_dsa_select( + selection_qi, + select_keys[:, None], + selection_weights, + plan=b.select_plan, + workspace=b.select_workspace, + deterministic=False, + ) + self.mark("selection") + with torch.cuda.nvtx.range("csa.selection_restore"): + if self.redistribution_scope == "selection": + selected_ids = self.selection_exchange.restore(selected.block_ids) + selected_scores = self.selection_exchange.restore(selected.scores) + else: + selected_ids, selected_scores = selected.block_ids, selected.scores + del selection_qi, selection_weights, select_keys, selected + self.mark("selection_restore") + with torch.cuda.nvtx.range("csa.expand_tail"): + ids, scores, members, tokens, counts = expand_blocks(selected_ids, selected_scores, self.doc_start, self.visible) + block_ids = (ids + self.doc_start[:, None] // 4).masked_fill(ids < 0, -1) + self.mark("expand_tail") + with torch.cuda.nvtx.range("csa.kv_gather"): + dist.all_gather_into_tensor(self.global_kv, self.kv) + self.mark("kv_gather") + with torch.cuda.nvtx.range("csa.attention_fwd"): + fw = self.ops.sparse_attention_forward_wrapper( + self.q, self.global_kv, tokens, attn_sink=self.sink, topk_length=counts, softmax_scale=self.scale + ) + self.mark("attention_fwd") + with torch.cuda.nvtx.range("csa.teacher_kl"): + teacher = self.ops.sparse_attn_score_recompute_wrapper( + self.q[None], + self.global_kv[None], + fw["lse"][None], + members[None], + softmax_scale=self.scale, + qhead_per_kv_head=self.heads, + topk_indices_global=True, + )["target"][0] + target = teacher.view(self.n, -1, 4).amax(-1) + target = target / target.sum(-1, keepdim=True).clamp_min(1e-10) + valid = ids >= 0 + # Empty selected sets (first three queries) have zero auxiliary loss. + safe_scores = scores.masked_fill(~valid, -torch.inf) + safe_scores = torch.where(valid.any(-1, keepdim=True), safe_scores, torch.zeros_like(safe_scores)) + log_predict = safe_scores.log_softmax(-1).masked_fill(~valid, 0) + predict = log_predict.exp().masked_fill(~valid, 0) + loss = (target * (target.clamp_min(1e-10).log() - log_predict)).sum() / self.total + self.mark("teacher_kl") + with torch.cuda.nvtx.range("csa.output_restore"): + source_output = self.exchange.restore(fw["out"]) + self.mark("output_restore") + self.last = dict( + source_output=source_output, + pooled=pooled, + ids=ids, + block_ids=block_ids, + members=members, + tokens=tokens, + counts=counts, + target=target, + predict=predict, + loss=loss, + fw=fw, + ) + return source_output + + def backward(self): + b, r = self.indexer, self.last + with torch.cuda.nvtx.range("csa.dout_redistribute"): + self.dout = self.exchange.distribute(self.source_dout) + self.mark("dout_redistribute") + with torch.cuda.nvtx.range("csa.attention_bwd"): + bw = self.ops.sparse_attention_backward_wrapper( + self.q, + self.global_kv, + r["fw"]["out"], + self.dout, + r["fw"]["lse"], + self.sink, + r["tokens"], + softmax_scale=self.scale, + topk_length=r["counts"], + deterministic=False, + ) + self.mark("attention_bwd") + with torch.cuda.nvtx.range("csa.kv_reduce_scatter"): + local_dkv = reduce_scatter_fp32(bw["dkv"], self.source_n) + self.mark("kv_reduce_scatter") + with torch.cuda.nvtx.range("csa.indexer_bwd"): + # The backend overwrites target with its score gradient. Keep the + # reference target intact for numerical validation. + ibw = self.ops.indexer_backward_wrapper( + b.qi[None], + b.weights[None], + b.global_kbar[None], + r["target"][None].clone(), + r["predict"][None].clamp_min(1e-30).masked_fill(r["block_ids"][None] < 0, 0), + r["block_ids"][None], + self.unit_grad, + loss_coeff=self.n / self.total, + topk_indices_global=True, + deterministic=False, + d_index_k=torch.empty_like(b.global_kbar[None], dtype=torch.float32), + ) + self.mark("indexer_bwd") + with torch.cuda.nvtx.range("csa.pool_grad_exchange"): + if self.pool_backward_order == "sum-first": + local_dkbar = reduce_scatter_fp32(ibw["d_index_k"][0], b.local_blocks) + else: + # Preserve cuLA's gather-then-pool numerical ordering: do not + # merge loss-rank gradients before the native BF16 pool API. + incoming = ibw["d_index_k"][0].bfloat16() + received = torch.empty_like(incoming) + dist.all_to_all_single(received, incoming) + received = received.view(self.world, b.local_blocks, 128) + local_dkbar = received.float().sum(0) + self.mark("pool_grad_exchange") + with torch.cuda.nvtx.range("csa.pool_bwd"): + if self.pool_backward_order == "sum-first": + r["pooled"].backward(local_dkbar.bfloat16()) + else: + dkv = torch.zeros_like(b.local_kv, dtype=torch.float32) + ds = torch.zeros_like(b.pool_score, dtype=torch.float32) + da = torch.zeros_like(b.ape) + for loss_rank in range(self.world): + pool_bw = self.pool_backward( + b.local_kv, + b.pool_score, + b.ape, + b.pool_cu_tokens, + b.pool_cu_blocks, + received[loss_rank], + ratio=4, + head_dim=128, + coff=1, + deterministic=False, + ) + dkv.add_(pool_bw["grad_kv"]) + ds.add_(pool_bw["grad_score"]) + da.add_(pool_bw["grad_ape"]) + b.local_kv.grad = dkv.bfloat16() + b.pool_score.grad = ds.bfloat16() + b.ape.grad = da + dist.all_reduce(b.ape.grad) + self.mark("pool_bwd") + with torch.cuda.nvtx.range("csa.query_grad_restore"): + source_dq = self.exchange.restore(bw["dq"]) + source_dqi = self.exchange.restore(ibw["d_index_q"][0]) + source_dw = self.exchange.restore(ibw["d_weights"][0]) + self.mark("query_grad_restore") + r.update(source_dq=source_dq, source_dqi=source_dqi, source_dw=source_dw) + r.update(bw=bw, ibw=ibw, local_dkv=local_dkv, local_dkbar=local_dkbar) + + def timings(self): + return { + name + "_ms": previous.elapsed_time(event) for (_, previous), (name, event) in zip(self.events, self.events[1:]) + } + + +def check_reference(csa): + """Independent FP32 attention/teacher/KL and end-to-end pool gradients. + + Top-K is discrete: check the same selected set, plus exact document, + causality, uniqueness and complete/tail coverage. This oracle does not + independently validate selector ranking. + """ + b, r = csa.indexer, csa.last + ids, tokens = r["ids"], r["tokens"] + valid = tokens >= 0 + global_q = csa.query_global_rows + metadata_ok = bool(torch.all((~valid) | ((tokens >= csa.doc_start[:, None]) & (tokens <= global_q[:, None])))) + sorted_tokens = tokens.sort(-1).values + metadata_ok &= not bool(((sorted_tokens[:, 1:] == sorted_tokens[:, :-1]) & (sorted_tokens[:, 1:] >= 0)).any()) + metadata_ok &= bool(torch.equal(valid.sum(-1), r["counts"])) + metadata_ok &= bool(torch.equal((ids >= 0).sum(-1), (csa.visible // 4).clamp(max=ids.shape[1]))) + # Every causal tail token appears exactly once, even with no full block. + for offset in range(3): + tail = csa.doc_start + csa.visible // 4 * 4 + offset + found = (tokens == tail[:, None]).sum(-1) + metadata_ok &= bool(torch.all(found == (offset < csa.visible % 4).long())) + + kv = csa.global_kv.detach().float().requires_grad_(True) + kbar = b.global_kbar.detach().float().requires_grad_(True) + outputs, dq, dqi, dw, targets = [], [], [], [], [] + reference_loss = 0.0 + for start in range(0, csa.n, 16): + end = min(start + 16, csa.n) + ti = tokens[start:end].long() + mask = ti >= 0 + q = csa.q[start:end].float().detach().requires_grad_(True) + selected_kv = kv[ti.clamp_min(0)] + logits = torch.einsum("qhd,qkd->qhk", q, selected_kv) * csa.scale + probs = logits.masked_fill(~mask[:, None], -torch.inf).softmax(-1) + output = torch.einsum("qhk,qkv->qhv", probs, selected_kv[..., :512]) + output.backward(csa.dout[start:end].float()) + outputs.append(output.detach()) + dq.append(q.grad) + # Complete members occupy the prefix; causal tail starts after the last + # valid block. Mask before pooling to exclude tail slots from teacher. + teacher = probs.detach().sum(1)[:, : ids.shape[1] * 4].reshape(end - start, -1, 4).amax(-1) + teacher = teacher.masked_fill(ids[start:end] < 0, 0) + teacher = teacher / teacher.sum(-1, keepdim=True).clamp_min(1e-10) + targets.append(teacher) + qi = b.qi[start:end].float().detach().requires_grad_(True) + w = b.weights[start:end].float().detach().requires_grad_(True) + bi = r["block_ids"][start:end].long() + selected_kbar = kbar[bi.clamp_min(0)] + scores = (torch.einsum("qhd,qkd->qkh", qi, selected_kbar).relu() * w[:, None]).sum(-1) + valid_blocks = bi >= 0 + scores = scores.masked_fill(~valid_blocks, -torch.inf) + scores = torch.where(valid_blocks.any(-1, keepdim=True), scores, torch.zeros_like(scores)) + log_predict = scores.log_softmax(-1).masked_fill(~valid_blocks, 0) + loss = (teacher * (teacher.clamp_min(1e-10).log() - log_predict)).sum() / csa.total + loss.backward() + reference_loss += loss.item() + dqi.append(qi.grad) + dw.append(w.grad) + local_dkv = reduce_scatter_fp32(kv.grad, csa.source_n) + local_dkbar = reduce_scatter_fp32(kbar.grad, b.local_blocks) + pool_kv, pool_score, ape = [x.detach().float().requires_grad_(True) for x in (b.local_kv, b.pool_score, b.ape)] + weights = (pool_score.view(-1, 4, 128) + ape[None]).softmax(1) + pooled = (weights * pool_kv.view(-1, 4, 128)).sum(1) + pooled.backward(local_dkbar) + dist.all_reduce(ape.grad) + comparisons = { + "output": (r["source_output"], csa.exchange.restore(torch.cat(outputs))), + "dq": (r["source_dq"], csa.exchange.restore(torch.cat(dq))), + "dkv": (r["local_dkv"], local_dkv), + "teacher": (r["target"], torch.cat(targets)), + "d_index_q": (r["source_dqi"], csa.exchange.restore(torch.cat(dqi))), + "d_weights": (r["source_dw"], csa.exchange.restore(torch.cat(dw))), + "d_kbar": (r["local_dkbar"], local_dkbar), + "d_pool_kv": (b.local_kv.grad, pool_kv.grad), + "d_pool_score": (b.pool_score.grad, pool_score.grad), + "d_ape": (b.ape.grad, ape.grad), + } + errors = {key: metrics(actual, expected) for key, (actual, expected) in comparisons.items()} + # Fixed BF16 envelope, reported alongside raw errors; no threshold tuning. + passed = metadata_ok and all(x["finite"] and x["relative_rms"] < 0.03 for x in errors.values()) + loss_error = abs(r["loss"].item() - reference_loss) + passed &= loss_error < 1e-5 + return dict(pass_=passed, metadata_ok=metadata_ok, errors=errors, loss_abs_error=loss_error) + + +def check_redistribution(csa): + device = csa.source_q.device + source_rows = torch.arange(csa.plan.token_start, csa.plan.token_end, device=device) + source = (source_rows[:, None] * 3 + torch.arange(3, device=device)).float().requires_grad_(True) + ok = True + routes = [(csa.exchange, csa.layout)] + if csa.selection_exchange is not csa.exchange: + routes.append((csa.selection_exchange, csa.selection_layout)) + for exchange, layout in routes: + rows = ( + torch.cat([torch.arange(s.token_start, s.token_end, device=device) for s in layout.spans]) + if layout.spans + else torch.empty(0, device=device, dtype=torch.int64) + ) + expected = (rows[:, None] * 3 + torch.arange(3, device=device)).float() + source.grad = None + shuffled = exchange.distribute(source) + ok &= torch.equal(shuffled, expected) + shuffled.backward(expected * 2) + ok &= torch.equal(source.grad, source.detach() * 2) + leaf = shuffled.detach().requires_grad_(True) + restored = exchange.restore(leaf) + ok &= torch.equal(restored, source.detach()) + restored.backward(source.detach() * 3) + ok &= torch.equal(leaf.grad, expected * 3) + flag = torch.tensor(int(ok), device=device) + dist.all_reduce(flag, op=dist.ReduceOp.MIN) + if not flag.item(): + raise RuntimeError("row redistribution/inverse/autograd marker check failed") + return True + + +def compare_cpu_reference(actual, reference): + """Full-tensor error, streamed from lossless CPU storage; outside timing.""" + if actual.shape != reference.shape or actual.dtype != reference.dtype: + raise ValueError("baseline shape/dtype mismatch") + actual = actual.detach().reshape(-1) + reference = reference.reshape(-1) + stats = torch.zeros(3, device=actual.device, dtype=torch.float64) + finite = torch.ones((), device=actual.device, dtype=torch.bool) + for start in range(0, actual.numel(), 1 << 22): + a = actual[start : start + (1 << 22)].float() + b = reference[start : start + (1 << 22)].to(actual.device).float() + delta = a - b + stats[0] += delta.square().sum(dtype=torch.float64) + stats[1] += b.square().sum(dtype=torch.float64) + stats[2] = torch.maximum(stats[2], delta.abs().max().double()) + finite &= torch.isfinite(a).all() + values = stats.cpu().tolist() + return dict(relative_rms=math.sqrt(values[0] / max(values[1], 1e-24)), max_abs=values[2], finite=bool(finite)) + + +def fingerprint(tensor): + flat = tensor.detach().reshape(-1) + sums = torch.zeros(2, device=flat.device, dtype=torch.float64) + for start in range(0, flat.numel(), 1 << 22): + x = flat[start : start + (1 << 22)].float() + sums[0] += x.sum(dtype=torch.float64) + sums[1] += x.square().sum(dtype=torch.float64) + return sums.cpu().tolist() + + +def allocator_usage(): + return dict( + allocated_gib=torch.cuda.memory_allocated() / 2**30, + peak_allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + reserved_gib=torch.cuda.memory_reserved() / 2**30, + ) + + +def main(*, keep_group=False, baseline=None, save_baseline=False, comparison_name="layout_vs_contiguous"): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sequence-lengths", nargs="+", type=int, default=[4096]) + parser.add_argument("--heads", type=int, default=64) + parser.add_argument("--top-k", type=int, default=512) + parser.add_argument("--candidate-mib", type=int, default=0, help="0 uses cuLA's balanced scratch policy") + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--check", action="store_true") + parser.add_argument("--with-kda", action="store_true") + parser.add_argument("--kda-batch-invariant", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument( + "--kda-gate-scale", type=float, default=1.0, help="Scale negative KDA log-decay for sensitivity checks" + ) + parser.add_argument("--layout", choices=("contiguous", "zigzag", "planner"), default="contiguous") + parser.add_argument("--redistribution-scope", choices=("selection", "queries"), default="selection") + parser.add_argument("--planner-cost-model", help="JSON calibrated THDSelectionCostModel coefficients") + parser.add_argument("--planner-strategy", choices=("balanced", "selection"), default="balanced") + parser.add_argument("--memory-stages", action="store_true", help="Diagnostic allocator counters at each phase boundary") + parser.add_argument("--pool-backward-order", choices=("per-loss", "sum-first"), default="per-loss") + parser.add_argument("--output", required=True) + args = parser.parse_args() + if args.top_k <= 0 or args.top_k % 128 or args.warmups < 0 or args.repeats < 1: + parser.error("top-k must be positive and divisible by 128; warmups >= 0 and repeats >= 1") + if any(n <= 0 or n % 4 for n in args.sequence_lengths): + parser.error("this harness requires positive document lengths divisible by four") + if args.candidate_mib < 0: + parser.error("candidate-mib must be nonnegative") + if not math.isfinite(args.kda_gate_scale) or args.kda_gate_scale <= 0: + parser.error("kda-gate-scale must be finite and positive") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + if not dist.is_initialized(): + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=30)) + rank, world = dist.get_rank(), dist.get_world_size() + torch.manual_seed(20261112 + rank) + # Disable TF32 in the independent FP32 oracle. + torch.backends.cuda.matmul.allow_tf32 = False + cost_model = None + if args.planner_cost_model: + with open(args.planner_cost_model) as f: + parameters = json.load(f)["model"] + parameters["rank_node_ids"] = tuple(parameters["rank_node_ids"]) + cost_model = THDSelectionCostModel(**parameters) + csa = FullCSA( + args.sequence_lengths, + args.heads, + args.top_k, + device, + args.candidate_mib, + args.pool_backward_order, + args.layout, + args.redistribution_scope, + cost_model, + args.planner_strategy, + ) + csa.record_memory = args.memory_stages + redistribution_ok = check_redistribution(csa) + kda_prepare_ms = 0.0 + if args.with_kda: + plan = csa.plan + torch.cuda.synchronize(device) + kda_prepare_start = time.perf_counter() + transport = AllGatherKimiCPTransport.prepare(plan, heads=32, key_dim=128, value_dim=128, device=device) + local_cu = torch.tensor(plan.local_cu_seqlens, device=device, dtype=torch.int32) + torch.cuda.synchronize(device) + kda_prepare_ms = (time.perf_counter() - kda_prepare_start) * 1000 + q, k = [ + torch.nn.functional.normalize(torch.randn(csa.source_n, 32, 128, device=device), dim=-1).bfloat16() + for _ in range(2) + ] + v = torch.randn_like(q) + g = -0.01 - 0.04 * torch.rand(csa.source_n, 32, 128, device=device) + if args.kda_gate_scale != 1.0: + g *= args.kda_gate_scale + beta = torch.sigmoid(torch.randn(csa.source_n, 32, device=device)) + kda_inputs = [x.requires_grad_(True) for x in (q, k, v, g, beta)] + kda_do = torch.randn_like(v) + samples, check = [], None + for iteration in range(args.warmups + args.repeats): + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, kf, cb, end = [torch.cuda.Event(enable_timing=True) for _ in range(4)] + wall = time.perf_counter() + begin.record() + if args.with_kda: + for x in kda_inputs: + x.grad = None + with torch.cuda.nvtx.range("kda.forward"): + kda_out = cudnn_kda_cp( + *kda_inputs, + local_cu, + plan=plan, + group=dist.group.WORLD, + boundary_transport=transport, + batch_invariant=args.kda_batch_invariant, + ) + kf.record() + memory_after_kda_forward = allocator_usage() if args.memory_stages else None + csa.forward() + csa.backward() + cb.record() + if args.with_kda: + with torch.cuda.nvtx.range("kda.backward"): + kda_out.backward(kda_do) + end.record() + end.synchronize() + memory_after_kda_backward = allocator_usage() if args.memory_stages else None + sample = dict( + total_ms=begin.elapsed_time(end), + wall_ms=(time.perf_counter() - wall) * 1000, + kda_fwd_ms=begin.elapsed_time(kf), + kda_bwd_ms=cb.elapsed_time(end), + csa_ms=kf.elapsed_time(cb), + peak_allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + peak_reserved_gib=torch.cuda.max_memory_reserved() / 2**30, + **csa.timings(), + ) + if iteration >= args.warmups: + samples.append(sample) + if rank == 0: + print("ITER", iteration, json.dumps(sample), flush=True) + if args.check and check is None: + check = check_reference(csa) + finite = all( + bool(torch.isfinite(x).all()) + for x in ( + csa.last["fw"]["out"], + csa.last["bw"]["dq"], + csa.last["local_dkv"], + csa.last["ibw"]["d_index_q"], + csa.last["ibw"]["d_weights"], + csa.last["local_dkbar"], + csa.last["target"], + csa.last["predict"], + csa.last["loss"], + csa.indexer.local_kv.grad, + csa.indexer.pool_score.grad, + csa.indexer.ape.grad, + ) + ) + if args.with_kda: + finite &= all(bool(torch.isfinite(x).all()) for x in [kda_out, *(x.grad for x in kda_inputs)]) + r, b = csa.last, csa.indexer + tensors = dict( + output=r["source_output"], + dq=r["source_dq"], + dkv=r["local_dkv"], + d_index_q=r["source_dqi"], + d_weights=r["source_dw"], + d_pool_kv=b.local_kv.grad, + d_pool_score=b.pool_score.grad, + d_ape=b.ape.grad, + ) + if args.with_kda: + tensors["kda_output"] = kda_out + tensors.update({"kda_" + key: x.grad for key, x in zip(("dq", "dk", "dv", "dg", "dbeta"), kda_inputs)}) + global_loss = r["loss"].detach().clone() + dist.all_reduce(global_loss) + errors = ( + None + if baseline is None + else {key: compare_cpu_reference(value, baseline["tensors"][key]) for key, value in tensors.items()} + ) + loss_delta = None if baseline is None else abs(global_loss.item() - baseline["global_loss"]) + # These operands verify identical source data across the three layouts. + inputs = dict( + q=csa.source_q, + kv=csa.kv, + qi=csa.source_qi, + weights=csa.source_weights, + pool_kv=b.local_kv, + pool_score=b.pool_score, + ape=b.ape, + dout=csa.source_dout, + ) + if args.with_kda: + inputs.update({"kda_" + key: x for key, x in zip(("q", "k", "v", "g", "beta"), kda_inputs)}) + inputs["kda_dout"] = kda_do + input_checksums = {key: fingerprint(value) for key, value in inputs.items()} + input_match = baseline is None or input_checksums == baseline["input_checksums"] + result = dict( + rank=rank, + samples=samples, + check=check, + finite=finite, + loss=global_loss.item(), + loss_abs_diff=loss_delta, + input_match=input_match, + redistribution_bitwise=redistribution_ok, + layout_prepare_ms=csa.layout_prepare_ms, + common_prepare_ms=csa.common_prepare_ms, + kda_prepare_ms=kda_prepare_ms, + total_prepare_ms=csa.common_prepare_ms + csa.layout_prepare_ms + kda_prepare_ms, + selected_policy=csa.selected_policy, + local_query_rows=csa.n, + spans=len(csa.layout.spans), + selection_spans=len(csa.selection_layout.spans), + selection_query_rows=csa.selection_layout.local_tokens, + remote_query_rows=csa.selection_exchange.plan.remote_rows, + selection_workspace_bytes=0 if b.select_plan is None else b.select_plan.candidate_workspace_floats * 4, + memory_stages=( + dict(kda_forward=memory_after_kda_forward, **csa.memory_stages, kda_backward=memory_after_kda_backward) + if args.memory_stages + else None + ), + ) + result[comparison_name] = errors + gathered = [None] * world + dist.all_gather_object(gathered, result) + if rank == 0: + critical = { + key: statistics.median(max(r["samples"][i][key] for r in gathered) for i in range(args.repeats)) + for key in samples[0] + } + report = dict(config=vars(args), world_size=world, critical_rank_median=critical, ranks=gathered) + report["planner_cost_model"] = None if cost_model is None else asdict(cost_model) + with open(args.output, "w") as f: + json.dump(report, f, indent=2) + print("RESULT", json.dumps(critical), flush=True) + okay = all( + r["finite"] + and r["input_match"] + and r["redistribution_bitwise"] + and (r["loss_abs_diff"] is None or 0 <= r["loss_abs_diff"] < 1e-5) + and (r["check"] is None or r["check"]["pass_"]) + and (r[comparison_name] is None or all(e["finite"] and e["relative_rms"] < 0.03 for e in r[comparison_name].values())) + for r in gathered + ) + saved = ( + None + if not save_baseline + else dict( + tensors={key: value.detach().cpu() for key, value in tensors.items()}, + global_loss=global_loss.item(), + input_checksums=input_checksums, + ) + ) + if not keep_group: + dist.destroy_process_group() + if not okay: + raise RuntimeError("Numerical check failed; raw per-rank errors were saved") + return saved + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_kda_independent_shards.py b/benchmarks/bench_kda_independent_shards.py new file mode 100644 index 00000000..cf9c9bb9 --- /dev/null +++ b/benchmarks/bench_kda_independent_shards.py @@ -0,0 +1,148 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Paired KDA check for globally document-local CP shards, including backward. + +The baseline conservatively forces the original boundary route. This compares +complete output/gradients on identical operands, retaining the baseline on CPU. +""" + +import argparse +import gc +import hashlib +import json +import os +import statistics +from dataclasses import replace +from itertools import accumulate +from pathlib import Path + +import torch +import torch.distributed as dist + +from benchmarks.bench_full_csa_cp import compare_cpu_reference +from cula.distributed import AllGatherKimiCPTransport, KimiCPPlan, cudnn_kda_cp + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=10) + args = parser.parse_args() + if args.warmups < 0 or args.repeats < 1: + parser.error("warmups must be nonnegative and repeats positive") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + torch.backends.cuda.matmul.allow_tf32 = False + device = torch.device("cuda", local_rank) + dist.init_process_group("nccl", device_id=device) + rank, world = dist.get_rank(), dist.get_world_size() + if world < 2 or (1 << 20) % world: + raise ValueError("require CP >= 2 dividing 1M") + n = (1 << 20) // world + cases = [(f"uniform-{length}", [length] * ((1 << 20) // length), False) for length in (1024, 4096, 16384)] + cases += [ + ("near-uniform", [15360, 17408] * 32, False), + ("weak-decay-uniform", [16384] * 64, True), + ("isolated-ranks-with-crossing-peer", [n] * (world - 2) + [2 * n], False), + ] + results = [] + for name, lengths, weak in cases: + plan = KimiCPPlan.from_global_cu_seqlens((0, *accumulate(lengths)), rank=rank, world_size=world) + transport = AllGatherKimiCPTransport.prepare(plan, heads=32, key_dim=128, value_dim=128, device=device) + cu = torch.tensor(plan.local_cu_seqlens, dtype=torch.int32, device=device) + torch.manual_seed(88171 + rank) + q, k = [torch.nn.functional.normalize(torch.randn(n, 32, 128, device=device), dim=-1).bfloat16() for _ in range(2)] + v = torch.randn_like(q) + g = ( + (-0.0001 - 0.0004 * torch.rand(n, 32, 128, device=device)) + if weak + else (-0.01 - 0.04 * torch.rand(n, 32, 128, device=device)) + ) + beta = torch.sigmoid(torch.randn(n, 32, device=device)) + inputs = [x.requires_grad_() for x in (q, k, v, g, beta)] + dout = torch.randn_like(v) + reference = None + for label, active_plan in (("baseline", replace(plan, requires_boundary_exchange=True)), ("candidate", plan)): + samples = [] + for iteration in range(args.warmups + args.repeats): + for x in inputs: + x.grad = None + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, forward, end = [torch.cuda.Event(enable_timing=True) for _ in range(3)] + begin.record() + out = cudnn_kda_cp(*inputs, cu, plan=active_plan, group=dist.group.WORLD, boundary_transport=transport) + forward.record() + out.backward(dout) + end.record() + end.synchronize() + if iteration >= args.warmups: + samples.append( + dict( + total_ms=begin.elapsed_time(end), + fwd_ms=begin.elapsed_time(forward), + bwd_ms=forward.elapsed_time(end), + allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + ) + ) + if iteration + 1 < args.warmups + args.repeats: + del out + tensors = dict(output=out, **{key: x.grad for key, x in zip(("dq", "dk", "dv", "dg", "dbeta"), inputs)}) + errors = {} if reference is None else {key: compare_cpu_reference(x, reference[key]) for key, x in tensors.items()} + if reference is None: + reference = {key: x.detach().cpu() for key, x in tensors.items()} + record = dict( + case=name, + label=label, + rank=rank, + requires_boundary_exchange=active_plan.requires_boundary_exchange, + intrinsic_requires_boundary_exchange=plan.requires_boundary_exchange, + samples=samples, + errors=errors, + ) + gathered = [None] * world + dist.all_gather_object(gathered, record) + passed = all(e["finite"] and e["max_abs"] == 0 for r in gathered for e in r["errors"].values()) + if rank == 0: + results.append( + dict( + case=name, + label=label, + ranks=gathered, + passed=passed, + total_ms=statistics.median( + max(r["samples"][i]["total_ms"] for r in gathered) for i in range(args.repeats) + ), + ) + ) + print(name, label, results[-1]["total_ms"], "exact", passed, flush=True) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps( + dict( + world_size=world, + source_sha256=hashlib.sha256( + (Path(__file__).resolve().parents[1] / "cula/distributed/kda.py").read_bytes() + ).hexdigest(), + results=results, + ), + indent=2, + ) + + "\n" + ) + if not passed: + raise RuntimeError("independent-shard path changed output or gradients; see saved errors") + del out, tensors + for x in inputs: + x.grad = None + del reference, inputs, q, k, v, g, beta, dout, transport, cu + gc.collect() + torch.cuda.empty_cache() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_thd_transport.py b/benchmarks/bench_thd_transport.py new file mode 100644 index 00000000..9f069cf3 --- /dev/null +++ b/benchmarks/bench_thd_transport.py @@ -0,0 +1,204 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Calibrate selection transport independently of KDA/selection arrival skew. + +The caller supplies process-group topology. This uses the actual eager +redistributor and Q/weights/IDs/scores dtypes, with no attention kernels. +""" + +import argparse +import datetime +import gc +import json +import os +import statistics +from pathlib import Path + +import torch +import torch.distributed as dist + +from cula.distributed import THDRankLayout, THDRedistributionPlan, THDRedistributor, THDSpan + + +def transport_layout(world, rows, ranks_per_node, kind): + if world < 2 or rows < world or rows % world or rows % 4 or ranks_per_node < 1 or world % ranks_per_node: + raise ValueError("need complete nodes and row counts divisible by CP") + assignments = [[] for _ in range(world)] + for source in range(world): + if kind in ("mixed_dense_quarter", "mixed_dense_three_quarters"): + if ranks_per_node == world or rows % (4 * ranks_per_node): + raise ValueError("mixed validation needs multiple nodes and quarter-node aligned rows") + local_start = source // ranks_per_node * ranks_per_node + remote_start = (local_start + ranks_per_node) % world + remote_rows = rows // 4 * (1 if kind == "mixed_dense_quarter" else 3) + cursor = source * rows + for node_start, count in ((local_start, rows - remote_rows), (remote_start, remote_rows)): + for rank in range(node_start, node_start + ranks_per_node): + end = cursor + count // ranks_per_node + assignments[rank].append(THDSpan(0, cursor, end, 0, world * rows, rank)) + cursor = end + continue + if kind in ("intra_pair", "inter_pair"): + peer = 1 if kind == "intra_pair" else ranks_per_node + if peer >= world or (kind == "intra_pair" and ranks_per_node < 2): + raise ValueError("pair fixture needs two ranks on the declared link") + destinations = [peer if source == 0 else 0 if source == peer else source] + elif kind in ("intra", "intra_dense"): + node_start = source // ranks_per_node * ranks_per_node + destinations = ( + list(range(node_start, node_start + ranks_per_node)) + if kind == "intra_dense" + else [node_start + (source + 1) % ranks_per_node] + ) + if ranks_per_node == 1: + raise ValueError("intra-node calibration needs at least two ranks per node") + elif kind in ("inter", "inter_dense"): + if ranks_per_node == world: + raise ValueError("inter-node calibration needs at least two nodes") + next_node = ((source // ranks_per_node + 1) * ranks_per_node) % world + destinations = ( + list(range(next_node, next_node + ranks_per_node)) + if kind == "inter_dense" + else [(source + ranks_per_node) % world] + ) + elif kind == "striped": + destinations = list(range(world)) + else: + raise ValueError(f"unknown transport fixture: {kind}") + for i, rank in enumerate(destinations): + start = source * rows + i * rows // len(destinations) + end = source * rows + (i + 1) * rows // len(destinations) + assignments[rank].append(THDSpan(0, start, end, 0, world * rows, rank)) + return tuple(THDRankLayout(rank, tuple(spans)) for rank, spans in enumerate(assignments)) + + +def ordered(ranges): + cursor = 0 + for a, z in ranges: + if a != cursor: + return False + cursor += z - a + return True + + +def features(layouts, rows, rpn): + plans = [ + THDRedistributionPlan.build(layouts, tuple(i * rows for i in range(len(layouts) + 1)), r) for r in range(len(layouts)) + ] + edges = {(r, t): n for r, p in enumerate(plans) for t, n in enumerate(p.send_counts) if r != t and n} + intra_out = [0] * len(plans) + inter_out = intra_out.copy() + intra_in = intra_out.copy() + inter_in = intra_out.copy() + node_out = [0] * (len(plans) // rpn) + node_in = node_out.copy() + degree_out = [0] * len(plans) + degree_in = degree_out.copy() + for (s, t), n in edges.items(): + a, b = s // rpn, t // rpn + degree_out[s] += 1 + degree_in[t] += 1 + if a == b: + intra_out[s] += n + intra_in[t] += n + else: + inter_out[s] += n + inter_in[t] += n + node_out[a] += n + node_in[b] += n + return dict( + rank_intra_rows=intra_out + intra_in, + rank_inter_rows=inter_out + inter_in, + peer_rows=max(edges.values(), default=0), + node_rows=max(node_out + node_in), + peer_degree=max(degree_out + degree_in), + copies=max( + (0 if ordered(p.send_ranges) else p.source_rows) + (0 if ordered(p.receive_ranges) else p.target_rows) + for p in plans + ), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--validation-only", action="store_true") + parser.add_argument("--ranks-per-node", type=int, required=True) + parser.add_argument("--rows-per-rank", nargs="+", type=int, default=[256, 4096, 32768, 65536]) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=10) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.ranks_per_node < 1 or args.warmups < 0 or args.repeats < 1: + parser.error("require positive ranks-per-node/repeats and nonnegative warmups") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=30)) + world, rank = dist.get_world_size(), dist.get_rank() + if world < 2 or world % args.ranks_per_node: + raise ValueError("require CP >= 2 and complete nodes") + kinds = ["intra_pair", "intra", "intra_dense"] if args.ranks_per_node > 1 else [] + kinds += (["inter_pair", "inter", "inter_dense"] if world > args.ranks_per_node else []) + ["striped"] + sizes = args.rows_per_rank + if args.validation_only: + sizes, kinds = [8192, 49152], ["mixed_dense_quarter", "mixed_dense_three_quarters"] + # Validate every fixture before starting any timed GPU work. + for rows in sizes: + for kind in kinds: + transport_layout(world, rows, args.ranks_per_node, kind) + results = [] + for rows in sizes: + for kind in kinds: + layouts = transport_layout(world, rows, args.ranks_per_node, kind) + plan = THDRedistributionPlan.build(layouts, tuple(r * rows for r in range(world + 1)), rank) + exchange = THDRedistributor(plan, device, group=dist.group.WORLD) + marker = torch.arange(rank * rows, (rank + 1) * rows, device=device) + routed = exchange.distribute(marker) + restored = exchange.restore(routed) + expected = torch.cat([torch.arange(s.token_start, s.token_end, device=device) for s in layouts[rank].spans]) + okay = torch.tensor(int(torch.equal(marker, restored) and torch.equal(routed, expected)), device=device) + dist.all_reduce(okay, op=dist.ReduceOp.MIN) + if not okay.item(): + raise RuntimeError("transport calibration marker mismatch") + # H64/D128 BF16 query, BF16 weights; Top-K512 int32 IDs and FP32 scores. + source = [torch.zeros(rows, *shape, device=device, dtype=torch.bfloat16) for shape in [(64, 128), (64,)]] + target = [torch.zeros(rows, 512, device=device, dtype=dtype) for dtype in (torch.int32, torch.float32)] + samples = [] + for step in range(args.warmups + args.repeats): + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + begin, end = [torch.cuda.Event(enable_timing=True) for _ in range(2)] + begin.record() + outputs = [exchange.distribute(x) for x in source] + [exchange.restore(x) for x in target] + end.record() + end.synchronize() + if step >= args.warmups: + samples.append(begin.elapsed_time(end)) + del outputs + gathered = [None] * world + dist.all_gather_object(gathered, samples) + record = dict( + kind=kind, + rows=rows, + features=features(layouts, rows, args.ranks_per_node), + samples_ms=gathered, + critical_median_ms=statistics.median(max(s[i] for s in gathered) for i in range(args.repeats)), + marker_exact=True, + heldout=args.validation_only or rows == 32768 or kind == "striped", + ) + results.append(record) + if rank == 0: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(dict(world=world, ranks_per_node=args.ranks_per_node, records=results), indent=2) + "\n" + ) + print(rows, kind, record["critical_median_ms"], flush=True) + del source, target, marker, routed, expected, restored, exchange + gc.collect() + torch.cuda.empty_cache() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/calibrate_thd_selection.py b/benchmarks/calibrate_thd_selection.py new file mode 100644 index 00000000..12e4c272 --- /dev/null +++ b/benchmarks/calibrate_thd_selection.py @@ -0,0 +1,188 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Fit explicit CPU planner coefficients from saved selection-routing runs. + +This does not run GPU work or tune kernels. Use separate, held-out THD inputs +to evaluate the resulting policy; calibration fit alone is not validation. +""" + +import argparse +import hashlib +import json +import math +import statistics +from itertools import accumulate, combinations +from pathlib import Path + +from benchmarks.cp_layouts import query_layouts +from cula.distributed.thd_planner import _candidate_work + + +def _solve(matrix, rhs): + rows = [list(row) + [value] for row, value in zip(matrix, rhs)] + for col in range(len(rhs)): + pivot = max(range(col, len(rhs)), key=lambda r: abs(rows[r][col])) + if abs(rows[pivot][col]) < 1e-14: + return None + rows[col], rows[pivot] = rows[pivot], rows[col] + scale = rows[col][col] + rows[col] = [x / scale for x in rows[col]] + for r in range(len(rhs)): + if r != col: + scale = rows[r][col] + rows[r] = [a - scale * b for a, b in zip(rows[r], rows[col])] + return [row[-1] for row in rows] + + +def nonnegative_fit(features, targets): + """Exact active-set enumeration for three nonnegative cost coefficients.""" + if not features or len(features) != len(targets) or any(len(row) != 3 for row in features): + raise ValueError("need matched, nonempty measurements with three features") + if any(not math.isfinite(v) or v < 0 for row in features for v in row) or any(not math.isfinite(v) for v in targets): + raise ValueError("fit inputs must be finite and feature counts nonnegative") + best, error = [0.0] * 3, sum(y * y for y in targets) + for size in range(1, 4): + for active in combinations(range(3), size): + matrix = [[sum(x[i] * x[j] for x in features) for j in active] for i in active] + rhs = [sum(x[i] * y for x, y in zip(features, targets)) for i in active] + solution = _solve(matrix, rhs) + if solution is None or any(x < 0 for x in solution): + continue + weights = [0.0] * 3 + for i, value in zip(active, solution): + weights[i] = value + loss = sum((sum(a * b for a, b in zip(x, weights)) - y) ** 2 for x, y in zip(features, targets)) + if loss < error: + best, error = weights, loss + return best, math.sqrt(error / len(targets)) + + +def routing_critical_ms(samples): + """Critical-path increments, excluding pre-existing rank arrival skew. + + Each sample starts at the iteration's post-barrier event. At every phase, + max(cumulative end) - max(cumulative start) is the phase's contribution to + the distributed makespan. Summing local exchange durations instead would + count waiting for slow KDA/selection ranks again as network time. + """ + phases = ( + "query_redistribute_ms", + "pool_fwd_ms", + "pool_gather_ms", + "selection_key_layout_ms", + "selection_ms", + "selection_restore_ms", + ) + if not samples or any(not math.isfinite(s[k]) or s[k] < 0 for s in samples for k in ("kda_fwd_ms", *phases)): + raise ValueError("routing phases must be finite, nonnegative and nonempty") + arrival = [s["kda_fwd_ms"] for s in samples] + duration = 0.0 + for phase in phases: + before = max(arrival) + arrival = [a + s[phase] for a, s in zip(arrival, samples)] + if phase in ("query_redistribute_ms", "selection_key_layout_ms", "selection_restore_ms"): + duration += max(arrival) - before + return duration + + +def calibrate(paths, ranks_per_node, margin=0.05): + features, targets, rates, sources, digests = [], [], [], [], {} + world = None + # Current benchmark: BF16 H64/D128 Q, BF16 H64 weights, + # int32 IDs + FP32 scores for Top-K 512. + payload = 2 * 64 * 128 + 2 * 64 + (4 + 4) * 512 + for path in paths: + raw = json.loads(path.read_text()) + config = raw["config"] + if config["check"]: + continue + if config.get("redistribution_scope") != "selection" or raw.get("planner_cost_model") is not None: + raise ValueError("calibrate from unmodeled selection-only runs, not query-wide or already calibrated runs") + if config["top_k"] != 512: + raise ValueError("this benchmark profile requires indexer H64/D128 and Top-K 512") + if world is not None and world != raw["world_size"]: + raise ValueError("cannot combine different CP group sizes") + world = raw["world_size"] + bounds = (0, *accumulate(config["sequence_lengths"])) + ranks, selected = query_layouts(bounds, world, config["layout"]) + observed = sorted(raw["ranks"], key=lambda x: x["rank"]) + if [r["rank"] for r in observed] != list(range(world)): + raise ValueError("calibration requires exactly one result per process-group rank") + if any(not all(r[k] for k in ("finite", "input_match", "redistribution_bitwise")) for r in observed): + raise ValueError("cannot calibrate from a failed numerical/redistribution check") + if any( + not e["finite"] or not math.isfinite(e["relative_rms"]) or e["relative_rms"] >= 0.03 + for r in observed + for e in (r["layout_vs_contiguous"] or {}).values() + ): + raise ValueError("cannot calibrate from failed output/gradient comparisons") + if any(len(r["samples"]) != config["repeats"] for r in observed): + raise ValueError("incomplete timing samples") + if selected != observed[0]["selected_policy"]: + raise ValueError("source planner differs from the measured assignment") + for layout, result in zip(ranks, observed): + candidates = sum(_candidate_work(s.causal_offset, s.tokens, 4) for s in layout.spans) + features.append([candidates / 1e9, layout.local_tokens / 1e6, len(layout.spans) / 1000]) + targets.append(statistics.median(s["selection_ms"] for s in result["samples"])) + repeats = len(observed[0]["samples"]) + duration = statistics.median(routing_critical_ms([r["samples"][i] for r in observed]) for i in range(repeats)) + remote_rows = max(r["remote_query_rows"] for r in observed) + if remote_rows: + if not math.isfinite(duration) or duration <= 0: + raise ValueError("routing timings must be finite and positive") + rates.append(remote_rows * payload / (duration * 1e6)) + sources.append(str(path)) + digests[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() + if not features or not rates or world is None: + raise ValueError("need measured selection work and at least one nonidentity routing case") + if ranks_per_node < 1 or world % ranks_per_node: + raise ValueError("ranks_per_node must divide the measured group size") + coefficients, rmse = nonnegative_fit(features, targets) + # The measurements do not isolate NVLink vs inter-node links. Use one + # conservative effective rate, including packing, for both. Cumulative + # critical increments remove pre-existing arrival skew from the fit. + # Do not present it as a physical-link bandwidth measurement. + rate = min(rates) + return dict( + schema_version=2, + model=dict( + candidate_ns=coefficients[0] / 1000, + query_ns=coefficients[1], + span_us=coefficients[2], + bytes_per_query=payload, + intra_node_gbps=rate, + inter_node_gbps=rate, + rank_node_ids=[r // ranks_per_node for r in range(world)], + packing_ns_per_query=0.0, + exchange_us=0.0, + min_gain_fraction=margin, + ), + calibration=dict( + sources=sources, + source_sha256=digests, + samples=len(targets), + selection_fit_rmse_ms=rmse, + observed_routing_effective_gbps=rates, + transport_note="Critical phase increments remove pre-existing rank arrival skew. Shared conservative effective rate includes packing; links were not isolated.", + validation_note="Evaluate held-out layouts and seeds; fitting is not a performance guarantee.", + ), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--records", type=Path, nargs="+", required=True) + parser.add_argument("--ranks-per-node", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + paths = sorted({p for root in args.records for p in root.glob("layout-1m-*.json")}) + result = calibrate(paths, args.ranks_per_node) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["model"], indent=2)) + print("selection fit RMSE ms", result["calibration"]["selection_fit_rmse_ms"]) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cp_layouts.py b/benchmarks/cp_layouts.py new file mode 100644 index 00000000..5370d052 --- /dev/null +++ b/benchmarks/cp_layouts.py @@ -0,0 +1,37 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Comparable contiguous, per-document zigzag and adaptive THD layouts.""" + +from cula._metadata import balanced_shards +from cula.distributed import THDRankLayout, THDSpan, plan_thd_layout, plan_thd_selection, zigzag_thd_layout + + +def query_layouts(boundaries, world, policy, *, cost_model=None, planner_strategy="balanced"): + if policy == "planner": + if planner_strategy == "selection": + if cost_model is None: + raise ValueError("selection planning requires calibrated costs") + source_rows = 4 * ((boundaries[-1] // 4 + world - 1) // world) + plan = plan_thd_selection(boundaries, world_size=world, cost_model=cost_model, max_tokens_per_rank=2 * source_rows) + elif planner_strategy == "balanced": + plan = plan_thd_layout(boundaries, world_size=world, cost_model=cost_model) + else: + raise ValueError(f"unknown planner strategy: {planner_strategy}") + return plan.ranks, plan.policy + assignments = [[] for _ in range(world)] + if policy == "contiguous": + offsets = tuple(n * 4 for n in balanced_shards(boundaries[-1] // 4, world)) + for rank, (left, right) in enumerate(zip(offsets, offsets[1:])): + for seq, (start, end) in enumerate(zip(boundaries, boundaries[1:])): + a, z = max(left, start), min(right, end) + if a < z: + assignments[rank].append(THDSpan(seq, a, z, start, end, rank)) + elif policy == "zigzag": + return zigzag_thd_layout(boundaries, world_size=world), policy + else: + raise ValueError(f"unknown layout: {policy}") + ranks = tuple( + THDRankLayout(rank, tuple(sorted(spans, key=lambda s: s.token_start))) for rank, spans in enumerate(assignments) + ) + return ranks, policy diff --git a/benchmarks/fit_thd_transport.py b/benchmarks/fit_thd_transport.py new file mode 100644 index 00000000..0afbeeff --- /dev/null +++ b/benchmarks/fit_thd_transport.py @@ -0,0 +1,108 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Fit nonnegative effective transport costs from independent routing sweeps. + +SciPy is a calibration-only dependency. Production planning does not import it. +Held-out fixtures are reported, never included in the optimization objective. +""" + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np + + +def predict(features, coefficients, overlap_links=False): + """Same ordered-phase resource model used by THDSelectionCostModel.""" + intra, inter, peer, node, packing, startup, fixed = coefficients + unit = 65536 + combine = max if overlap_links else lambda a, b: a + b + rank = max(combine(a * intra, b * inter) for a, b in zip(features["rank_intra_rows"], features["rank_inter_rows"])) / unit + transfer = max(rank, features["peer_rows"] * peer / unit, features["node_rows"] * node / unit) + return transfer + features["copies"] * packing / unit + features["peer_degree"] * startup + fixed + + +def fit(raw, profile, *, overlap_links=False): + from scipy.optimize import differential_evolution, least_squares + + rows = [r for r in raw["records"] if not r["heldout"]] + if len(rows) < 6 or any(not r["marker_exact"] or r["critical_median_ms"] <= 0 for r in raw["records"]): + raise ValueError("need complete, positive transport timings and exact row markers") + y = np.array([r["critical_median_ms"] for r in rows]) + + def residual(x): + return (np.array([predict(r["features"], x, overlap_links) for r in rows]) - y) / np.maximum(y, 0.25) + + # Inactive max-resource branches have zero derivative. A bounded global + # initialization avoids silently accepting a rank-bandwidth coefficient + # that never became active during local least-squares optimization. + scale = max(y) + startup_limit = min(r["critical_median_ms"] / max(r["features"]["peer_degree"], 1) for r in rows) + bounds = [(0, 2 * scale)] * 5 + [(0, startup_limit), (0, min(y))] + initialized = differential_evolution( + lambda x: np.sum(residual(x) ** 2), bounds, seed=1, popsize=10, maxiter=300, tol=1e-8, polish=False + ) + best = least_squares(residual, initialized.x, bounds=(0, np.inf), max_nfev=4000, ftol=1e-10, xtol=1e-10, gtol=1e-10) + x = best.x + # Ordered sweep fixtures elide every permutation. Their timings cannot + # identify copy bandwidth; use a separately measured packing coefficient. + if not any(r["features"]["copies"] for r in rows): + x[4] = profile["model"].get("packing_ns_per_query", 0) * 65536 / 1e6 + # A single-node calibration cannot identify inter-node bandwidth. + # Use the same unused fallback rate, not an arbitrary optimizer parameter. + model = dict(profile["model"], overlap_intra_inter=overlap_links) + factor = 65536 * model["bytes_per_query"] / 1e6 + model.update( + intra_node_gbps=factor / max(x[0], 1e-10), + inter_node_gbps=factor / max(x[1] if raw["world"] > raw["ranks_per_node"] else x[0], 1e-10), + peer_gbps=factor / max(x[2], 1e-10), + inter_node_aggregate_gbps=factor / max(x[3], 1e-10) if raw["world"] > raw["ranks_per_node"] else None, + packing_ns_per_query=x[4] * 1e6 / 65536, + peer_startup_us=x[5] * 1000, + exchange_us=x[6] * 1000, + ) + predictions = [ + dict( + kind=r["kind"], + rows=r["rows"], + heldout=r["heldout"], + actual_ms=r["critical_median_ms"], + predicted_ms=predict(r["features"], x, overlap_links), + relative_error=predict(r["features"], x, overlap_links) / r["critical_median_ms"] - 1, + ) + for r in raw["records"] + ] + return dict( + profile, + schema_version=5, + model=model, + transport_calibration=dict( + method="Nonnegative max-resource fit; independent eager transport timings", + coefficients_ms_per_65536_rows=x.tolist(), + predictions=predictions, + ranks_per_node=raw["ranks_per_node"], + world_size=raw["world"], + heldout_rule="32768 rows or striped layout; never used in fitting", + note="Effective coefficients include eager implementation costs; not physical bandwidth or universal optimality guarantees.", + ), + ) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--records", type=Path, required=True) + p.add_argument("--profile", type=Path, required=True) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--overlap-links", action="store_true") + a = p.parse_args() + result = fit(json.loads(a.records.read_text()), json.loads(a.profile.read_text()), overlap_links=a.overlap_links) + result["transport_calibration"]["source_sha256"] = hashlib.sha256(a.records.read_bytes()).hexdigest() + a.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["model"], indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/profiles/README.md b/benchmarks/profiles/README.md new file mode 100644 index 00000000..0a33d33a --- /dev/null +++ b/benchmarks/profiles/README.md @@ -0,0 +1,23 @@ +# THD selection profile examples + +These are explicit profiles from the 2026-11-15 real-CP experiment, not library +defaults or physical-link bandwidth specifications. They apply to indexer +H64/D128, Top-K512, BF16 query/weights and int32 IDs/FP32 scores, at 1M total +tokens with eager PyTorch/NCCL variable-split all-to-all. + +- `thd_selection_h20_cp16.json`: two nodes, eight H20-3e ranks per node. +- `thd_selection_b300_cp8.json`: one node, eight B300 ranks. The inter-node + rate is unused; it is not a cross-node Blackwell measurement. + +The profiles retain compute-fit provenance and single-peer/dense/striped +transport samples. `min_gain_ms` is a conservative **single-use preparation +budget**, 40 ms and 20 ms respectively, not a measured kernel cost. A caller +reusing a plan can explicitly amortize it; zero keeps throughput-only planning. +No plan is selected by a case name, document-count lookup, or hardware detector. + +Pass one explicitly to `bench_csa_layout_matrix.py --planner-cost-model` with +`--planner-strategy selection`. Recalibrate with `calibrate_thd_selection.py` +and the independent `bench_thd_transport.py` / `fit_thd_transport.py` workflow for other indexer shapes, process-group topology, +message sizes or NCCL/software configuration. See +[the experiment report](../../docs/THD_SELECTION_RESULTS.md) for measured +latency, precision, memory and limitations. diff --git a/benchmarks/profiles/thd_selection_b300_cp8.json b/benchmarks/profiles/thd_selection_b300_cp8.json new file mode 100644 index 00000000..bfc73640 --- /dev/null +++ b/benchmarks/profiles/thd_selection_b300_cp8.json @@ -0,0 +1,646 @@ +{ + "schema_version": 5, + "model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "min_gain_ms": 20, + "overlap_intra_inter": false, + "peer_gbps": 408.72149682964135, + "inter_node_aggregate_gbps": null, + "peer_startup_us": 7.460149512894069e-21 + }, + "calibration": { + "sources": [ + "layout-1m-20s2l-contiguous.json", + "layout-1m-20s2l-zigzag.json", + "layout-1m-count-mixed-256-1long-contiguous.json", + "layout-1m-count-mixed-256-1long-zigzag.json", + "layout-1m-count-mixed-512-2long-contiguous.json", + "layout-1m-count-mixed-512-2long-zigzag.json", + "layout-1m-count-tiny-mix-512-contiguous.json", + "layout-1m-count-tiny-mix-512-zigzag.json", + "layout-1m-count-uniform-256-contiguous.json", + "layout-1m-count-uniform-256-zigzag.json", + "layout-1m-single-contiguous.json", + "layout-1m-single-zigzag.json", + "layout-1m-uniform-contiguous.json", + "layout-1m-uniform-zigzag.json" + ], + "source_sha256": { + "layout-1m-20s2l-contiguous.json": "31c62725b5ff3567bd916db656b186051ca80f30c37f0cb3a1f50837311f14e9", + "layout-1m-20s2l-zigzag.json": "7e1fff2be4be88a44ed5670becc04946bdaec464ee4a9ddcb5a501976d0de389", + "layout-1m-count-mixed-256-1long-contiguous.json": "54a36612d35922e9a2d8032b4a641a0afc3484191ce9d3fa19d9328c3efd22e2", + "layout-1m-count-mixed-256-1long-zigzag.json": "73c6f8f9e70a9c1e6896ea4ca412f8fe467ff3525fc757329c8ee9520f39c377", + "layout-1m-count-mixed-512-2long-contiguous.json": "f8786cc6ff782bf0b48bcf3d1acfac5de0d328216e0790d29852f139ddbaaf07", + "layout-1m-count-mixed-512-2long-zigzag.json": "3d08c30f9b05728c06c3564ea7b88f27a7bd523b0de17f1df612818504d702a4", + "layout-1m-count-tiny-mix-512-contiguous.json": "644d77f672d4a16a7cc27a8cb68e0bd9f6a55221d5d6b533b22b9514eccb6473", + "layout-1m-count-tiny-mix-512-zigzag.json": "5bf51c02bb8a083e8e19614aef4720efc3cfbc413ac3780fdba924c9b8849695", + "layout-1m-count-uniform-256-contiguous.json": "dfda88005564f268e7140f5e42c60fc8a7a670402cba201a811b7f093e084b6b", + "layout-1m-count-uniform-256-zigzag.json": "2450fdc87d83207dd66e333bffbbb3f89123b18afbfdac6b6d0bfa3f649c91af", + "layout-1m-single-contiguous.json": "ed4598764f4760f8467d40e9a351cc5586129a3a23180fb0a35bf71aedb10d6f", + "layout-1m-single-zigzag.json": "b6ab3b9f7cf1a89dd1ebb1d76980781276d728e559e8e8b7ee23f15de557c112", + "layout-1m-uniform-contiguous.json": "da9f168ffb4109bcc6bab5e6f7112d9ea99741406fcbcacd9fdb57f84952cfca", + "layout-1m-uniform-zigzag.json": "2ce0a59edde0db530c63d5c20a09bccaa41a9865b109ad5e10bda46a0d43f246" + }, + "samples": 112, + "selection_fit_rmse_ms": 4.720245923945194, + "observed_routing_effective_gbps": [ + 305.18791099916496, + 331.06902736714636, + 390.5160264901337, + 387.31209063170303, + 442.2725855454942, + 330.9084931142364, + 441.57185423427615 + ], + "transport_note": "Critical phase increments remove pre-existing rank arrival skew. Shared conservative effective rate includes packing; links were not isolated.", + "validation_note": "Evaluate held-out layouts and seeds; fitting is not a performance guarantee." + }, + "packing_calibration": { + "world": 8, + "records": [ + { + "rows": 256, + "critical_median_ms": 0.043455999344587326, + "ns_per_query": 169.74999743979424, + "samples_ms": [ + [ + 0.04495999962091446, + 0.04396799951791763, + 0.04649600014090538, + 0.04335999861359596, + 0.04291199892759323, + 0.04259200021624565, + 0.04383999854326248, + 0.04214400053024292, + 0.043327998369932175, + 0.0424639992415905 + ], + [ + 0.04476799815893173, + 0.0435199998319149, + 0.048895999789237976, + 0.042208001017570496, + 0.042047999799251556, + 0.04230400174856186, + 0.042047999799251556, + 0.04211200028657913, + 0.04320000112056732, + 0.042047999799251556 + ], + [ + 0.04095999896526337, + 0.03951999917626381, + 0.040383998304605484, + 0.03872000053524971, + 0.038495998829603195, + 0.038047999143600464, + 0.03923200070858002, + 0.039103999733924866, + 0.03836800158023834, + 0.03961599990725517 + ], + [ + 0.04243199899792671, + 0.04185599833726883, + 0.04134399816393852, + 0.03903999924659729, + 0.03766399994492531, + 0.03961599990725517, + 0.04838399961590767, + 0.038656000047922134, + 0.039103999733924866, + 0.03852799907326698 + ], + [ + 0.04032000154256821, + 0.03932800143957138, + 0.04016000032424927, + 0.039712000638246536, + 0.038656000047922134, + 0.041600000113248825, + 0.03907199949026108, + 0.03929600119590759, + 0.03862399980425835, + 0.03827200084924698 + ], + [ + 0.040608000010252, + 0.03936000168323517, + 0.04022400081157684, + 0.03920000046491623, + 0.038336001336574554, + 0.03948799893260002, + 0.040063999593257904, + 0.040192000567913055, + 0.03984000161290169, + 0.039103999733924866 + ], + [ + 0.044544000178575516, + 0.04294399917125702, + 0.04396799951791763, + 0.04281599819660187, + 0.04262400045990944, + 0.04320000112056732, + 0.04339199885725975, + 0.04259200021624565, + 0.04355200007557869, + 0.041919998824596405 + ], + [ + 0.04012800008058548, + 0.03999999910593033, + 0.03891199827194214, + 0.03891199827194214, + 0.03743999823927879, + 0.03852799907326698, + 0.038176000118255615, + 0.03779200091958046, + 0.03862399980425835, + 0.037856001406908035 + ] + ], + "marker_exact": true + }, + { + "rows": 4096, + "critical_median_ms": 0.058239998295903206, + "ns_per_query": 14.218749583960744, + "samples_ms": [ + [ + 0.07593599706888199, + 0.05734400078654289, + 0.05615999922156334, + 0.0570559985935688, + 0.05584000051021576, + 0.0575999990105629, + 0.05711999908089638, + 0.05859199911355972, + 0.05488000065088272, + 0.05523199960589409 + ], + [ + 0.07100799679756165, + 0.05907199904322624, + 0.05846399813890457, + 0.05632000043988228, + 0.05567999929189682, + 0.05801599845290184, + 0.057023998349905014, + 0.05523199960589409, + 0.05689600110054016, + 0.05907199904322624 + ], + [ + 0.0663679987192154, + 0.05756799876689911, + 0.05552000179886818, + 0.05375999957323074, + 0.05369599908590317, + 0.05728000029921532, + 0.05644800141453743, + 0.053247999399900436, + 0.055296000093221664, + 0.05552000179886818 + ], + [ + 0.06598400324583054, + 0.05648000165820122, + 0.05686400085687637, + 0.055743999779224396, + 0.05571199953556061, + 0.05580800026655197, + 0.05587200075387955, + 0.055296000093221664, + 0.0541439987719059, + 0.055424001067876816 + ], + [ + 0.06329599767923355, + 0.05728000029921532, + 0.05596800148487091, + 0.056223999708890915, + 0.05305600166320801, + 0.056671999394893646, + 0.05353600159287453, + 0.05331199988722801, + 0.05516799911856651, + 0.05398400127887726 + ], + [ + 0.06611199676990509, + 0.05660799890756607, + 0.05644800141453743, + 0.05475199967622757, + 0.05417599901556969, + 0.056992001831531525, + 0.05615999922156334, + 0.05369599908590317, + 0.05526399984955788, + 0.05612799897789955 + ], + [ + 0.06188800185918808, + 0.05804799869656563, + 0.05782400071620941, + 0.05564799904823303, + 0.05523199960589409, + 0.05686400085687637, + 0.05711999908089638, + 0.05398400127887726, + 0.057631999254226685, + 0.056543998420238495 + ], + [ + 0.05507199838757515, + 0.056992001831531525, + 0.056223999708890915, + 0.05503999814391136, + 0.0533440001308918, + 0.05676800012588501, + 0.05478399991989136, + 0.05350400134921074, + 0.05503999814391136, + 0.054687999188899994 + ] + ], + "marker_exact": true + }, + { + "rows": 32768, + "critical_median_ms": 0.2985280007123947, + "ns_per_query": 9.110351584240561, + "samples_ms": [ + [ + 0.29609599709510803, + 0.3033280074596405, + 0.30131199955940247, + 0.29923200607299805, + 0.2980160117149353, + 0.2963840067386627, + 0.29635199904441833, + 0.29721599817276, + 0.30006399750709534, + 0.2970559895038605 + ], + [ + 0.29766398668289185, + 0.2970559895038605, + 0.2961919903755188, + 0.29846400022506714, + 0.2975040078163147, + 0.2964800000190735, + 0.29865598678588867, + 0.2950400114059448, + 0.29760000109672546, + 0.2967680096626282 + ], + [ + 0.2961919903755188, + 0.3020800054073334, + 0.29583999514579773, + 0.2961919903755188, + 0.297760009765625, + 0.2956799864768982, + 0.297791987657547, + 0.29552000761032104, + 0.2959679961204529, + 0.2950400114059448 + ], + [ + 0.29769599437713623, + 0.29654398560523987, + 0.29683199524879456, + 0.29657599329948425, + 0.2956480085849762, + 0.2948159873485565, + 0.296671986579895, + 0.29606398940086365, + 0.29574400186538696, + 0.2954240143299103 + ], + [ + 0.296671986579895, + 0.2959040105342865, + 0.2970240116119385, + 0.29583999514579773, + 0.2957119941711426, + 0.2964160144329071, + 0.29657599329948425, + 0.2970559895038605, + 0.2959679961204529, + 0.29574400186538696 + ], + [ + 0.2969599962234497, + 0.2972800135612488, + 0.29712000489234924, + 0.2977280020713806, + 0.29600000381469727, + 0.295199990272522, + 0.29580798745155334, + 0.2954240143299103, + 0.29609599709510803, + 0.29740801453590393 + ], + [ + 0.29606398940086365, + 0.2993279993534088, + 0.29760000109672546, + 0.29651200771331787, + 0.29712000489234924, + 0.29686400294303894, + 0.29721599817276, + 0.29654398560523987, + 0.2970240116119385, + 0.29840001463890076 + ], + [ + 0.2945280075073242, + 0.2952960133552551, + 0.2947840094566345, + 0.2972480058670044, + 0.29631999135017395, + 0.2950400114059448, + 0.29500800371170044, + 0.29440000653266907, + 0.29449599981307983, + 0.2954240143299103 + ] + ], + "marker_exact": true + }, + { + "rows": 131072, + "critical_median_ms": 1.1141119599342346, + "ns_per_query": 8.499999694322469, + "samples_ms": [ + [ + 1.115007996559143, + 1.1131839752197266, + 1.1146240234375, + 1.113759994506836, + 1.1129599809646606, + 1.1128000020980835, + 1.113919973373413, + 1.1228480339050293, + 1.1110080480575562, + 1.1130239963531494 + ], + [ + 1.1131199598312378, + 1.1105279922485352, + 1.1111680269241333, + 1.1111040115356445, + 1.1095679998397827, + 1.1131839752197266, + 1.1103999614715576, + 1.1095679998397827, + 1.1111680269241333, + 1.107103943824768 + ], + [ + 1.1123839616775513, + 1.111583948135376, + 1.1119040250778198, + 1.1112639904022217, + 1.1103999614715576, + 1.1127040386199951, + 1.1113280057907104, + 1.1091200113296509, + 1.111807942390442, + 1.1102080345153809 + ], + [ + 1.107424020767212, + 1.1072640419006348, + 1.1094720363616943, + 1.10806405544281, + 1.1081279516220093, + 1.1077439785003662, + 1.1066880226135254, + 1.1055999994277954, + 1.1091519594192505, + 1.1077439785003662 + ], + [ + 1.1094720363616943, + 1.1099519729614258, + 1.1125439405441284, + 1.111232042312622, + 1.1100480556488037, + 1.1092480421066284, + 1.1102080345153809, + 1.1091840267181396, + 1.112831950187683, + 1.1085439920425415 + ], + [ + 1.111807942390442, + 1.1117440462112427, + 1.1116160154342651, + 1.1108479499816895, + 1.1138559579849243, + 1.1098239421844482, + 1.1092480421066284, + 1.1085760593414307, + 1.114367961883545, + 1.1143039464950562 + ], + [ + 1.113312005996704, + 1.1125760078430176, + 1.1120959520339966, + 1.1128640174865723, + 1.111840009689331, + 1.1126400232315063, + 1.1093120574951172, + 1.112064003944397, + 1.1125760078430176, + 1.1100159883499146 + ], + [ + 1.1059839725494385, + 1.106719970703125, + 1.106112003326416, + 1.1084480285644531, + 1.1059199571609497, + 1.1046080589294434, + 1.1082559823989868, + 1.10806405544281, + 1.10697603225708, + 1.1062719821929932 + ] + ], + "marker_exact": true + } + ] + }, + "gain_policy": "Retain declared 5% / 20 ms single-use migration headroom; no heldout E2E tuning.", + "transport_calibration": { + "method": "Nonnegative max-resource fit; independent eager transport timings", + "coefficients_ms_per_65536_rows": [ + 2.0728231593528488, + 5.25403206689115, + 3.3043671509231323, + 8.017647329379857, + 0.5570559799671173, + 7.460149512894069e-24, + 0.1813690545886826 + ], + "predictions": [ + { + "kind": "intra_pair", + "rows": 256, + "heldout": false, + "actual_ms": 0.17163199931383133, + "predicted_ms": 0.1942767387719761, + "relative_error": 0.13193774790642965 + }, + { + "kind": "intra", + "rows": 256, + "heldout": false, + "actual_ms": 0.17878399789333344, + "predicted_ms": 0.1942767387719761, + "relative_error": 0.08665619440888661 + }, + { + "kind": "intra_dense", + "rows": 256, + "heldout": false, + "actual_ms": 0.1780639961361885, + "predicted_ms": 0.1884538993716269, + "relative_error": 0.05834926465141166 + }, + { + "kind": "striped", + "rows": 256, + "heldout": true, + "actual_ms": 0.18012800067663193, + "predicted_ms": 0.1884538993716269, + "relative_error": 0.04622212351061261 + }, + { + "kind": "intra_pair", + "rows": 4096, + "heldout": false, + "actual_ms": 0.6821600198745728, + "predicted_ms": 0.38789200152137837, + "relative_error": -0.431376817432515 + }, + { + "kind": "intra", + "rows": 4096, + "heldout": false, + "actual_ms": 1.0989919900894165, + "predicted_ms": 0.38789200152137837, + "relative_error": -0.6470474716655409 + }, + { + "kind": "intra_dense", + "rows": 4096, + "heldout": false, + "actual_ms": 0.2651199996471405, + "predicted_ms": 0.29472657111579154, + "relative_error": 0.11167234274311899 + }, + { + "kind": "striped", + "rows": 4096, + "heldout": true, + "actual_ms": 0.26867200434207916, + "predicted_ms": 0.29472657111579154, + "relative_error": 0.09697536904715665 + }, + { + "kind": "intra_pair", + "rows": 32768, + "heldout": true, + "actual_ms": 2.0732479095458984, + "predicted_ms": 1.8335526300502487, + "relative_error": -0.11561341911501066 + }, + { + "kind": "intra", + "rows": 32768, + "heldout": true, + "actual_ms": 2.6429920196533203, + "predicted_ms": 1.8335526300502487, + "relative_error": -0.3062587338834437 + }, + { + "kind": "intra_dense", + "rows": 32768, + "heldout": true, + "actual_ms": 1.1571199893951416, + "predicted_ms": 1.088229186805554, + "relative_error": -0.05953643807121389 + }, + { + "kind": "striped", + "rows": 32768, + "heldout": true, + "actual_ms": 1.1546720266342163, + "predicted_ms": 1.088229186805554, + "relative_error": -0.05754260802726663 + }, + { + "kind": "intra_pair", + "rows": 131072, + "heldout": false, + "actual_ms": 6.2092320919036865, + "predicted_ms": 6.790103356434948, + "relative_error": 0.09354961385461302 + }, + { + "kind": "intra", + "rows": 131072, + "heldout": false, + "actual_ms": 6.001280069351196, + "predicted_ms": 6.790103356434948, + "relative_error": 0.1314425052602206 + }, + { + "kind": "intra_dense", + "rows": 131072, + "heldout": false, + "actual_ms": 4.027600049972534, + "predicted_ms": 3.808809583456168, + "relative_error": -0.05432278870834217 + }, + { + "kind": "striped", + "rows": 131072, + "heldout": true, + "actual_ms": 4.0313920974731445, + "predicted_ms": 3.808809583456168, + "relative_error": -0.05521232086466865 + } + ], + "ranks_per_node": 8, + "world_size": 8, + "heldout_rule": "32768 rows or striped layout; never used in fitting", + "note": "Effective coefficients include eager implementation costs; not physical bandwidth or universal optimality guarantees.", + "source_sha256": "a965b310b107366576791580b49e74f6b987ffd4307107518666fd444c061f11" + }, + "cudnn_source": "855ccd77ba9efed22441fd454f7cc0206a068e94" +} diff --git a/benchmarks/profiles/thd_selection_h20_cp16.json b/benchmarks/profiles/thd_selection_h20_cp16.json new file mode 100644 index 00000000..4c6bbae6 --- /dev/null +++ b/benchmarks/profiles/thd_selection_h20_cp16.json @@ -0,0 +1,1168 @@ +{ + "schema_version": 5, + "model": { + "candidate_ns": 0.12856368106272384, + "query_ns": 9.515262475665075, + "span_us": 6.381563913018475, + "bytes_per_query": 20608, + "intra_node_gbps": 43.3481035770435, + "inter_node_gbps": 20.68645784116552, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "packing_ns_per_query": 14.45166026314837, + "exchange_us": 3.9301084700615614, + "min_gain_fraction": 0.05, + "min_gain_ms": 40, + "overlap_intra_inter": true, + "peer_gbps": 12.233793384063304, + "inter_node_aggregate_gbps": null, + "peer_startup_us": 25.098063419906637 + }, + "calibration": { + "sources": [ + "layout-1m-20s2l-contiguous.json", + "layout-1m-20s2l-zigzag.json", + "layout-1m-count-mixed-256-1long-contiguous.json", + "layout-1m-count-mixed-256-1long-zigzag.json", + "layout-1m-count-mixed-512-2long-contiguous.json", + "layout-1m-count-mixed-512-2long-zigzag.json", + "layout-1m-count-tiny-mix-512-contiguous.json", + "layout-1m-count-tiny-mix-512-zigzag.json", + "layout-1m-count-uniform-256-contiguous.json", + "layout-1m-count-uniform-256-zigzag.json", + "layout-1m-single-contiguous.json", + "layout-1m-single-zigzag.json", + "layout-1m-uniform-contiguous.json", + "layout-1m-uniform-zigzag.json" + ], + "source_sha256": { + "layout-1m-20s2l-contiguous.json": "5088f225f871b029b39697501736b82dd8d3823ed48c24c15ff90174c4c15d1f", + "layout-1m-20s2l-zigzag.json": "40cab9e437964fdf61d1c3a8e7f0972a709075099299241182035e44986490ae", + "layout-1m-count-mixed-256-1long-contiguous.json": "8ba6524aa34c393f62c7231df830e30e27265f02d4c458d90521bd5b5d2f2c08", + "layout-1m-count-mixed-256-1long-zigzag.json": "3762489337627a368407b485c2a7ed02d60fada369c7d637d5c4e947bfbfbc8d", + "layout-1m-count-mixed-512-2long-contiguous.json": "d77f701c8180e3ec93b0f72551cf64cb04c76f7cf4ce4f7531eaa32b72258298", + "layout-1m-count-mixed-512-2long-zigzag.json": "e34a2928e4ee83ea68742c2963b75a9eb059c3fd7c3ad2de95a714497977d41b", + "layout-1m-count-tiny-mix-512-contiguous.json": "ee7b848ac033ced04bb4ca6af87b50fd6a41dd3b38e4d48579fad45bf2cef6d2", + "layout-1m-count-tiny-mix-512-zigzag.json": "f563c2d0c1a3dc1043d0d358891250297ab9d1aecd7bd69ee723f23a68792eb4", + "layout-1m-count-uniform-256-contiguous.json": "76a035301c254c6a000cac83629f85caac91079b07d2d79e25a6b67aad355d0c", + "layout-1m-count-uniform-256-zigzag.json": "7854f5a37814e4d97760324d11054bac40fee1bc10d9262589b005ee105451ac", + "layout-1m-single-contiguous.json": "628d7b57b8edf1a2191e75bc666ed9dcc37e9ecb875297f68590bb0623cf682a", + "layout-1m-single-zigzag.json": "58786fa4f74b6cc644d2143647f65b644cab602254d6fa201ee5dd2c414d1e97", + "layout-1m-uniform-contiguous.json": "70e57ca6facc163aea921505d2bf839b611618f9088f440a414c3a4f1be84301", + "layout-1m-uniform-zigzag.json": "26d5ecbaf81f5c0311cb08dcc218c1ecb3dd569a442358e61bc790aa7710c965" + }, + "samples": 224, + "selection_fit_rmse_ms": 2.949003900413, + "observed_routing_effective_gbps": [ + 21.003766295884223, + 22.729257916064537, + 37.48939138205569, + 38.61166479509942, + 39.58319101689069, + 20.138357216688668, + 39.28377120939799 + ], + "transport_note": "Critical phase increments remove pre-existing rank arrival skew. Shared conservative effective rate includes packing; links were not isolated.", + "validation_note": "Evaluate held-out layouts and seeds; fitting is not a performance guarantee." + }, + "packing_calibration": { + "world": 16, + "records": [ + { + "rows": 256, + "critical_median_ms": 0.04848000034689903, + "ns_per_query": 189.37500135507435, + "samples_ms": [ + [ + 0.04598399996757507, + 0.045152001082897186, + 0.04572800174355507, + 0.04447999969124794, + 0.043136000633239746, + 0.04396799951791763, + 0.04396799951791763, + 0.04816000163555145, + 0.04335999861359596, + 0.04307200014591217 + ], + [ + 0.04633599892258644, + 0.0459199994802475, + 0.04368000105023384, + 0.04447999969124794, + 0.04291199892759323, + 0.04335999861359596, + 0.042688000947237015, + 0.04633599892258644, + 0.04416000097990036, + 0.042080000042915344 + ], + [ + 0.05820799991488457, + 0.04732799902558327, + 0.045632001012563705, + 0.04473600164055824, + 0.043807998299598694, + 0.04886399954557419, + 0.04492799937725067, + 0.04790399968624115, + 0.04390399903059006, + 0.04342399910092354 + ], + [ + 0.04710400104522705, + 0.04825599864125252, + 0.045024000108242035, + 0.04383999854326248, + 0.043616000562906265, + 0.04636799916625023, + 0.04451199993491173, + 0.04790399968624115, + 0.04374400153756142, + 0.04224000126123428 + ], + [ + 0.04668800160288811, + 0.04707200080156326, + 0.0451200008392334, + 0.04467200115323067, + 0.044415999203920364, + 0.04374400153756142, + 0.04297599941492081, + 0.04601600021123886, + 0.04335999861359596, + 0.043136000633239746 + ], + [ + 0.04652800038456917, + 0.04665600135922432, + 0.04476799815893173, + 0.045632001012563705, + 0.04307200014591217, + 0.043935999274253845, + 0.0432640016078949, + 0.048128001391887665, + 0.04371200129389763, + 0.04371200129389763 + ], + [ + 0.04787199944257736, + 0.04896000027656555, + 0.045504000037908554, + 0.044544000178575516, + 0.04399999976158142, + 0.044544000178575516, + 0.043807998299598694, + 0.05251200124621391, + 0.04623999819159508, + 0.04435199871659279 + ], + [ + 0.04713600128889084, + 0.04572800174355507, + 0.04396799951791763, + 0.04499199986457825, + 0.04342399910092354, + 0.04467200115323067, + 0.04368000105023384, + 0.04822399839758873, + 0.043296001851558685, + 0.043296001851558685 + ], + [ + 0.0461760014295578, + 0.047168001532554626, + 0.045504000037908554, + 0.044576000422239304, + 0.04374400153756142, + 0.0435199998319149, + 0.043776001781225204, + 0.04355200007557869, + 0.043455999344587326, + 0.04355200007557869 + ], + [ + 0.048608001321554184, + 0.047520000487565994, + 0.0480320006608963, + 0.04726399853825569, + 0.0459199994802475, + 0.04630399867892265, + 0.04588799923658371, + 0.04608000069856644, + 0.04524800181388855, + 0.04572800174355507 + ], + [ + 0.046720001846551895, + 0.05827200040221214, + 0.04604800045490265, + 0.04473600164055824, + 0.043487999588251114, + 0.04447999969124794, + 0.04390399903059006, + 0.044064000248909, + 0.04335999861359596, + 0.04390399903059006 + ], + [ + 0.04927999898791313, + 0.04636799916625023, + 0.044704001396894455, + 0.04438399896025658, + 0.04556800052523613, + 0.043616000562906265, + 0.04368000105023384, + 0.04310400038957596, + 0.04249599948525429, + 0.04275200143456459 + ], + [ + 0.045024000108242035, + 0.04435199871659279, + 0.04403200000524521, + 0.04339199885725975, + 0.04230400174856186, + 0.042367998510599136, + 0.04275200143456459, + 0.04278400167822838, + 0.04310400038957596, + 0.04281599819660187 + ], + [ + 0.04681599885225296, + 0.04726399853825569, + 0.047200001776218414, + 0.04620800167322159, + 0.045951999723911285, + 0.04569600149989128, + 0.04473600164055824, + 0.04566400125622749, + 0.04473600164055824, + 0.045471999794244766 + ], + [ + 0.052032001316547394, + 0.05023999884724617, + 0.048576001077890396, + 0.04838399961590767, + 0.04835199937224388, + 0.047359999269247055, + 0.045791998505592346, + 0.047520000487565994, + 0.046560000628232956, + 0.046751998364925385 + ], + [ + 0.0488319993019104, + 0.04864000156521797, + 0.04806400090456009, + 0.04652800038456917, + 0.04614400118589401, + 0.047231998294591904, + 0.04668800160288811, + 0.047520000487565994, + 0.046592000871896744, + 0.046431999653577805 + ] + ], + "marker_exact": true + }, + { + "rows": 4096, + "critical_median_ms": 0.07840000092983246, + "ns_per_query": 19.140625227009878, + "samples_ms": [ + [ + 0.07708799839019775, + 0.07785599678754807, + 0.07744000107049942, + 0.07568000257015228, + 0.07692799717187881, + 0.07631999999284744, + 0.07599999755620956, + 0.07487999647855759, + 0.07599999755620956, + 0.07561600208282471 + ], + [ + 0.07827199995517731, + 0.07753600180149078, + 0.07715199887752533, + 0.07635200023651123, + 0.07795199751853943, + 0.07503999769687653, + 0.07667200267314911, + 0.07628799974918365, + 0.0772479996085167, + 0.07475200295448303 + ], + [ + 0.07763200253248215, + 0.07756800204515457, + 0.07676800340414047, + 0.07718399912118912, + 0.0764480009675026, + 0.07462400197982788, + 0.07756800204515457, + 0.07507199794054031, + 0.07577600330114365, + 0.0783040001988411 + ], + [ + 0.07705599814653397, + 0.07638400048017502, + 0.0761599987745285, + 0.07587199658155441, + 0.07625599950551987, + 0.07558400183916092, + 0.08560000360012054, + 0.07823999971151352, + 0.07606399804353714, + 0.07568000257015228 + ], + [ + 0.07744000107049942, + 0.0759039968252182, + 0.07744000107049942, + 0.07580800354480743, + 0.07577600330114365, + 0.07587199658155441, + 0.07689599692821503, + 0.07612799853086472, + 0.07648000121116638, + 0.07568000257015228 + ], + [ + 0.0783040001988411, + 0.07827199995517731, + 0.0777600035071373, + 0.07657600194215775, + 0.07651200145483017, + 0.07692799717187881, + 0.07747200131416321, + 0.0761599987745285, + 0.07683199644088745, + 0.07651200145483017 + ], + [ + 0.078015998005867, + 0.07558400183916092, + 0.07727999985218048, + 0.07648000121116638, + 0.0764480009675026, + 0.07676800340414047, + 0.07654400169849396, + 0.07503999769687653, + 0.07631999999284744, + 0.07583999633789062 + ], + [ + 0.07817599922418594, + 0.07705599814653397, + 0.07731200009584427, + 0.07638400048017502, + 0.07753600180149078, + 0.07612799853086472, + 0.07740800082683563, + 0.07532799988985062, + 0.07638400048017502, + 0.07580800354480743 + ], + [ + 0.07648000121116638, + 0.07555200159549713, + 0.07622399926185608, + 0.07526399940252304, + 0.07571200281381607, + 0.07446400076150894, + 0.07676800340414047, + 0.0745600014925003, + 0.07558400183916092, + 0.07446400076150894 + ], + [ + 0.07887999713420868, + 0.07868800312280655, + 0.07820799946784973, + 0.07571200281381607, + 0.077504001557827, + 0.07660800218582153, + 0.07740800082683563, + 0.07651200145483017, + 0.07903999835252762, + 0.07625599950551987 + ], + [ + 0.0769599974155426, + 0.07635200023651123, + 0.07849600166082382, + 0.0753600001335144, + 0.07622399926185608, + 0.07568000257015228, + 0.07625599950551987, + 0.07491199672222137, + 0.0772159993648529, + 0.07609599828720093 + ], + [ + 0.07740800082683563, + 0.07609599828720093, + 0.07680000364780426, + 0.07491199672222137, + 0.07753600180149078, + 0.07599999755620956, + 0.0772479996085167, + 0.07689599692821503, + 0.07747200131416321, + 0.07548800110816956 + ], + [ + 0.07760000228881836, + 0.07667200267314911, + 0.0761599987745285, + 0.07622399926185608, + 0.07715199887752533, + 0.07552000135183334, + 0.0769599974155426, + 0.07603199779987335, + 0.07628799974918365, + 0.07491199672222137 + ], + [ + 0.07868800312280655, + 0.07683199644088745, + 0.07734400033950806, + 0.07760000228881836, + 0.07804799824953079, + 0.07744000107049942, + 0.07779199630022049, + 0.07606399804353714, + 0.07811199873685837, + 0.07689599692821503 + ], + [ + 0.07788799703121185, + 0.07583999633789062, + 0.07788799703121185, + 0.07513599842786789, + 0.07596799731254578, + 0.07603199779987335, + 0.0769599974155426, + 0.07568000257015228, + 0.07542400062084198, + 0.07513599842786789 + ], + [ + 0.07872000336647034, + 0.07734400033950806, + 0.07689599692821503, + 0.07756800204515457, + 0.07791999727487564, + 0.07654400169849396, + 0.07779199630022049, + 0.07593599706888199, + 0.07756800204515457, + 0.077504001557827 + ] + ], + "marker_exact": true + }, + { + "rows": 32768, + "critical_median_ms": 0.495168000459671, + "ns_per_query": 15.111328139028046, + "samples_ms": [ + [ + 0.4846400022506714, + 0.48422399163246155, + 0.4835520088672638, + 0.4837760031223297, + 0.4890559911727905, + 0.4880639910697937, + 0.48630398511886597, + 0.4833280146121979, + 0.4864319860935211, + 0.4830079972743988 + ], + [ + 0.4832319915294647, + 0.48476800322532654, + 0.4800640046596527, + 0.48451200127601624, + 0.4832319915294647, + 0.48390400409698486, + 0.5678719878196716, + 0.49299201369285583, + 0.48403200507164, + 0.4846400022506714 + ], + [ + 0.482015997171402, + 0.4830720126628876, + 0.48156800866127014, + 0.48368000984191895, + 0.4830079972743988, + 0.481471985578537, + 0.48044800758361816, + 0.48524799942970276, + 0.4825280010700226, + 0.48230400681495667 + ], + [ + 0.48281601071357727, + 0.4836159944534302, + 0.4824639856815338, + 0.4837439954280853, + 0.482015997171402, + 0.48240000009536743, + 0.48211199045181274, + 0.4838080108165741, + 0.4822719991207123, + 0.4832639992237091 + ], + [ + 0.48342400789260864, + 0.48368000984191895, + 0.48368000984191895, + 0.4848960041999817, + 0.4822719991207123, + 0.4830400049686432, + 0.659168004989624, + 0.4891839921474457, + 0.4853439927101135, + 0.4845759868621826 + ], + [ + 0.4935680031776428, + 0.4917759895324707, + 0.4864000082015991, + 0.4891520142555237, + 0.48416000604629517, + 0.4912639856338501, + 0.4907520115375519, + 0.4926080107688904, + 0.4842880070209503, + 0.48710399866104126 + ], + [ + 0.48550400137901306, + 0.4867840111255646, + 0.48364800214767456, + 0.48230400681495667, + 0.4846400022506714, + 0.48447999358177185, + 0.4832639992237091, + 0.48524799942970276, + 0.4850879907608032, + 0.4859839975833893 + ], + [ + 0.4856320023536682, + 0.48576000332832336, + 0.483487993478775, + 0.48399999737739563, + 0.48284798860549927, + 0.48473599553108215, + 0.4832639992237091, + 0.48416000604629517, + 0.4830720126628876, + 0.48576000332832336 + ], + [ + 0.48124799132347107, + 0.4818879961967468, + 0.4793280065059662, + 0.48153600096702576, + 0.48051199316978455, + 0.4798080027103424, + 0.4789760112762451, + 0.4806399941444397, + 0.48051199316978455, + 0.48150399327278137 + ], + [ + 0.4848000109195709, + 0.48655998706817627, + 0.4845759868621826, + 0.48390400409698486, + 0.4891839921474457, + 0.4854080080986023, + 0.483487993478775, + 0.4849280118942261, + 0.4829440116882324, + 0.4832000136375427 + ], + [ + 0.48240000009536743, + 0.48524799942970276, + 0.4813440144062042, + 0.48604801297187805, + 0.48550400137901306, + 0.4848960041999817, + 0.4816319942474365, + 0.4835520088672638, + 0.48028799891471863, + 0.48287999629974365 + ], + [ + 0.5635200142860413, + 0.488319993019104, + 0.48419201374053955, + 0.4840959906578064, + 0.48099198937416077, + 0.4856320023536682, + 0.48393601179122925, + 0.4859839975833893, + 0.4830079972743988, + 0.4822080135345459 + ], + [ + 0.48470398783683777, + 0.485152006149292, + 0.4819520115852356, + 0.4846400022506714, + 0.48284798860549927, + 0.48265600204467773, + 0.48451200127601624, + 0.48364800214767456, + 0.4830720126628876, + 0.4837439954280853 + ], + [ + 0.4851840138435364, + 0.4853760004043579, + 0.4843519926071167, + 0.4835520088672638, + 0.4819520115852356, + 0.4843840003013611, + 0.4817599952220917, + 0.49718400835990906, + 0.4858880043029785, + 0.48393601179122925 + ], + [ + 0.4880000054836273, + 0.4862079918384552, + 0.48396798968315125, + 0.48185598850250244, + 0.4809280037879944, + 0.4840959906578064, + 0.4854719936847687, + 0.4825280010700226, + 0.4824639856815338, + 0.4829440116882324 + ], + [ + 0.4848319888114929, + 0.48502400517463684, + 0.484607994556427, + 0.4850560128688812, + 0.481471985578537, + 0.48585599660873413, + 0.4817599952220917, + 0.5250239968299866, + 0.512224018573761, + 0.49856001138687134 + ] + ], + "marker_exact": true + }, + { + "rows": 65536, + "critical_median_ms": 0.9471040070056915, + "ns_per_query": 14.45166026314837, + "samples_ms": [ + [ + 0.9458879828453064, + 0.9456959962844849, + 0.9457280039787292, + 0.9461119771003723, + 0.9447680115699768, + 0.9431679844856262, + 0.9455680251121521, + 0.9448959827423096, + 0.9446079730987549, + 0.9431679844856262 + ], + [ + 0.9470400214195251, + 0.9518719911575317, + 0.9548479914665222, + 0.9454399943351746, + 0.9466879963874817, + 0.9461759924888611, + 0.9453759789466858, + 0.9455040097236633, + 0.9439039826393127, + 0.9450560212135315 + ], + [ + 0.9453759789466858, + 0.9442880153656006, + 0.9488000273704529, + 0.9441919922828674, + 0.9446719884872437, + 0.9445440173149109, + 0.9446719884872437, + 0.9427520036697388, + 0.9429119825363159, + 0.9422720074653625 + ], + [ + 0.9474560022354126, + 0.9570239782333374, + 0.948095977306366, + 0.9434239864349365, + 0.9453439712524414, + 0.9433599710464478, + 0.9449920058250427, + 0.9431359767913818, + 0.9460800290107727, + 0.9411839842796326 + ], + [ + 0.9461759924888611, + 0.9458240270614624, + 0.9471359848976135, + 0.9443519711494446, + 0.9463359713554382, + 0.9470080137252808, + 0.9451839923858643, + 0.9464960098266602, + 0.9453439712524414, + 0.9448639750480652 + ], + [ + 0.9487360119819641, + 0.9451199769973755, + 0.9447360038757324, + 0.9431040287017822, + 0.945792019367218, + 0.9423360228538513, + 0.9433919787406921, + 0.9450240135192871, + 0.9434239864349365, + 0.9451839923858643 + ], + [ + 0.9472640156745911, + 0.9432960152626038, + 0.9454399943351746, + 0.94377601146698, + 0.9430080056190491, + 0.9435520172119141, + 0.9461759924888611, + 0.9435840249061584, + 0.9455680251121521, + 0.944320023059845 + ], + [ + 0.9450240135192871, + 0.9433919787406921, + 0.9450240135192871, + 0.9413440227508545, + 0.9461439847946167, + 0.9453120231628418, + 0.9457600116729736, + 0.9438080191612244, + 0.9462720155715942, + 0.9450240135192871 + ], + [ + 0.9440640211105347, + 0.9446719884872437, + 0.9467840194702148, + 0.9463679790496826, + 0.9463679790496826, + 0.9440640211105347, + 0.9461119771003723, + 0.941536009311676, + 0.9463040232658386, + 0.9431040287017822 + ], + [ + 0.9471679925918579, + 0.9464640021324158, + 0.9434880018234253, + 0.9440000057220459, + 0.9467200040817261, + 0.9445120096206665, + 0.9435200095176697, + 0.9442880153656006, + 0.9439359903335571, + 0.9440640211105347 + ], + [ + 0.9433919787406921, + 0.943231999874115, + 0.9468160271644592, + 0.9446079730987549, + 0.9435520172119141, + 0.9436159729957581, + 0.9437119960784912, + 0.9442560076713562, + 0.9421759843826294, + 0.9425600171089172 + ], + [ + 0.9450560212135315, + 0.9459840059280396, + 0.9427840113639832, + 0.9410880208015442, + 0.9449920058250427, + 0.9431040287017822, + 0.9437440037727356, + 0.9419519901275635, + 0.9431679844856262, + 0.9477120041847229 + ], + [ + 0.9472320079803467, + 0.9468799829483032, + 0.9467520117759705, + 0.944383978843689, + 0.9447360038757324, + 0.9464960098266602, + 0.9450880289077759, + 0.9472000002861023, + 0.9467520117759705, + 0.9451519846916199 + ], + [ + 0.9446719884872437, + 0.9464640021324158, + 0.944927990436554, + 0.9429759979248047, + 0.9435840249061584, + 0.943231999874115, + 0.9460480213165283, + 0.944096028804779, + 0.9459519982337952, + 0.9434239864349365 + ], + [ + 0.9466239809989929, + 0.9458240270614624, + 0.944927990436554, + 0.9418560266494751, + 0.9429759979248047, + 0.9424960017204285, + 0.9442560076713562, + 0.9439039826393127, + 0.9446399807929993, + 0.9423999786376953 + ], + [ + 0.946943998336792, + 0.9460480213165283, + 0.9494720101356506, + 0.9453120231628418, + 0.9448320269584656, + 0.9444159865379333, + 0.9444800019264221, + 0.944159984588623, + 0.9457600116729736, + 0.9445120096206665 + ] + ], + "marker_exact": true + } + ] + }, + "gain_policy": "Retain declared 5% / 40 ms single-use migration headroom; no heldout E2E tuning.", + "transport_calibration": { + "method": "Nonnegative max-resource fit; independent eager transport timings", + "coefficients_ms_per_65536_rows": [ + 31.15628543240907, + 65.28744062274444, + 110.39632970746038, + 4.357277541747443, + 0.9471040070056915, + 0.025098063419906638, + 0.003930108470061562 + ], + "predictions": [ + { + "kind": "intra_pair", + "rows": 256, + "heldout": false, + "actual_ms": 0.3941280096769333, + "predicted_ms": 0.4602638348097353, + "relative_error": 0.16780290542408682 + }, + { + "kind": "intra", + "rows": 256, + "heldout": false, + "actual_ms": 0.3874559998512268, + "predicted_ms": 0.4602638348097353, + "relative_error": 0.1879125242258859 + }, + { + "kind": "intra_dense", + "rows": 256, + "heldout": false, + "actual_ms": 0.2534399926662445, + "predicted_ms": 0.28610776238346247, + "relative_error": 0.12889745368734373 + }, + { + "kind": "inter_pair", + "rows": 256, + "heldout": false, + "actual_ms": 0.6665279865264893, + "predicted_ms": 0.4602638348097353, + "relative_error": -0.309460601634552 + }, + { + "kind": "inter", + "rows": 256, + "heldout": false, + "actual_ms": 0.7499519884586334, + "predicted_ms": 0.4602638348097353, + "relative_error": -0.38627559911440523 + }, + { + "kind": "inter_dense", + "rows": 256, + "heldout": false, + "actual_ms": 0.6300639808177948, + "predicted_ms": 0.4597436807619102, + "relative_error": -0.2703222295532852 + }, + { + "kind": "striped", + "rows": 256, + "heldout": true, + "actual_ms": 0.44177600741386414, + "predicted_ms": 0.507915592234959, + "relative_error": 0.1497129398408774 + }, + { + "kind": "intra_pair", + "rows": 4096, + "heldout": false, + "actual_ms": 6.484655857086182, + "predicted_ms": 6.928798778606242, + "relative_error": 0.06849136350616325 + }, + { + "kind": "intra", + "rows": 4096, + "heldout": false, + "actual_ms": 6.525344133377075, + "predicted_ms": 6.928798778606242, + "relative_error": 0.061828868636291556 + }, + { + "kind": "intra_dense", + "rows": 4096, + "heldout": false, + "actual_ms": 1.9262880086898804, + "predicted_ms": 1.883475911994279, + "relative_error": -0.022225179465618417 + }, + { + "kind": "inter_pair", + "rows": 4096, + "heldout": false, + "actual_ms": 7.343456029891968, + "predicted_ms": 6.928798778606242, + "relative_error": -0.056466226473997905 + }, + { + "kind": "inter", + "rows": 4096, + "heldout": false, + "actual_ms": 7.85427188873291, + "predicted_ms": 6.928798778606242, + "relative_error": -0.11783054154953243 + }, + { + "kind": "inter_dense", + "rows": 4096, + "heldout": false, + "actual_ms": 4.239135980606079, + "predicted_ms": 4.285179654750842, + "relative_error": 0.010861570460445558 + }, + { + "kind": "striped", + "rows": 4096, + "heldout": true, + "actual_ms": 2.2811520099639893, + "predicted_ms": 2.420633579229425, + "relative_error": 0.06114523217049328 + }, + { + "kind": "intra_pair", + "rows": 32768, + "heldout": true, + "actual_ms": 50.98955154418945, + "predicted_ms": 55.22719302562015, + "relative_error": 0.08310803592297145 + }, + { + "kind": "intra", + "rows": 32768, + "heldout": true, + "actual_ms": 51.211055755615234, + "predicted_ms": 55.22719302562015, + "relative_error": 0.0784232469092292 + }, + { + "kind": "intra_dense", + "rows": 32768, + "heldout": true, + "actual_ms": 14.252928256988525, + "predicted_ms": 13.810491429088376, + "relative_error": -0.031041819612276034 + }, + { + "kind": "inter_pair", + "rows": 32768, + "heldout": true, + "actual_ms": 56.860286712646484, + "predicted_ms": 55.22719302562015, + "relative_error": -0.02872116518299428 + }, + { + "kind": "inter", + "rows": 32768, + "heldout": true, + "actual_ms": 61.266624450683594, + "predicted_ms": 55.22719302562015, + "relative_error": -0.09857620652701149 + }, + { + "kind": "inter_dense", + "rows": 32768, + "heldout": true, + "actual_ms": 30.82592010498047, + "predicted_ms": 32.84843492720153, + "relative_error": 0.06561085006816358 + }, + { + "kind": "striped", + "rows": 32768, + "heldout": true, + "actual_ms": 15.788095951080322, + "predicted_ms": 16.702261215454772, + "relative_error": 0.05790218574849093 + }, + { + "kind": "intra_pair", + "rows": 65536, + "heldout": false, + "actual_ms": 102.41177368164062, + "predicted_ms": 110.42535787935034, + "relative_error": 0.07824866135627051 + }, + { + "kind": "intra", + "rows": 65536, + "heldout": false, + "actual_ms": 102.45684814453125, + "predicted_ms": 110.42535787935034, + "relative_error": 0.07777430088009618 + }, + { + "kind": "intra_dense", + "rows": 65536, + "heldout": false, + "actual_ms": 28.466943740844727, + "predicted_ms": 27.441366305767346, + "relative_error": -0.03602695970505143 + }, + { + "kind": "inter_pair", + "rows": 65536, + "heldout": false, + "actual_ms": 114.2823371887207, + "predicted_ms": 110.42535787935034, + "relative_error": -0.03374956624312919 + }, + { + "kind": "inter", + "rows": 65536, + "heldout": false, + "actual_ms": 122.30096054077148, + "predicted_ms": 110.42535787935034, + "relative_error": -0.09710146681523546 + }, + { + "kind": "inter_dense", + "rows": 65536, + "heldout": false, + "actual_ms": 60.029855728149414, + "predicted_ms": 65.49215523857376, + "relative_error": 0.09099304744560532 + }, + { + "kind": "striped", + "rows": 65536, + "heldout": true, + "actual_ms": 31.16423988342285, + "predicted_ms": 33.02412137114088, + "relative_error": 0.05967998881652026 + } + ], + "ranks_per_node": 8, + "world_size": 16, + "heldout_rule": "32768 rows or striped layout; never used in fitting", + "note": "Effective coefficients include eager implementation costs; not physical bandwidth or universal optimality guarantees.", + "source_sha256": "be7130a522549221bc2c1d2b538da8101ef5f7c3f640d6f5bbb83cb573bf0de5", + "node_limit_note": "Removed node envelope mathematically dominated by per-rank inter-node limits for this topology; it is not independently identified by these fixtures." + }, + "transport_validation": { + "source_sha256": "ef98c8708210f495df09c0c78d7e2d099cfb9713213b671abde4da5ad9da50e9", + "records": [ + { + "kind": "mixed_dense_quarter", + "rows": 8192, + "actual_ms": 2.960976004600525, + "predicted_ms": 3.054578100021679, + "relative_error": 0.031611906099787035 + }, + { + "kind": "mixed_dense_three_quarters", + "rows": 8192, + "actual_ms": 6.393343925476074, + "predicted_ms": 6.619486619026664, + "relative_error": 0.03537158272519347 + }, + { + "kind": "mixed_dense_quarter", + "rows": 49152, + "actual_ms": 16.762415885925293, + "predicted_ms": 16.425463301286772, + "relative_error": -0.020101671914813046 + }, + { + "kind": "mixed_dense_three_quarters", + "rows": 49152, + "actual_ms": 36.45444869995117, + "predicted_ms": 37.81491441531668, + "relative_error": 0.03731960745211671 + } + ] + }, + "cudnn_source": "855ccd77ba9efed22441fd454f7cc0206a068e94" +} diff --git a/benchmarks/summarize_csa_layout_matrix.py b/benchmarks/summarize_csa_layout_matrix.py new file mode 100644 index 00000000..675ad5d1 --- /dev/null +++ b/benchmarks/summarize_csa_layout_matrix.py @@ -0,0 +1,132 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Summarize complete paired full-CSA matrices, retaining worst-rank errors.""" + +import argparse +import json +import math +import statistics +from pathlib import Path + + +def summarize(directory): + grouped = {} + for path in sorted(directory.glob("layout-1m-*.json")): + raw = json.loads(path.read_text()) + layout = raw["config"]["layout"] + name = path.stem.removeprefix("layout-").removesuffix("-" + layout) + ranks = raw["ranks"] + if sorted(r["rank"] for r in ranks) != list(range(raw["world_size"])): + raise ValueError(f"incomplete rank results: {path}") + if any(not all(r[k] for k in ("finite", "input_match", "redistribution_bitwise")) for r in ranks): + raise ValueError(f"failed correctness check: {path}") + repeats = raw["config"]["repeats"] + if not repeats or any(len(r["samples"]) != repeats for r in ranks): + raise ValueError(f"incomplete timing samples: {path}") + if any( + not math.isfinite(s[k]) or s[k] < 0 + for r in ranks + for s in r["samples"] + for k in ("total_ms", "peak_allocated_gib", "peak_reserved_gib") + ) or any( + not math.isfinite(r[k]) or r[k] < 0 + for r in ranks + for k in ("layout_prepare_ms", "common_prepare_ms", "kda_prepare_ms", "total_prepare_ms") + ): + raise ValueError(f"invalid timing/memory samples: {path}") + if any( + not math.isclose(r["total_prepare_ms"], r["layout_prepare_ms"] + r["common_prepare_ms"] + r["kda_prepare_ms"]) + for r in ranks + ): + raise ValueError(f"inconsistent preparation accounting: {path}") + errors = {} + for rank in ranks: + if layout != "contiguous": + required = {"output", "dq", "dkv", "d_index_q", "d_weights", "d_pool_kv", "d_pool_score", "d_ape"} + if raw["config"].get("with_kda"): + required |= {"kda_output", "kda_dq", "kda_dk", "kda_dv", "kda_dg", "kda_dbeta"} + if not required.issubset(rank.get("layout_vs_contiguous") or {}): + raise ValueError(f"missing output/gradient baseline comparison: {path}") + delta = rank.get("loss_abs_diff") + if delta is None or not math.isfinite(delta) or not 0 <= delta < 1e-5: + raise ValueError(f"failed loss comparison: {path}") + for tensor, error in (rank["layout_vs_contiguous"] or {}).items(): + if not error["finite"] or not 0 <= error["relative_rms"] < 0.03: + raise ValueError(f"failed precision comparison: {path}: {tensor}") + errors[tensor] = max(errors.get(tensor, 0.0), error["relative_rms"]) + critical = [max(r["samples"][i]["total_ms"] for r in ranks) for i in range(repeats)] + row = dict( + source=str(path), + total_ms=statistics.median(critical), + min_ms=min(critical), + max_ms=max(critical), + allocated_gib=max(s["peak_allocated_gib"] for r in ranks for s in r["samples"]), + reserved_gib=max(s["peak_reserved_gib"] for r in ranks for s in r["samples"]), + prepare_ms=max(r["total_prepare_ms"] for r in ranks), + layout_prepare_ms=max(r["layout_prepare_ms"] for r in ranks), + policy=ranks[0]["selected_policy"], + relative_rms=errors, + loss_abs_diff=max((r["loss_abs_diff"] or 0.0) for r in ranks), + workspace_gib=max(r["selection_workspace_bytes"] for r in ranks) / 2**30, + max_spans=max(r["selection_spans"] for r in ranks), + ) + contract = { + key: raw["config"].get(key) + for key in ( + "heads", + "top_k", + "with_kda", + "kda_batch_invariant", + "kda_gate_scale", + "candidate_mib", + "redistribution_scope", + "pool_backward_order", + "repeats", + "warmups", + ) + } + group = grouped.setdefault( + name, dict(lengths=raw["config"]["sequence_lengths"], world_size=raw["world_size"], contract=contract, layouts={}) + ) + if ( + group["lengths"] != raw["config"]["sequence_lengths"] + or group["world_size"] != raw["world_size"] + or group["contract"] != contract + ): + raise ValueError(f"unpaired shape/group: {name}") + group["layouts"][layout] = row + if not grouped: + raise ValueError("no 1M results found") + for name, case in grouped.items(): + if set(case["layouts"]) != {"contiguous", "zigzag", "planner"}: + raise ValueError(f"incomplete comparison: {name}") + c, z, p = (case["layouts"][k] for k in ("contiguous", "zigzag", "planner")) + case["speedup_vs_contiguous"] = c["total_ms"] / p["total_ms"] + case["gap_to_best_baseline_pct"] = 100 * (p["total_ms"] / min(c["total_ms"], z["total_ms"]) - 1) + case["prepare_plus_iteration_ms"] = {k: v["prepare_ms"] + v["total_ms"] for k, v in case["layouts"].items()} + first = case["prepare_plus_iteration_ms"] + case["prepare_inclusive_gap_pct"] = 100 * (first["planner"] / min(first["contiguous"], first["zigzag"]) - 1) + return grouped + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + results = summarize(args.directory) + if args.output: + args.output.write_text(json.dumps(results, indent=2) + "\n") + print("| Case | Docs | Contiguous ms | Zigzag ms | Planner ms | Speedup | Planner allocated GiB | Prepare ms | Policy |") + print("|---|---:|---:|---:|---:|---:|---:|---:|---|") + for name, case in results.items(): + c, z, p = (case["layouts"][k] for k in ("contiguous", "zigzag", "planner")) + print( + f"| {name} | {len(case['lengths'])} | {c['total_ms']:.2f} | {z['total_ms']:.2f} | {p['total_ms']:.2f} | " + f"{case['speedup_vs_contiguous']:.3f}x | {p['allocated_gib']:.2f} | {p['prepare_ms']:.2f} | {p['policy']} |" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/thd_1m_cases.py b/benchmarks/thd_1m_cases.py new file mode 100644 index 00000000..72a806d2 --- /dev/null +++ b/benchmarks/thd_1m_cases.py @@ -0,0 +1,259 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Reproducible 1M THD cases inspired by a character-length count histogram. + +The screenshot supplies no exact percentages or tokenizer. Weights below are +explicit synthetic priors, not recovered measurements. Long-document spikes +are additional stress cases, not claims about the screenshot's hidden tail. +""" + +import argparse +import hashlib +import json +import math +import random +from dataclasses import asdict, dataclass, replace +from itertools import accumulate +from pathlib import Path + +TOTAL_TOKENS = 1 << 20 +DEFAULT_SEED = 20261112 +# (lower exclusive, upper inclusive); k=1024 is a synthetic convention. +CHARACTER_BINS = ((10, 100), (100, 1024), (1024, 2048), (2048, 4096), (4096, 8192), (8192, 16384)) +COUNT_PRIORS = { + "tiny90": (9000, 500, 200, 200, 80, 20), + "mix80": (8000, 700, 500, 500, 250, 50), + "mix70": (7000, 1200, 800, 600, 300, 100), +} + + +@dataclass(frozen=True) +class THDCase: + name: str + lengths: tuple[int, ...] + prior: str + character_divisor: int + long_documents: tuple[int, ...] + order: str + seed: int + final_document_truncation: int = 0 + + +def _fill(budget, prior, divisor, seed): + """Draw aligned lengths and truncate at most the last draw to fill exactly.""" + rng = random.Random(seed) + cumulative = tuple(accumulate(COUNT_PRIORS[prior])) + lengths, remaining, truncation = [], budget, 0 + while remaining: + draw = rng.randrange(cumulative[-1]) + bucket = next(i for i, bound in enumerate(cumulative) if draw < bound) + lower, upper = CHARACTER_BINS[bucket] + # Uniform aligned token proxies within each scaled character bin. + minimum = max(1, lower // (4 * divisor) + 1) + maximum = max(minimum, upper // (4 * divisor)) + proposed = 4 * rng.randint(minimum, maximum) + length = min(proposed, remaining) + lengths.append(length) + remaining -= length + truncation += proposed - length + return lengths, truncation + + +def _ordered(lengths, order, seed): + values = sorted(lengths) + if order == "ascending": + return tuple(values) + if order == "descending": + return tuple(reversed(values)) + if order == "alternating": + # Smallest, largest, second-smallest, second-largest, ... + return tuple( + v + for i in range((len(values) + 1) // 2) + for v in ((values[i], values[-1 - i]) if i != len(values) - 1 - i else (values[i],)) + ) + if order != "shuffled": + raise ValueError(f"unknown order: {order}") + random.Random(seed + 1).shuffle(values) + return tuple(values) + + +def statistics_cases(seed=DEFAULT_SEED): + """Return 18 block-aligned cases; same-histogram order variants share data.""" + cases = [THDCase("1m-stat-tiny64", (64,) * (TOTAL_TOKENS // 64), "fixed64", 1, (), "constant", seed)] + specifications = [ + ("tiny90", "tiny90", 1, ()), + ("mix80", "mix80", 1, ()), + ("mix70", "mix70", 1, ()), + ("mix80-div2", "mix80", 2, ()), + ("mix80-div4", "mix80", 4, ()), + ("mix80-1x128k", "mix80", 1, (128 << 10,)), + ("mix80-1x256k", "mix80", 1, (256 << 10,)), + ("mix80-1x512k", "mix80", 1, (512 << 10,)), + ("mix80-2x256k", "mix80", 1, (256 << 10,) * 2), + ("mix80-4x128k", "mix80", 1, (128 << 10,) * 4), + ("mix80-8x64k", "mix80", 1, (64 << 10,) * 8), + ] + for name, prior, divisor, long_docs in specifications: + small, truncated = _fill(TOTAL_TOKENS - sum(long_docs), prior, divisor, seed) + lengths = _ordered([*small, *long_docs], "shuffled", seed) + cases.append(THDCase(f"1m-stat-{name}", lengths, prior, divisor, long_docs, "shuffled", seed, truncated)) + for name in ("1m-stat-mix80", "1m-stat-mix80-2x256k"): + original = next(case for case in cases if case.name == name) + for order in ("ascending", "descending", "alternating"): + cases.append( + replace(original, name=f"{name}-{order}", lengths=_ordered(original.lengths, order, seed), order=order) + ) + return tuple(cases) + + +def counted_cases(seed=DEFAULT_SEED): + """1M with 256--1024 documents, controlling count independently of length.""" + + def balanced(tokens, count): + blocks, extra = divmod(tokens // 4, count) + if tokens % 4 or blocks < 1: + raise ValueError("invalid aligned token budget") + return [4 * (blocks + (i < extra)) for i in range(count)] + + tiny512 = [64] * 410 + tiny1000 = [64] * 800 + specs = [ + ("uniform-256", [4096] * 256, ()), + ("uniform-1024", [1024] * 1024, ()), + ("near-uniform-512", [1536, 2560] * 256, ()), + ("tiny-mix-512", tiny512 + [8192] * 90 + balanced(TOTAL_TOKENS - sum(tiny512) - 8192 * 90, 12), ()), + ("tiny-mix-1000", tiny1000 + [2048] * 190 + balanced(TOTAL_TOKENS - sum(tiny1000) - 2048 * 190, 10), ()), + ("mixed-256-1long", [64] * 205 + [8192] * 50, (TOTAL_TOKENS - 64 * 205 - 8192 * 50,)), + ("mixed-512-2long", tiny512 + balanced(TOTAL_TOKENS // 2 - sum(tiny512), 100), (256 << 10,) * 2), + ("mixed-1000-1long", tiny1000 + balanced(TOTAL_TOKENS // 2 - sum(tiny1000), 199), (512 << 10,)), + ("mixed-512-4long", tiny512 + balanced(TOTAL_TOKENS // 2 - sum(tiny512), 98), (128 << 10,) * 4), + ] + cases = [ + THDCase(f"1m-count-{name}", _ordered([*short, *long], "shuffled", seed), "fixed-count", 1, long, "shuffled", seed) + for name, short, long in specs + ] + base = next(c for c in cases if c.name == "1m-count-mixed-512-2long") + for order in ("ascending", "descending"): + cases.append(replace(base, name=f"{base.name}-{order}", lengths=_ordered(base.lengths, order, seed), order=order)) + return tuple(cases) + + +def heldout_cases(seed=20261114): + """Uncalibrated, variable-length cases with 257--1000 documents. + + Both lengths and order change with the seed. Budget apportionment uses + largest remainders after reserving one block per sample, so it cannot + introduce empty documents or hide padding tokens in a final correction. + """ + rng = random.Random(seed) + + def allocate(tokens, weights): + blocks = tokens // 4 - len(weights) + total_weight = sum(weights) + quotas = [blocks * weight / total_weight for weight in weights] + values = [math.floor(quota) for quota in quotas] + for i in sorted(range(len(weights)), key=lambda i: (-(quotas[i] - values[i]), i))[: blocks - sum(values)]: + values[i] += 1 + return [4 * (value + 1) for value in values] + + specs = [] + for count, sigma in ((257, 1.2), (1000, 1.8)): + specs.append((f"lognormal-{count}", allocate(TOTAL_TOKENS, [rng.lognormvariate(0, sigma) for _ in range(count)]), ())) + specs.append(("jittered-768", allocate(TOTAL_TOKENS, [rng.uniform(0.7, 1.3) for _ in range(768)]), ())) + tiny = [4 * rng.randint(3, 25) for _ in range(614)] + specs.append( + ("tiny-tail-768", tiny + allocate(TOTAL_TOKENS - sum(tiny), [rng.lognormvariate(0, 2) for _ in range(154)]), ()) + ) + short = [4 * rng.randint(3, 25) for _ in range(410)] + [4 * rng.randint(128, 1024) for _ in range(101)] + fraction = rng.uniform(0.2, 0.8) + long = tuple(allocate(TOTAL_TOKENS - sum(short), [fraction, 1 - fraction])) + specs.append(("trimodal-513", short + list(long), long)) + return tuple( + THDCase(f"1m-heldout-{name}", _ordered(lengths, "shuffled", seed), "heldout-weighted", 1, long, "shuffled", seed) + for name, lengths, long in specs + ) + + +def summarize(case): + lengths = case.lengths + boundaries = (0, 10, 100, 1024, 2048, 4096, 8192, 16384, 65536, 131072, 262144, 524288, TOTAL_TOKENS) + histogram = [] + for lower, upper in zip(boundaries, boundaries[1:]): + values = [n for n in lengths if lower < n <= upper] + histogram.append( + dict( + lower_exclusive=lower, + upper_inclusive=upper, + documents=len(values), + tokens=sum(values), + document_fraction=len(values) / len(lengths), + token_fraction=sum(values) / TOTAL_TOKENS, + ) + ) + return dict( + documents=len(lengths), + total_tokens=sum(lengths), + min_tokens=min(lengths), + max_tokens=max(lengths), + tiny_document_fraction=sum(n <= 100 for n in lengths) / len(lengths), + tiny_token_fraction=sum(n for n in lengths if n <= 100) / TOTAL_TOKENS, + explicit_long_token_fraction=sum(case.long_documents) / TOTAL_TOKENS, + lengths_sha256=hashlib.sha256(json.dumps(lengths, separators=(",", ":")).encode()).hexdigest(), + token_length_histogram=histogram, + ) + + +def manifest(seed=DEFAULT_SEED, suite="statistics"): + if suite not in ("statistics", "counted", "heldout", "all"): + raise ValueError(f"unknown THD suite: {suite}") + cases = () + if suite in ("statistics", "all"): + cases += statistics_cases(seed) + if suite in ("counted", "all"): + cases += counted_cases(seed) + if suite in ("heldout", "all"): + cases += heldout_cases(seed) + return dict( + schema_version=1, + provenance="Synthetic priors inspired by a character-length document-count screenshot; not tokenized corpus data.", + assumptions={ + "total_tokens": TOTAL_TOKENS, + "alignment": 4, + "k": 1024, + "character_bins_lower_exclusive_upper_inclusive": CHARACTER_BINS, + "document_count_prior_weights": COUNT_PRIORS, + "character_divisor": "1/2/4 are sensitivity assumptions, not a measured tokenizer conversion.", + "within_bin": "Uniform over aligned token lengths after scaling.", + "packing": "Fill short-mixture budget, truncate only the last draw, add explicit long docs, then reorder.", + "tail": "64K-512K spikes are extra stress cases; screenshot tail probabilities are unknown.", + "fixed_count": "256/512/1000/1024 documents; tiny mixtures use about 80% 64-token documents by count, with the remaining budget assigned explicitly. These are token stress shapes, not a measured character-to-token conversion.", + }, + cases=[dict(**asdict(case), statistics=summarize(case)) for case in cases], + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--suite", choices=("statistics", "counted", "heldout", "all"), default="statistics") + parser.add_argument("--output", type=Path, help="Write exact lengths, assumptions and count/token histograms as JSON") + args = parser.parse_args() + data = manifest(args.seed, args.suite) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(data, indent=2) + "\n") + print("| Case | Documents | <=100 count % | <=100 token % | Max tokens | Explicit long token % |") + print("|---|---:|---:|---:|---:|---:|") + for case in data["cases"]: + stats = case["statistics"] + print( + f"| {case['name']} | {stats['documents']} | {100 * stats['tiny_document_fraction']:.2f} | " + f"{100 * stats['tiny_token_fraction']:.2f} | {stats['max_tokens']} | " + f"{100 * stats['explicit_long_token_fraction']:.1f} |" + ) + + +if __name__ == "__main__": + main() diff --git a/cula/distributed/__init__.py b/cula/distributed/__init__.py index 7930bbde..41e289c1 100644 --- a/cula/distributed/__init__.py +++ b/cula/distributed/__init__.py @@ -9,13 +9,17 @@ cudnn_kda_cp, ) from cula.distributed.plan import KimiCPPlan, build_kimi_cp_plan +from cula.distributed.redistribute import THDRedistributionPlan, THDRedistributor from cula.distributed.thd_planner import ( THDLayoutMetrics, THDLayoutPlan, THDPlannerConfig, THDRankLayout, + THDSelectionCostModel, THDSpan, plan_thd_layout, + plan_thd_selection, + zigzag_thd_layout, ) __all__ = [ @@ -26,8 +30,13 @@ "THDLayoutPlan", "THDPlannerConfig", "THDRankLayout", + "THDRedistributionPlan", + "THDRedistributor", + "THDSelectionCostModel", "THDSpan", "build_kimi_cp_plan", "cudnn_kda_cp", "plan_thd_layout", + "plan_thd_selection", + "zigzag_thd_layout", ] diff --git a/cula/distributed/redistribute.py b/cula/distributed/redistribute.py new file mode 100644 index 00000000..4672ca7c --- /dev/null +++ b/cula/distributed/redistribute.py @@ -0,0 +1,186 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Lossless row redistribution from contiguous CP shards to THD span layouts. + +Only row ownership/order changes. Floating point values are neither reduced +nor cast. The backward is the exact inverse exchange. Preparation is CPU +metadata work plus device index allocation; execution is eager PyTorch/NCCL +and allocates outputs. No CUDA Graph contract is implied. +""" + +from dataclasses import dataclass + +import torch +import torch.distributed as dist +from torch.autograd.function import once_differentiable + +from cula.distributed.thd_planner import THDRankLayout + + +@dataclass(frozen=True) +class THDRedistributionPlan: + rank: int + world_size: int + source_rows: int + target_rows: int + send_counts: tuple[int, ...] + receive_counts: tuple[int, ...] + send_ranges: tuple[tuple[int, int], ...] + receive_ranges: tuple[tuple[int, int], ...] + identity: bool + + @classmethod + def build(cls, ranks: tuple[THDRankLayout, ...], source_offsets: tuple[int, ...], rank: int): + world = len(ranks) + if world < 1 or len(source_offsets) != world + 1 or source_offsets[0] != 0: + raise ValueError("source_offsets must describe one contiguous shard per rank starting at zero") + if any(isinstance(x, bool) or not isinstance(x, int) for x in source_offsets): + raise TypeError("source offsets must be integers") + if any(a > b for a, b in zip(source_offsets, source_offsets[1:])): + raise ValueError("source offsets must be nondecreasing") + if isinstance(rank, bool) or not isinstance(rank, int) or not 0 <= rank < world: + raise ValueError("invalid rank") + intervals = [] + for owner, layout in enumerate(ranks): + if isinstance(layout.rank, bool) or not isinstance(layout.rank, int): + raise TypeError("layout ranks must be integers") + if layout.rank != owner: + raise ValueError("rank layouts must be in group-rank order") + for span in layout.spans: + if any( + isinstance(value, bool) or not isinstance(value, int) + for value in (span.owner, span.token_start, span.token_end, span.sequence_start, span.sequence_end) + ): + raise TypeError("span owners and interval bounds must be integers") + if span.owner != owner: + raise ValueError("span owner does not match its rank layout") + if not (span.sequence_start <= span.token_start < span.token_end <= span.sequence_end): + raise ValueError("span must be a nonempty interval within its sequence") + intervals.append((span.token_start, span.token_end)) + cursor = 0 + for start, end in sorted(intervals): + if start != cursor: + raise ValueError("target layout must cover every source row exactly once, without gaps or overlap") + cursor = end + if cursor != source_offsets[-1]: + raise ValueError("target layout does not cover the source extent") + + send_ranges, receive_ranges = [], [] + send_counts, receive_counts = [0] * world, [0] * world + identity = True + left, right = source_offsets[rank : rank + 2] + for destination, layout in enumerate(ranks): + target_offset = 0 + for span in layout.spans: + # Global identity is required: one locally unchanged rank must + # still join the collective if any peer changes ownership. + identity &= span.token_start == source_offsets[destination] + target_offset + a, z = max(left, span.token_start), min(right, span.token_end) + if a < z: + send_counts[destination] += z - a + send_ranges.append((a - left, z - left)) + target_offset += span.tokens + identity &= target_offset == source_offsets[destination + 1] - source_offsets[destination] + + # Receive buffer groups rows by source, then by destination span order. + for source, (left, right) in enumerate(zip(source_offsets, source_offsets[1:])): + target_offset = 0 + for span in ranks[rank].spans: + a, z = max(left, span.token_start), min(right, span.token_end) + if a < z: + receive_counts[source] += z - a + receive_ranges.append((target_offset + a - span.token_start, target_offset + z - span.token_start)) + target_offset += span.tokens + return cls( + rank, + world, + source_offsets[rank + 1] - source_offsets[rank], + ranks[rank].local_tokens, + tuple(send_counts), + tuple(receive_counts), + tuple(send_ranges), + tuple(receive_ranges), + bool(identity), + ) + + @property + def remote_rows(self): + return sum(self.send_counts) - self.send_counts[self.rank] + + +def _permutations(ranges, rows, device): + # Avoid allocating or applying an index tensor for a naturally ordered side. + cursor, ordered = 0, True + for start, end in ranges: + ordered &= start == cursor + cursor = end + if ordered and cursor == rows: + return None, None + order = ( + torch.cat([torch.arange(a, z, dtype=torch.int64, device="cpu") for a, z in ranges]) + if rows + else torch.empty(0, dtype=torch.int64, device="cpu") + ) + if order.numel() != rows: + raise ValueError("incomplete permutation") + inverse = torch.empty_like(order) + inverse[order] = torch.arange(rows, device="cpu") + return order.to(device), inverse.to(device) + + +class THDRedistributor: + """Prepared eager exchange with explicit forward and inverse mappings. + + Every rank must prepare from identical global layouts and call exchanges + in the same order. Trailing shapes and dtypes must agree across peers. + The process group must stay alive for every backward using this object. + """ + + def __init__(self, plan: THDRedistributionPlan, device, *, group=None): + if dist.get_world_size(group) != plan.world_size or dist.get_rank(group) != plan.rank: + raise ValueError("plan identity does not match the process group") + self.plan, self.group, self.device = plan, group, torch.device(device) + if self.device.type == "cuda" and self.device.index is None: + self.device = torch.device("cuda", torch.cuda.current_device()) + self.send_order, self.send_inverse = _permutations(plan.send_ranges, plan.source_rows, self.device) + self.receive_order, self.receive_inverse = _permutations(plan.receive_ranges, plan.target_rows, self.device) + + def _exchange(self, tensor, reverse): + plan = self.plan + rows = plan.target_rows if reverse else plan.source_rows + if tensor.ndim < 1 or tensor.shape[0] != rows or tensor.device != self.device: + raise ValueError(f"expected {rows} rows on {self.device}, got {tuple(tensor.shape)} on {tensor.device}") + if not tensor.is_contiguous(): + raise ValueError("redistribution requires contiguous rows; explicit packing belongs to the caller") + if plan.identity: + return tensor + order = self.receive_order if reverse else self.send_order + inverse = self.send_inverse if reverse else self.receive_inverse + send_counts, receive_counts = ( + (plan.receive_counts, plan.send_counts) if reverse else (plan.send_counts, plan.receive_counts) + ) + packed = tensor if order is None else tensor.index_select(0, order) + received = tensor.new_empty((sum(receive_counts), *tensor.shape[1:])) + dist.all_to_all_single( + received, packed, output_split_sizes=list(receive_counts), input_split_sizes=list(send_counts), group=self.group + ) + return received if inverse is None else received.index_select(0, inverse) + + def distribute(self, tensor): + return _Redistribute.apply(tensor, self, False) + + def restore(self, tensor): + return _Redistribute.apply(tensor, self, True) + + +class _Redistribute(torch.autograd.Function): + @staticmethod + def forward(ctx, tensor, exchange, reverse): + ctx.exchange, ctx.reverse = exchange, reverse + return exchange._exchange(tensor, reverse) + + @staticmethod + @once_differentiable + def backward(ctx, grad): + return ctx.exchange._exchange(grad.contiguous(), not ctx.reverse), None, None diff --git a/cula/distributed/thd_planner.py b/cula/distributed/thd_planner.py index b7646fc0..009e7c5b 100644 --- a/cula/distributed/thd_planner.py +++ b/cula/distributed/thd_planner.py @@ -11,9 +11,12 @@ from __future__ import annotations +from bisect import bisect_left, bisect_right from collections import defaultdict from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace +from heapq import heapify, heappop, heappush +from itertools import groupby from math import ceil, isfinite from operator import index @@ -75,7 +78,7 @@ class THDLayoutMetrics: @dataclass(frozen=True, slots=True) class THDLayoutPlan: - """Complete global span assignment selected by :func:`plan_thd_layout`.""" + """Complete global span assignment selected by a THD planner.""" world_size: int block_size: int @@ -83,6 +86,7 @@ class THDLayoutPlan: ranks: tuple[THDRankLayout, ...] metrics: THDLayoutMetrics evaluated_scores: tuple[tuple[str, float], ...] + score_unit: str = "normalized" def for_rank(self, rank: int) -> THDRankLayout: if not isinstance(rank, int) or not 0 <= rank < self.world_size: @@ -108,6 +112,160 @@ class THDPlannerConfig: boundary_weight: float = 0.02 +@dataclass(frozen=True, slots=True) +class THDSelectionCostModel: + """Calibrated eager selection-routing costs; no hardware detection or I/O. + + Coefficients must describe the caller's indexer shape and transport. Q/O + attention and backward stay in source ownership. ``bytes_per_query`` sums + outgoing indexer Q/weights and returning IDs/scores, without dtype changes. + Bandwidths are effective per-rank GB/s, not advertised link speeds. Group + rank -> node IDs is explicit; the source is a contiguous balanced shard. + ``min_gain_fraction`` requires headroom over contiguous before migrating. + ``min_gain_ms`` additionally requires an absolute predicted saving per use; + callers may price one-time preparation divided by expected layout reuse. + ``peer_gbps`` optionally prices the maximum aggregated source/destination + edge, since sparse peer traffic may not saturate aggregate rank bandwidth. + ``inter_node_aggregate_gbps`` optionally bounds shared node injection and + receive bandwidth. ``peer_startup_us`` prices the busiest rank's nonempty + peer count, summed over the ordered forward/return collectives. Calibrate + these independently of kernel arrival waits; they are effective costs, + not physical-link specifications. Unspecified limits preserve old profiles. + """ + + candidate_ns: float + query_ns: float + span_us: float + bytes_per_query: int + intra_node_gbps: float + inter_node_gbps: float + rank_node_ids: tuple[int, ...] + packing_ns_per_query: float = 0.0 + exchange_us: float = 0.0 + min_gain_fraction: float = 0.05 + peer_gbps: float | None = None + min_gain_ms: float = 0.0 + peer_startup_us: float = 0.0 + inter_node_aggregate_gbps: float | None = None + overlap_intra_inter: bool = False + + def __post_init__(self): + if not isinstance(self.overlap_intra_inter, bool): + raise TypeError("overlap_intra_inter must be a boolean") + for name in ( + "candidate_ns", + "query_ns", + "span_us", + "packing_ns_per_query", + "exchange_us", + "min_gain_fraction", + "min_gain_ms", + "peer_startup_us", + ): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite and nonnegative") + for name in ("intra_node_gbps", "inter_node_gbps"): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive") + if self.min_gain_fraction >= 1: + raise ValueError("min_gain_fraction must be below one") + for name in ("peer_gbps", "inter_node_aggregate_gbps"): + value = getattr(self, name) + if value is not None and ( + isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(value) or value <= 0 + ): + raise ValueError(f"{name} must be finite and positive when supplied") + if isinstance(self.bytes_per_query, bool) or not isinstance(self.bytes_per_query, int) or self.bytes_per_query <= 0: + raise ValueError("bytes_per_query must be a positive integer") + if not isinstance(self.rank_node_ids, tuple) or not self.rank_node_ids: + raise TypeError("rank_node_ids must be a nonempty tuple") + if any(isinstance(n, bool) or not isinstance(n, int) or n < 0 for n in self.rank_node_ids): + raise ValueError("node ids must be nonnegative integers") + + def _migration_threshold_ms(self, baseline_ms): + return baseline_ms - max(baseline_ms * self.min_gain_fraction, self.min_gain_ms) + + def _outgoing_capacity_gbps(self, rank): + kinds = {node == self.rank_node_ids[rank] for peer, node in enumerate(self.rank_node_ids) if peer != rank} + rates = [self.intra_node_gbps if local else self.inter_node_gbps for local in kinds] + return sum(rates) if self.overlap_intra_inter and rates else max(rates, default=self.intra_node_gbps) + + def estimate_ms(self, ranks: tuple[THDRankLayout, ...], *, block_size: int) -> float: + world = len(ranks) + if len(self.rank_node_ids) != world: + raise ValueError("cost-model node map must have one entry per process-group rank") + total = sum(r.local_tokens for r in ranks) + offsets = tuple(x * block_size for x in balanced_shards(total // block_size, world)) + compute, incoming, outgoing = [0.0] * world, [0.0] * world, [0.0] * world + intra_in, intra_out, inter_in, inter_out = ([0.0] * world for _ in range(4)) + peer_rows = defaultdict(int) + source_cursor = list(offsets[:-1]) + send_ordered, receive_ordered = [True] * world, [True] * world + identity = True + for rank, layout in enumerate(ranks): + cursor = offsets[rank] + last_source = -1 + for span in layout.spans: + compute[rank] += ( + self.candidate_ns * _candidate_work(span.causal_offset, span.tokens, block_size) / 1e6 + + self.query_ns * span.tokens / 1e6 + + self.span_us / 1e3 + ) + identity &= span.token_start == cursor + cursor += span.tokens + for source, rows in _source_intersections(span, offsets): + send_ordered[source] &= max(offsets[source], span.token_start) == source_cursor[source] + source_cursor[source] += rows + receive_ordered[rank] &= source >= last_source + last_source = source + if source != rank: + gbps = ( + self.intra_node_gbps + if self.rank_node_ids[source] == self.rank_node_ids[rank] + else self.inter_node_gbps + ) + cost = rows * self.bytes_per_query / (gbps * 1e6) + outgoing[source] += cost + incoming[rank] += cost + local = self.rank_node_ids[source] == self.rank_node_ids[rank] + (intra_out if local else inter_out)[source] += cost + (intra_in if local else inter_in)[rank] += cost + peer_rows[source, rank] += rows + identity &= cursor == offsets[rank + 1] + if identity: + return max(compute) + # Execution has ordered exchange/compute/return phases. Model their + # critical durations separately rather than assuming unmeasured overlap. + transfer = max(*incoming, *outgoing) + if self.overlap_intra_inter: + transfer = max(*intra_in, *intra_out, *inter_in, *inter_out) + if self.peer_gbps is not None: + transfer = max(transfer, max(peer_rows.values(), default=0) * self.bytes_per_query / (self.peer_gbps * 1e6)) + node_in, node_out = defaultdict(int), defaultdict(int) + send_peers, receive_peers = [0] * world, [0] * world + for (source, target), rows in peer_rows.items(): + send_peers[source] += 1 + receive_peers[target] += 1 + a, b = self.rank_node_ids[source], self.rank_node_ids[target] + if a != b: + node_out[a] += rows + node_in[b] += rows + if self.inter_node_aggregate_gbps is not None: + node_rows = max((*node_in.values(), *node_out.values()), default=0) + transfer = max(transfer, node_rows * self.bytes_per_query / (self.inter_node_aggregate_gbps * 1e6)) + transfer += max(*send_peers, *receive_peers) * self.peer_startup_us / 1e3 + # Match the redistributor's no-op permutation elimination. A local + # forward pack and its inverse restore together copy bytes_per_query; + # the destination-side pair does the same only if receive order differs. + copies = max( + (offsets[r + 1] - offsets[r]) * (not send_ordered[r]) + layout.local_tokens * (not receive_ordered[r]) + for r, layout in enumerate(ranks) + ) + return max(compute) + transfer + copies * self.packing_ns_per_query / 1e6 + self.exchange_us / 1e3 + + @dataclass(frozen=True, slots=True) class _PlanningItem: sequence: int @@ -131,6 +289,45 @@ def _candidate_work(start: int, tokens: int, block_size: int) -> int: return _candidate_prefix(start + tokens, block_size) - _candidate_prefix(start, block_size) +def _source_intersections(span: THDSpan, offsets: tuple[int, ...]): + for source in range(bisect_right(offsets, span.token_start) - 1, bisect_left(offsets, span.token_end)): + yield source, min(offsets[source + 1], span.token_end) - max(offsets[source], span.token_start) + + +def _transfer_lower_bound(assignments, world, block_size, model, compute_mean, baseline): + """Optimistic lower bound on critical compute plus outgoing traffic. + + To reduce source rank i's compute below C, some candidate/token work must + leave it. A moved row can remove no more than the largest causal position + on that source. Ignore span overhead, incoming traffic, packing, capacity + and fixed exchange cost: all make this a lower bound, not a prediction. + Minimizing the maximum of affine functions only needs their intersections + and the interval endpoints. It remains valid for unequal destination rows. + """ + compute, density = [0.0] * world, [0.0] * world + for item, rank in assignments: + compute[rank] += (model.candidate_ns * item.causal_work + model.query_ns * item.tokens) / 1e6 + density[rank] = max( + density[rank], + (model.candidate_ns * ((item.token_end - item.sequence_start) // block_size) + model.query_ns) / 1e6, + ) + lines = [(1.0, 0.0)] + for rank in range(world): + if not density[rank]: + continue + rate = model._outgoing_capacity_gbps(rank) + slope = model.bytes_per_query / (rate * 1e6 * density[rank]) + lines.append((1 - slope, slope * compute[rank])) + points = [compute_mean, baseline] + for i, (a, b) in enumerate(lines): + for c, d in lines[:i]: + if a != c: + cross = (d - b) / (a - c) + if compute_mean < cross < baseline: + points.append(cross) + return min(max(a * x + b for a, b in lines) for x in points) + + def _validate_inputs( global_cu_seqlens: torch.Tensor | Sequence[int], *, @@ -186,7 +383,8 @@ def _contiguous_items(boundaries: tuple[int, ...], world_size: int, block_size: shard_offsets = tuple(value * block_size for value in block_shards) result = [] for rank, (shard_start, shard_end) in enumerate(zip(shard_offsets, shard_offsets[1:])): - for sequence, (sequence_start, sequence_end) in enumerate(zip(boundaries, boundaries[1:])): + for sequence in range(bisect_right(boundaries, shard_start) - 1, bisect_left(boundaries, shard_end)): + sequence_start, sequence_end = boundaries[sequence : sequence + 2] start = max(shard_start, sequence_start) end = min(shard_end, sequence_end) if start < end: @@ -200,6 +398,7 @@ def _adaptive_items( config: THDPlannerConfig, *, chunks_per_rank: int, + sequences: set[int] | None = None, ) -> list[_PlanningItem]: block_size = config.block_size total_blocks = boundaries[-1] // block_size @@ -208,6 +407,8 @@ def _adaptive_items( min_split_blocks = config.min_split_tokens // block_size result = [] for sequence, (sequence_start, sequence_end) in enumerate(zip(boundaries, boundaries[1:])): + if sequences is not None and sequence not in sequences: + continue length_blocks = (sequence_end - sequence_start) // block_size pieces = min(requested_grid, max(1, ceil(length_blocks / target_blocks))) # A sequence shorter than two useful spans stays whole. This is what @@ -223,7 +424,7 @@ def _adaptive_items( # Keep all ranks non-empty even for small batches containing fewer than W # sequences. The minimum span is a performance preference, not a semantic # restriction, so only this feasibility repair may split below it. - while len(result) < world_size: + while sequences is None and len(result) < world_size: splittable = [item for item in result if item.tokens >= 2 * block_size] if not splittable: raise ValueError("block alignment leaves fewer non-empty spans than ranks") @@ -275,23 +476,28 @@ def _greedy_assign(items: list[_PlanningItem], world_size: int) -> list[tuple[_P reverse=True, ) result = [] - for item in ordered: - owner = min( - range(world_size), - key=lambda rank: ( - max( - (token_loads[rank] + item.tokens) / ideal_tokens, - (causal_loads[rank] + item.causal_work) / ideal_causal, - ), - (token_loads[rank] + item.tokens) / ideal_tokens + (causal_loads[rank] + item.causal_work) / ideal_causal, - item_counts[rank], - rank, - ), - ) - result.append((item, owner)) - token_loads[owner] += item.tokens - causal_loads[owner] += item.causal_work - item_counts[owner] += 1 + for (tokens, work), group in groupby(ordered, key=lambda item: (item.tokens, item.causal_work)): + + def pressure(rank): + token_pressure = (token_loads[rank] + tokens) / ideal_tokens + causal_pressure = (causal_loads[rank] + work) / ideal_causal + return max(token_pressure, causal_pressure), token_pressure + causal_pressure, item_counts[rank], rank + + # Consecutive equal-cost pieces change only the selected rank's key. + # A heap retains the exact LPT ordering without scanning all CP ranks + # for every tiny document. Unique-cost pieces keep the simple minimum. + pieces = list(group) + heap = [pressure(rank) for rank in range(world_size)] + if len(pieces) > 1: + heapify(heap) + for item in pieces: + owner = heappop(heap)[-1] if len(pieces) > 1 else min(heap)[-1] + result.append((item, owner)) + token_loads[owner] += tokens + causal_loads[owner] += work + item_counts[owner] += 1 + if len(pieces) > 1: + heappush(heap, pressure(owner)) return result @@ -341,7 +547,19 @@ def _rebalance_tokens( loads[owner] += item.tokens causal_loads[owner] += item.causal_work ideal_causal = sum(causal_loads) / world_size - repaired = list(assignments) + # Stable serials preserve the old global-list tie order, while looking only + # at the selected donor avoids rescanning every unrelated short document. + bins = [dict() for _ in range(world_size)] + shapes = [defaultdict(list) for _ in range(world_size)] + + def insert(owner, serial, item): + bins[owner][serial] = item + shape = (item.tokens, item.token_start - item.sequence_start) + heappush(shapes[owner][shape], (item.sequence, item.token_start, serial)) + + for serial, (item, owner) in enumerate(assignments): + insert(owner, serial, item) + next_serial = len(assignments) while loads != capacities: receivers = [rank for rank in range(world_size) if loads[rank] < capacities[rank]] @@ -355,55 +573,56 @@ def _rebalance_tokens( loads[donor] - capacities[donor], ) - choices = [] - for assignment_index, (item, owner) in enumerate(repaired): - if owner != donor: - continue + other_pressure = max( + (causal_loads[r] / ideal_causal for r in range(world_size) if r not in (donor, receiver)), default=0.0 + ) + best = None + # Equal (length, causal offset) pieces have identical whole/prefix/ + # suffix costs. Only the earliest semantic tie-break can win, so one + # heap head represents all equivalent short documents exactly. + for heap in shapes[donor].values(): + serial = heap[0][2] + item = bins[donor][serial] transfer_tokens = min(item.tokens, transfer_limit) - if transfer_tokens == item.tokens: - variants = ((item, None),) - else: - variants = tuple( - _split_item( - item, - transfer_tokens, - take_prefix=take_prefix, - block_size=block_size, - ) - for take_prefix in (True, False) + split = transfer_tokens != item.tokens + for take_prefix in (True, False) if split else (True,): + moved_start = item.token_start if take_prefix else item.token_end - transfer_tokens + moved_work = ( + _candidate_work(moved_start - item.sequence_start, transfer_tokens, block_size) + if split + else item.causal_work ) - for moved, remainder in variants: - next_donor_causal = causal_loads[donor] - moved.causal_work - next_receiver_causal = causal_loads[receiver] + moved.causal_work causal_pressure = max( - next_donor_causal / ideal_causal, - next_receiver_causal / ideal_causal, - *(causal_loads[rank] / ideal_causal for rank in range(world_size) if rank not in (donor, receiver)), - ) - choices.append( - ( - causal_pressure, - remainder is not None, - -moved.tokens, - moved.sequence, - moved.token_start, - assignment_index, - moved, - remainder, - ) + (causal_loads[donor] - moved_work) / ideal_causal, + (causal_loads[receiver] + moved_work) / ideal_causal, + other_pressure, ) - if not choices: + choice = (causal_pressure, split, -transfer_tokens, item.sequence, moved_start, serial, take_prefix) + if best is None or choice < best: + best = choice + if best is None: raise RuntimeError("internal error: an overloaded rank has no movable THD span") - *_, assignment_index, moved, remainder = min(choices) - repaired.pop(assignment_index) + _, split, negative_tokens, _, _, serial, take_prefix = best + item = bins[donor].pop(serial) + shape = (item.tokens, item.token_start - item.sequence_start) + heappop(shapes[donor][shape]) + if not shapes[donor][shape]: + del shapes[donor][shape] + moved, remainder = ( + _split_item(item, -negative_tokens, take_prefix=take_prefix, block_size=block_size) if split else (item, None) + ) if remainder is not None: - repaired.append((remainder, donor)) - repaired.append((moved, receiver)) + insert(donor, next_serial, remainder) + next_serial += 1 + insert(receiver, next_serial, moved) + next_serial += 1 loads[donor] -= moved.tokens loads[receiver] += moved.tokens causal_loads[donor] -= moved.causal_work causal_loads[receiver] += moved.causal_work - return repaired + return [ + (item, owner) for _, item, owner in sorted((i, item, r) for r, items in enumerate(bins) for i, item in items.items()) + ] def _coalesce(assignments: list[tuple[_PlanningItem, int]]) -> list[THDSpan]: @@ -502,11 +721,91 @@ def _build_candidate( return policy, spans, metrics +def _minimum_assignment(costs: list[list[float]]) -> tuple[int, ...]: + """Minimum-cost bin -> rank permutation, O(W^3), with stable ties. + + The primal-dual augmenting-path algorithm keeps rank ownership bijective. + It needs no optional optimizer dependency and never enumerates W! maps. + """ + size = len(costs) + u, v = [0.0] * (size + 1), [0.0] * (size + 1) + matched, previous = [0] * (size + 1), [0] * (size + 1) + for row in range(1, size + 1): + matched[0] = row + column = 0 + best = [float("inf")] * (size + 1) + used = [False] * (size + 1) + while True: + used[column] = True + active = matched[column] + step, next_column = float("inf"), 0 + for candidate in range(1, size + 1): + if not used[candidate]: + reduced = costs[active - 1][candidate - 1] - u[active] - v[candidate] + if reduced < best[candidate]: + best[candidate], previous[candidate] = reduced, column + if best[candidate] < step: + step, next_column = best[candidate], candidate + for candidate in range(size + 1): + if used[candidate]: + u[matched[candidate]] += step + v[candidate] -= step + else: + best[candidate] -= step + column = next_column + if matched[column] == 0: + break + while column: + parent = previous[column] + matched[column] = matched[parent] + column = parent + assignment = [0] * size + for column in range(1, size + 1): + assignment[matched[column] - 1] = column - 1 + return tuple(assignment) + + +def _localize_spans(spans: list[THDSpan], world: int, block_size: int, model: THDSelectionCostModel) -> list[THDSpan]: + """Relabel equal-capacity bins to minimize aggregate transfer cost. + + Computation and fragmentation stay identical. This minimizes a sum, not + the critical path; the caller must still score the result against the + original candidate with the full max-rank model. + """ + offsets = tuple(x * block_size for x in balanced_shards(sum(s.tokens for s in spans) // block_size, world)) + overlaps = [[0] * world for _ in range(world)] + for span in spans: + for source, rows in _source_intersections(span, offsets): + overlaps[span.owner][source] += rows + capacities = defaultdict(list) + for rank, (a, z) in enumerate(zip(offsets, offsets[1:])): + capacities[z - a].append(rank) + owners = list(range(world)) + for peers in capacities.values(): + costs = [ + [ + sum( + rows + / (model.intra_node_gbps if model.rank_node_ids[s] == model.rank_node_ids[r] else model.inter_node_gbps) + for s, rows in enumerate(overlaps[owner]) + if s != r + ) + for r in peers + ] + for owner in peers + ] + assignment = _minimum_assignment(costs) + for owner, destination in zip(peers, assignment): + owners[owner] = peers[destination] + return [replace(span, owner=owners[span.owner]) for span in spans] + + def plan_thd_layout( global_cu_seqlens: torch.Tensor | Sequence[int], *, world_size: int, config: THDPlannerConfig | None = None, + cost_model: THDSelectionCostModel | None = None, ) -> THDLayoutPlan: """Plan a block-aligned THD layout and select the best modeled candidate. @@ -518,16 +817,44 @@ def plan_thd_layout( config = THDPlannerConfig() if config is None else config if not isinstance(config, THDPlannerConfig): raise TypeError("config must be a THDPlannerConfig") + if cost_model is not None and not isinstance(cost_model, THDSelectionCostModel): + raise TypeError("cost_model must be a THDSelectionCostModel") boundaries = _validate_inputs(global_cu_seqlens, world_size=world_size, config=config) + if cost_model is not None and len(cost_model.rank_node_ids) != world_size: + raise ValueError("cost-model node map must have one entry per process-group rank") + source_items = _contiguous_items(boundaries, world_size, config.block_size) contiguous = _build_candidate( - _contiguous_items(boundaries, world_size, config.block_size), + source_items, policy="contiguous", world_size=world_size, config=config, ) + if cost_model is not None and contiguous is not None: + policy, spans, metrics = contiguous + ranks = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world_size)) + baseline_ms = cost_model.estimate_ms(ranks, block_size=config.block_size) + # Every exact partition has the same candidate/token sums and at least + # one span per document. Even free transport cannot beat their mean. + lower_bound = ( + cost_model.candidate_ns * sum(metrics.causal_loads) / 1e6 + + cost_model.query_ns * boundaries[-1] / 1e6 + + cost_model.span_us * (len(boundaries) - 1) / 1e3 + ) / world_size + lower_bound = _transfer_lower_bound(source_items, world_size, config.block_size, cost_model, lower_bound, baseline_ms) + if lower_bound >= cost_model._migration_threshold_ms(baseline_ms): + return THDLayoutPlan( + world_size, + config.block_size, + policy, + ranks, + replace(metrics, score=baseline_ms), + ((policy, baseline_ms),), + "ms", + ) adaptive_candidates = [] seen_assignments = set() + seen_items = set() for chunks_per_rank in range(1, config.max_chunks_per_rank + 1): adaptive_items = _adaptive_items( boundaries, @@ -535,6 +862,10 @@ def plan_thd_layout( config, chunks_per_rank=chunks_per_rank, ) + item_signature = tuple(adaptive_items) + if item_signature in seen_items: + continue + seen_items.add(item_signature) adaptive = _build_candidate( _rebalance_tokens( _greedy_assign(adaptive_items, world_size), @@ -554,6 +885,35 @@ def plan_thd_layout( candidates = [candidate for candidate in (contiguous, *adaptive_candidates) if candidate is not None] if not candidates: raise RuntimeError("no feasible THD layout candidate") + if cost_model is not None: + localized = [] + for policy, spans, _ in candidates: + if policy == "contiguous": + continue + nearby = _localize_spans(spans, world_size, config.block_size, cost_model) + if nearby != spans: + localized.append( + ( + policy + "-local", + nearby, + _metrics(nearby, world_size=world_size, block_size=config.block_size, config=config), + ) + ) + candidates += localized + candidates = [ + ( + policy, + spans, + replace( + metrics, + score=cost_model.estimate_ms( + tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world_size)), + block_size=config.block_size, + ), + ), + ) + for policy, spans, metrics in candidates + ] selected_policy, selected_spans, selected_metrics = min( candidates, key=lambda candidate: ( @@ -562,6 +922,10 @@ def plan_thd_layout( candidate[0], ), ) + if cost_model is not None: + baseline = next(c for c in candidates if c[0] == "contiguous") + if selected_metrics.score >= cost_model._migration_threshold_ms(baseline[2].score): + selected_policy, selected_spans, selected_metrics = baseline ranks = tuple( THDRankLayout( rank, @@ -581,6 +945,387 @@ def plan_thd_layout( ranks, selected_metrics, tuple((policy, metrics.score) for policy, _, metrics in candidates), + "ms" if cost_model is not None else "normalized", + ) + + +def _selection_assign(items, pinned, *, world, capacity, offsets, model, block_size): + """Keep cheap source spans local; place long-span work under a row cap. + + Unlike a shared KDA/CSA layout, selection ownership need not have equal + token counts. Its invariant is bounded rows and exact coverage. Placement + scores ordered compute/transport phases using the same calibrated units + as final candidate selection, with stable load/locality tie breaks. + """ + loads, compute = [0] * world, [0.0] * world + incoming, outgoing = [0.0] * world, [0.0] * world + intra_in, intra_out, inter_in, inter_out = ([0.0] * world for _ in range(4)) + peer_rows = [[0] * world for _ in range(world)] + peer_peak = 0 + send_peers, receive_peers = [0] * world, [0] * world + degree_peak, node_peak = 0, 0 + node_in, node_out = defaultdict(int), defaultdict(int) + # Pinned spans initialize loads but are materialized once by the caller. + result = [] + + def work(item): + return model.candidate_ns * item.causal_work / 1e6 + model.query_ns * item.tokens / 1e6 + model.span_us / 1e3 + + for item, owner in pinned: + loads[owner] += item.tokens + compute[owner] += work(item) + for original in sorted(items, key=lambda item: (-work(item), -item.tokens, item.sequence, item.token_start)): + item = original + while item is not None: + # A cap is not an exact bin size. Split only when no remaining + # bin can hold this item, keeping short sequences intact otherwise. + free = max(capacity - n for n in loads) + if free <= 0: + raise RuntimeError("internal error: selection row capacity exhausted") + placed, remainder = ( + _split_item(item, free, take_prefix=True, block_size=block_size) if item.tokens > free else (item, None) + ) + intersections = tuple(_source_intersections(placed, offsets)) + cost = work(placed) + max_compute, max_incoming, max_outgoing = max(compute), max(incoming), max(outgoing) + if model.overlap_intra_inter: + max_incoming, max_outgoing = max(*intra_in, *inter_in), max(*intra_out, *inter_out) + best = None + for rank in range(world): + if loads[rank] + placed.tokens > capacity: + continue + additions = [] + moved = 0 + for source, rows in intersections: + if source == rank: + continue + rate = ( + model.intra_node_gbps + if model.rank_node_ids[source] == model.rank_node_ids[rank] + else model.inter_node_gbps + ) + additions.append((source, rows * model.bytes_per_query / (rate * 1e6))) + moved += rows + received = sum(value for _, value in additions) + transfer = max(max_incoming, incoming[rank] + received, max_outgoing) + transfer = max(transfer, max((outgoing[s] + value for s, value in additions), default=0.0)) + if model.overlap_intra_inter: + local_received = sum(v for s, v in additions if model.rank_node_ids[s] == model.rank_node_ids[rank]) + remote_received = sum(v for s, v in additions if model.rank_node_ids[s] != model.rank_node_ids[rank]) + transfer = max( + max_incoming, max_outgoing, intra_in[rank] + local_received, inter_in[rank] + remote_received + ) + transfer = max( + transfer, + max( + ( + (intra_out if model.rank_node_ids[s] == model.rank_node_ids[rank] else inter_out)[s] + v + for s, v in additions + ), + default=0.0, + ), + ) + candidate_peer_peak = max( + peer_peak, max((peer_rows[s][rank] + n for s, n in intersections if s != rank), default=0) + ) + if model.peer_gbps is not None: + transfer = max(transfer, candidate_peer_peak * model.bytes_per_query / (model.peer_gbps * 1e6)) + new_edges = [s for s, n in intersections if s != rank and not peer_rows[s][rank]] + candidate_degree = max( + degree_peak, + receive_peers[rank] + len(new_edges), + max((send_peers[s] + 1 for s in new_edges), default=0), + ) + node_additions = defaultdict(int) + for s, n in intersections: + if model.rank_node_ids[s] != model.rank_node_ids[rank]: + node_additions[model.rank_node_ids[s]] += n + candidate_node = max( + node_peak, + node_in[model.rank_node_ids[rank]] + sum(node_additions.values()), + max((node_out[n] + rows for n, rows in node_additions.items()), default=0), + ) + if model.inter_node_aggregate_gbps is not None: + transfer = max(transfer, candidate_node * model.bytes_per_query / (model.inter_node_aggregate_gbps * 1e6)) + transfer += candidate_degree * model.peer_startup_us / 1e3 + # Partial placement uses an upper bound on copies. Final + # scoring checks the actual source/destination permutations. + copies = max( + offsets[r + 1] - offsets[r] + n + (placed.tokens if r == rank else 0) for r, n in enumerate(loads) + ) + critical = max(max_compute, compute[rank] + cost) + transfer + if candidate_peer_peak: + critical += copies * model.packing_ns_per_query / 1e6 + model.exchange_us / 1e3 + key = (critical, compute[rank] + cost + incoming[rank] + received, moved, loads[rank], rank) + if best is None or key < best[0]: + best = key, rank, additions, received, candidate_peer_peak, candidate_degree, candidate_node + _, owner, additions, received, peer_peak, degree_peak, node_peak = best + result.append((placed, owner)) + loads[owner] += placed.tokens + compute[owner] += cost + incoming[owner] += received + for source, value in additions: + outgoing[source] += value + local = model.rank_node_ids[source] == model.rank_node_ids[owner] + (intra_out if local else inter_out)[source] += value + (intra_in if local else inter_in)[owner] += value + for source, rows in intersections: + if source != owner: + if not peer_rows[source][owner]: + send_peers[source] += 1 + receive_peers[owner] += 1 + peer_rows[source][owner] += rows + a, b = model.rank_node_ids[source], model.rank_node_ids[owner] + if a != b: + node_out[a] += rows + node_in[b] += rows + item = remainder + return result + + +def _paired_selection_items(boundaries, movable, world, block_size, offsets, model): + """Pair early/late chunks only within long documents, keeping shorts local.""" + result = [] + for sequence in sorted(movable): + start, end = boundaries[sequence : sequence + 2] + # Unlike balanced_shards, empty pieces are legitimate for tiny custom + # split thresholds. They never become empty THDSpan objects. + base, remainder = divmod((end - start) // block_size, 2 * world) + cuts = [start + (i * base + min(i, remainder)) * block_size for i in range(2 * world + 1)] + pairs = [] + for rank in range(world): + pairs.append( + [ + _make_item(sequence, cuts[i], cuts[i + 1], start, end, block_size) + for i in (rank, 2 * world - 1 - rank) + if cuts[i] < cuts[i + 1] + ] + ) + costs = [] + for pair in pairs: + overlaps = [0] * world + for item in pair: + for source, rows in _source_intersections(item, offsets): + overlaps[source] += rows + costs.append( + [ + sum( + rows + / ( + model.intra_node_gbps + if model.rank_node_ids[source] == model.rank_node_ids[rank] + else model.inter_node_gbps + ) + for source, rows in enumerate(overlaps) + if source != rank + ) + for rank in range(world) + ] + ) + owners = _minimum_assignment(costs) + result.extend((item, owners[rank]) for rank, pair in enumerate(pairs) for item in pair) + return result + + +def zigzag_thd_layout( + global_cu_seqlens: torch.Tensor | Sequence[int], *, world_size: int, block_size: int = 4 +) -> tuple[THDRankLayout, ...]: + """Conventional per-document early/late pairing, including short documents. + + Rank r owns pieces r and 2*CP-1-r. Empty pieces are omitted; adjacent + pieces remain distinct, matching the selection backend's execution spans. + This layout can have unequal or empty ranks and does not impose a row cap. + """ + config = THDPlannerConfig(block_size=block_size, min_split_tokens=block_size) + boundaries = _validate_inputs(global_cu_seqlens, world_size=world_size, config=config) + ranks = [[] for _ in range(world_size)] + for sequence, (start, end) in enumerate(zip(boundaries, boundaries[1:])): + base, remainder = divmod((end - start) // block_size, 2 * world_size) + cuts = [start + (i * base + min(i, remainder)) * block_size for i in range(2 * world_size + 1)] + for rank in range(world_size): + for piece in (rank, 2 * world_size - 1 - rank): + a, z = cuts[piece : piece + 2] + if a < z: + ranks[rank].append(THDSpan(sequence, a, z, start, end, rank)) + return tuple(THDRankLayout(r, tuple(spans)) for r, spans in enumerate(ranks)) + + +def _zigzag_lower_bound(boundaries, source, world, block_size, model): + """Average compute plus exact outgoing traffic, without building THD spans. + + Consecutive zigzag pieces assigned to the same node form interval runs. + Intersect each source span with its rank's two pieces and its node's runs: + this gives exact self, intra-node and inter-node row counts. The number of + runs is small for contiguous node ranks; arbitrary explicit topologies are + supported too. Average compute and outgoing-only transport are lower bounds + on their respective critical times. Packing and incoming/peer bottlenecks + are omitted, so this cannot overprice the concrete full-zigzag candidate. + """ + piece_nodes = [model.rank_node_ids[min(i, 2 * world - 1 - i)] for i in range(2 * world)] + node_runs = defaultdict(list) + begin = 0 + for node, pieces in groupby(piece_nodes): + end = begin + sum(1 for _ in pieces) + node_runs[node].append((begin, end)) + begin = end + geometry, total_work, spans = [], 0, 0 + for a, z in zip(boundaries, boundaries[1:]): + blocks = (z - a) // block_size + geometry.append(divmod(blocks, 2 * world)) + total_work += _candidate_work(0, z - a, block_size) + spans += min(blocks, 2 * world) + intra, inter = [0] * world, [0] * world + for item, rank in source: + base, remainder = geometry[item.sequence] + + def overlap(left, right): + a = item.sequence_start + (left * base + min(left, remainder)) * block_size + z = item.sequence_start + (right * base + min(right, remainder)) * block_size + return max(0, min(item.token_end, z) - max(item.token_start, a)) + + own_rows = overlap(rank, rank + 1) + overlap(2 * world - 1 - rank, 2 * world - rank) + node_rows = sum(overlap(a, z) for a, z in node_runs[model.rank_node_ids[rank]]) + intra[rank] += node_rows - own_rows + inter[rank] += item.tokens - node_rows + transfer = 0.0 + for local, remote in zip(intra, inter): + a, b = local / model.intra_node_gbps, remote / model.inter_node_gbps + transfer = max(transfer, (max(a, b) if model.overlap_intra_inter else a + b) * model.bytes_per_query / 1e6) + if model.inter_node_aggregate_gbps is not None: + node_out = defaultdict(int) + for rank, rows in enumerate(inter): + node_out[model.rank_node_ids[rank]] += rows + transfer = max(transfer, max(node_out.values()) * model.bytes_per_query / (model.inter_node_aggregate_gbps * 1e6)) + if any(intra) or any(inter): + # Each used link kind requires at least one distinct remote peer. + peers = max((a > 0) + (b > 0) for a, b in zip(intra, inter)) + transfer += (model.exchange_us + model.peer_startup_us * peers) / 1e3 + compute = (model.candidate_ns * total_work + model.query_ns * boundaries[-1]) / (world * 1e6) + compute += model.span_us * spans / (world * 1e3) + return compute + transfer + + +def plan_thd_selection( + global_cu_seqlens: torch.Tensor | Sequence[int], + *, + world_size: int, + max_tokens_per_rank: int, + cost_model: THDSelectionCostModel, + config: THDPlannerConfig | None = None, +) -> THDLayoutPlan: + """Plan selection-only routing with an explicit per-rank row capacity. + + Compare contiguous, complete per-document zigzag and locality-preserving + long-document candidates in the same cost model. Only the latter pin + short source spans and split documents at least twice ``min_split_tokens``. + Target row counts + may differ: attention and KDA must retain source ownership, and selection + IDs/scores must be restored through the inverse exchange before use. + + The capacity includes pinned rows and must fit every contiguous source + shard. It bounds query-shaped buffers, not total backend workspace; the + caller must also budget selection scratch and gathered keys. This eager + CPU planner does no device probing, timing or implicit caching. + """ + config = THDPlannerConfig() if config is None else config + if not isinstance(config, THDPlannerConfig): + raise TypeError("config must be a THDPlannerConfig") + if not isinstance(cost_model, THDSelectionCostModel): + raise TypeError("cost_model must be a THDSelectionCostModel") + boundaries = _validate_inputs(global_cu_seqlens, world_size=world_size, config=config) + if len(cost_model.rank_node_ids) != world_size: + raise ValueError("cost-model node map must have one entry per process-group rank") + if isinstance(max_tokens_per_rank, bool) or not isinstance(max_tokens_per_rank, int): + raise TypeError("max_tokens_per_rank must be an integer") + offsets = tuple(x * config.block_size for x in balanced_shards(boundaries[-1] // config.block_size, world_size)) + if max_tokens_per_rank % config.block_size or max_tokens_per_rank < max(b - a for a, b in zip(offsets, offsets[1:])): + raise ValueError("max_tokens_per_rank must be block aligned and fit every source shard") + base = _contiguous_items(boundaries, world_size, config.block_size) + movable = {i for i, (a, z) in enumerate(zip(boundaries, boundaries[1:])) if z - a >= 2 * config.min_split_tokens} + pinned = [(item, rank) for item, rank in base if item.sequence not in movable] + pinned_spans = () + candidates = [] + + def add(assignments, policy, *, include_pinned=True): + spans = _coalesce(assignments) + if include_pinned: + # Pinned and movable documents are disjoint. Reusing their spans + # avoids rebuilding hundreds of identical short-document objects + # for every planning grid. + spans = sorted((*pinned_spans, *spans), key=lambda s: (s.sequence, s.token_start)) + grouped = [[] for _ in range(world_size)] + for span in spans: + grouped[span.owner].append(span) + ranks = tuple(THDRankLayout(r, tuple(group)) for r, group in enumerate(grouped)) + if any(rank.local_tokens > max_tokens_per_rank for rank in ranks): + return + score = cost_model.estimate_ms(ranks, block_size=config.block_size) + candidates.append((policy, ranks, score)) + + add(base, "contiguous", include_pinned=False) + baseline_ms = candidates[0][2] + lower_bound = baseline_ms + # This bound allows arbitrary row movement, including short documents. + # It may prune full zigzag too, but the long-document threshold may not. + if baseline_ms > cost_model.min_gain_ms: + lower_bound = ( + cost_model.candidate_ns * sum(item.causal_work for item, _ in base) / 1e6 + + cost_model.query_ns * boundaries[-1] / 1e6 + + cost_model.span_us * (len(boundaries) - 1) / 1e3 + ) / world_size + lower_bound = _transfer_lower_bound(base, world_size, config.block_size, cost_model, lower_bound, baseline_ms) + if movable and lower_bound < cost_model._migration_threshold_ms(baseline_ms): + pinned_spans = _coalesce(pinned) + add( + _paired_selection_items(boundaries, movable, world_size, config.block_size, offsets, cost_model), + "selection-paired-local", + ) + seen = set() + for grid in range(1, config.max_chunks_per_rank + 1): + items = tuple(_adaptive_items(boundaries, world_size, config, chunks_per_rank=grid, sequences=movable)) + if items in seen: + continue + seen.add(items) + add( + _selection_assign( + items, + pinned, + world=world_size, + capacity=max_tokens_per_rank, + offsets=offsets, + model=cost_model, + block_size=config.block_size, + ), + f"selection-local-{grid}x", + ) + # Score the cheap local candidates first. They tighten the incumbent, + # allowing an admissible bound to reject a fragmented full zigzag without + # allocating O(documents * CP) span objects. No candidate-specific heuristic + # threshold is used: any full zigzag that can win is still materialized. + threshold = min(cost_model._migration_threshold_ms(baseline_ms), min(c[2] for c in candidates)) + if ( + lower_bound < threshold + and _zigzag_lower_bound(boundaries, base, world_size, config.block_size, cost_model) < threshold + ): + zigzag = zigzag_thd_layout(boundaries, world_size=world_size, block_size=config.block_size) + if all(rank.local_tokens <= max_tokens_per_rank for rank in zigzag): + candidates.append(("zigzag", zigzag, cost_model.estimate_ms(zigzag, block_size=config.block_size))) + selected = min(candidates, key=lambda c: (c[2], c[0] != "contiguous", c[0])) + if selected[2] >= cost_model._migration_threshold_ms(baseline_ms): + selected = candidates[0] + policy, ranks, score = selected + metrics = _metrics( + [span for rank in ranks for span in rank.spans], world_size=world_size, block_size=config.block_size, config=config + ) + metrics = replace(metrics, score=score) + return THDLayoutPlan( + world_size, + config.block_size, + policy, + ranks, + metrics, + tuple((name, value) for name, _, value in candidates), + "ms", ) @@ -588,7 +1333,10 @@ def plan_thd_layout( "THDLayoutMetrics", "THDLayoutPlan", "THDPlannerConfig", + "THDSelectionCostModel", "THDRankLayout", "THDSpan", "plan_thd_layout", + "plan_thd_selection", + "zigzag_thd_layout", ] diff --git a/docs/THD_1M_STATISTICS_CASES.md b/docs/THD_1M_STATISTICS_CASES.md new file mode 100644 index 00000000..0790927c --- /dev/null +++ b/docs/THD_1M_STATISTICS_CASES.md @@ -0,0 +1,114 @@ +# Statistics-inspired 1M THD workload catalog + +The motivating histogram describes **document counts by character length**. Its largest visible bucket is `(10,100]`; exact percentages, +the full long tail and tokenizer information were not supplied. This catalog +uses explicit synthetic priors to explore that shape. It is not a reconstruction +of the real token distribution or a claim about hidden chart values. + +There are **18 new cases**, in addition to the previous five baseline cases. +Every case contains exactly **1,048,576 tokens**, with positive document lengths +divisible by four for the current complete-CSA CP harness. All source CP16 +shards consequently have 65,536 tokens. The generator uses only Python's +standard library and does not initialize CUDA or distributed execution. + +## Distribution assumptions + +Weights below are **sample-count priors**, not token shares or measurements +read from the chart. The primary `mix80` prior approximates the chart's visible +shape; `tiny90` and `mix70` test sensitivity to its dominant short bucket. +The generator samples uniformly over aligned lengths within each bucket. + +| Synthetic prior | (10,100] | (100,1K] | (1K,2K] | (2K,4K] | (4K,8K] | (8K,16K] | +|---|---:|---:|---:|---:|---:|---:| +| tiny90 | 90% | 5% | 2% | 2% | 0.8% | 0.2% | +| mix80 | 80% | 7% | 5% | 5% | 2.5% | 0.5% | +| mix70 | 70% | 12% | 8% | 6% | 3% | 1% | + +K=1024 is a synthetic convention. Default bucket endpoints act as token-length +proxies. Two additional cases divide those endpoints by 2 and 4 before drawing +aligned token lengths; these are character/token **sensitivity assumptions**, +not measured conversions. Actual tokenization is required to calibrate them. +For these scaled cases, the reported `<=100 token` fraction is not the same +quantity as the source chart's `(10,100] character` fraction. + +Sampling stops at the desired mixture token budget. Only the final draw may +be shortened to fill exactly; the manifest records the number of tokens removed. +It never fills a residual with an invented giant document. Explicit long stress +documents are then added without truncation. Fixed seed `20261112` and a +separate deterministic permutation make the complete ordered length list +reproducible. Different seeds provide additional replicas. + +## Twelve primary cases + +All numbers below are the **generated default-seed values**, not expected +probabilities. Prefix every short name with `1m-stat-` to obtain the benchmark +case id. Randomized cases use a reproducible shuffled order. + +| Case | Documents | <=100-token document share | <=100-token token share | Largest document | Explicit long-token share | +|---|---:|---:|---:|---:|---:| +| tiny64 | 16,384 | 100.00% | 100.00% | 64 | 0% | +| tiny90 | 4,100 | 89.68% | 19.75% | 14,148 | 0% | +| mix80 | 1,857 | 79.43% | 7.88% | 15,064 | 0% | +| mix70 | 1,403 | 69.57% | 5.27% | 15,932 | 0% | +| mix80-div2 | 3,860 | 80.00% | 8.37% | 7,532 | 0% | +| mix80-div4 | 7,783 | 82.04% | 9.53% | 4,052 | 0% | +| mix80-1x128k | 1,639 | 79.44% | 6.94% | 131,072 | 12.5% | +| mix80-1x256k | 1,450 | 79.03% | 6.10% | 262,144 | 25% | +| mix80-1x512k | 1,066 | 79.55% | 4.60% | 524,288 | 50% | +| mix80-2x256k | 1,067 | 79.48% | 4.60% | 262,144 | 50% | +| mix80-4x128k | 1,069 | 79.33% | 4.60% | 131,072 | 50% | +| mix80-8x64k | 1,073 | 79.03% | 4.60% | 65,536 | 50% | + +The distinction between document share and token share is material: `mix80` +contains nearly 80% tiny documents, but those documents contribute only 7.88% +of tokens. In `mix80-2x256k`, just two of 1,067 documents contribute 50% of all +tokens. These cases expose both many-document overhead and long-prefix causal +work, which the earlier 22-document mixed case could not represent together. + +The four 50%-long-token cases share **the identical 524,288-token mixture +background**, while splitting the remaining 524,288 tokens as one, two, four +or eight long documents. This controls total long-token volume and tests +whether the planner handles changing document count and causal lengths. +The 64K–512K spikes are deliberate stress additions; their frequency is not +inferred from the chart. + +`tiny64` stresses many short segments, boundaries, padding and scheduling. +`mix70`/`mix80`/`tiny90` vary document-count concentration. `div2`/`div4` increase +document density at a fixed token budget. Long-spike cases stress causal load, +communication and scratch memory without making document-count histograms +look predominantly long. + +## Six order-only controls + +For **both** `mix80` and `mix80-2x256k`, add these suffixes: + +| Suffix | Ordering | Purpose | +|---|---|---| +| `-ascending` | shortest to longest | Cluster longer documents at the end of packed input | +| `-descending` | longest to shortest | Move long-document ownership toward earlier source ranks | +| `-alternating` | shortest, longest, second shortest, second longest, ... | Interleave the same histogram | + +Each variant retains exactly the same length multiset and total as its +shuffled counterpart. Thus performance differences can be associated with +packing order/rank ownership rather than a different length histogram. The +control fixes lengths, not document-content tensors across order variants: +the current GPU harness still creates synthetic operands from rank-local RNG. + +## Reproducible suites + +The generator `benchmarks/thd_1m_cases.py` writes exact lengths, seeds, +assumptions, SHA256 digests and count/token histograms. The matrix runner +`benchmarks/bench_csa_layout_matrix.py` accepts `--list-cases` without CUDA. + +- `baseline`: five single, uniform, near-uniform and long-tail cases. +- `counted`: eleven 256--1024-document cases, including order-only controls. +- `heldout`: five lognormal, jittered, tiny-tail and trimodal cases for a supplied seed. +- `statistics`: the eighteen synthetic histogram cases described above. +- `validation`: the five baseline cases, eleven counted cases at seed 20261112, + and five held-out cases at each of seeds 20261114, 20261115 and 20261116. + This fixes the exact **31-case** matrix in the [measured report](THD_SELECTION_RESULTS.md). + +The statistics suite is an additional stress catalog; it is not part of that +31-case GPU result. All suites use synthetic tensor operands, not training data. +A planner does not receive case names or histogram priors: only CPU sequence +boundaries, its configuration and explicit deployment coefficients. diff --git a/docs/THD_LAYOUT_PLANNER.md b/docs/THD_LAYOUT_PLANNER.md index bb46035c..00df1066 100644 --- a/docs/THD_LAYOUT_PLANNER.md +++ b/docs/THD_LAYOUT_PLANNER.md @@ -1,117 +1,197 @@ -# Cost-driven THD context-parallel layout planner - -## Why the alternating prototype is not the production policy - -Alternating the shard direction of adjacent long samples exactly balances the -original `20 x 1K + 2 x 514048` experiment, but it encodes properties of that -batch into the policy. It is weak for a single dominant sequence, an odd -number of long sequences, and power-law mixtures. - -`plan_thd_layout` instead consumes arbitrary CPU `global_cu_seqlens` and -balances two independent resources: - -- token-linear KDA and index-pooling work; -- position-dependent causal candidate work in CSA/DSA. - -Short samples remain whole when exact token capacity permits it. Long samples -are considered on several planning grids from `1 x CP` through `6 x CP`. Equal-token pieces are assigned by a -deterministic multi-resource largest-processing-time scheduler, then adjacent -pieces with the same owner are coalesced. This naturally produces an -early/late zigzag for one long sequence and cross-sample compensation when -several long sequences are available. - -After compute scheduling, a capacity-repair pass moves whole spans where -possible and otherwise splits the cheapest span boundary. Every rank receives -its exact balanced block capacity (at most one block difference when the global -block count is not divisible by CP), so the plan remains compatible with -fixed-shape KDA and pooled all-gather collectives. - -## Objective and fallback guarantee +# THD context-parallel planning + +cuLA separates **where selection runs** from **where recurrent state and +attention activations live**. `plan_thd_selection` balances causal indexer work +while KDA, pooling, attention forward/backward and indexer backward retain their +original contiguous CP shards. Only indexer Q/weights move; document-local +block IDs and FP32 scores return to the source query owners before attention. +The [redistributor](THD_REDISTRIBUTION.md) implements both directions and their +first-order autograd, without casting or reducing row values. + +This avoids applying full per-document zigzag to every short document or moving +wide MLA attention activations just to balance selection. It is an eager +integration; it does not promise CUDA Graph capture or whole-model E2E speedups. +See the [31-distribution measurements](THD_SELECTION_RESULTS.md) for scope, +preparation costs, precision, memory and measured limitations. + +## Public interfaces + +- `plan_thd_selection`: calibrated, capacity-bounded selection ownership; target + row counts can differ or be zero. Keep attention and KDA in source ownership. +- `zigzag_thd_layout`: conventional per-document early/late pairing, shared by + the planner and benchmark baseline. Rank r receives pieces r and 2*CP-1-r + from each document, including short documents. Empty pieces are omitted; + adjacent pieces remain distinct execution spans. +- `plan_thd_layout`: existing balanced-token policy, for consumers that need + exact balanced block capacities. Its default analytic objective is preserved. + This API alone does not make shuffled KDA spans executable by a contiguous + Kimi CP state chain. + +All planning uses CPU sequence boundaries and explicit caller configuration. +There is no device probing, online timing, hidden cache, process collective or +case-name lookup. Peers must use identical global metadata, coefficients and +configuration. Plan once at microbatch ingress and reuse where appropriate. +Planning is deterministic; kernel `deterministic=False` remains the throughput +option and is independent of the planner's stable tie-breaking. + +```python +from cula.distributed import plan_thd_selection, THDRedistributionPlan, THDRedistributor + +layout = plan_thd_selection( + global_cu_seqlens, + world_size=cp_world_size, + max_tokens_per_rank=query_capacity, + cost_model=calibrated_model, +) +route = THDRedistributionPlan.build(layout.ranks, source_offsets, cp_rank) +exchange = THDRedistributor(route, local_index_q.device, group=cp_group) +selection_q = exchange.distribute(local_index_q) +selection_weights = exchange.distribute(local_weights) +# Selection must use layout.for_rank(cp_rank)'s original document origins, +# causal offsets and document-local block numbering, with the required keys. +selected = select_for_spans(selection_q, selection_weights, layout.for_rank(cp_rank)) +local_ids = exchange.restore(selected.block_ids) +local_scores = exchange.restore(selected.scores) +``` -For rank `r`, let `T_r` be its tokens and `C_r` its exact analytic causal -candidate count. The default robust objective is +The model assumes balanced, block-aligned contiguous source shards. Pass those +same offsets to the redistributor. `query_capacity` must be block aligned and +fit the largest source shard. It bounds query-shaped buffers, **not** gathered +keys or backend scratch. The full-CSA benchmark uses twice the source row count; +callers must budget actual backend workspace separately. Documents must be +block aligned for these planners; the current CSA harness uses ratio four. + +## Candidate pool and fallback + +Selection compares all feasible candidates under one cost model: + +1. Contiguous source ownership. +2. Complete per-document zigzag, including documents below the split threshold. +3. Early/late pairs within long documents, relabeled onto ranks with a + minimum-cost assignment to reduce aggregate traffic. Short source spans stay + local in this candidate family. +4. Greedy long-span placement on grids from 1 through `max_chunks_per_rank` + pieces per rank. Descending modeled work is placed using current compute, + incoming/outgoing traffic and row capacity; a span splits only when no bin + can hold it. Adjacent same-owner spans coalesce. + +Only candidates 3 and 4 pin documents shorter than twice `min_split_tokens` +(default 16K). Full zigzag is independently considered and obeys the same row +cap. A zigzag that concentrates tiny documents beyond that cap is infeasible. +The minimum-cost assignment is a stable O(CP^3) primal-dual algorithm. It +minimizes aggregate traffic, so the result is still scored by the full +critical-path model rather than assumed to be best. + +Migration requires both `min_gain_fraction` (default 5%) and `min_gain_ms` +(default zero) over contiguous. `min_gain_ms` is an explicit minimum predicted +saving per selection use. A caller can price one-time preparation divided by +an expected reuse count; cuLA neither guesses that count nor hides preparation +in steady-state timing. `evaluated_scores` contains only materialized/scored +candidates. The selected modeled cost cannot exceed contiguous; this is **not** +a guarantee of minimum measured GPU latency over arbitrary distributions. + +## Compute and communication model + +For each rank, compute cost in nanoseconds is ```text -max(max(T_r) / mean(T_r), max(C_r) / mean(C_r)) - + 0.02 * max(cross_rank_boundary_count_r) +candidate_ns * exact_causal_candidates + + query_ns * query_rows + 1000 * span_us * execution_spans ``` -The boundary term prevents fine planning grids from winning on an immaterial -compute difference while introducing extra KDA state chains. The weights and -maximum grid are explicit configuration rather than hidden thresholds. - -The planner always evaluates the ordinary contiguous CP partition with the -same objective and returns the lowest-scoring candidate. Therefore its result -is never worse than contiguous under the declared model. This is a model-level -guarantee; GPU profiling is still needed to calibrate how candidate work, -token work, and boundary communication overlap on a target system. - -## Representative CP4 distributions - -The CPU benchmark is `benchmarks/large_cp_emulation/bench_thd_planner.py`. -The following table uses 1M total tokens and the default cost model. - -| THD distribution | selected grid | token max/avg | causal max/avg | selected score | contiguous score | remote boundaries | -|---|---:|---:|---:|---:|---:|---:| -| 1024 equal short samples | contiguous | 1.000 | 1.000 | 1.000 | 1.000 | 0 | -| one 1M sequence | 2 x CP | 1.000 | 1.000 | 1.080 | 1.790 | 6 | -| 20 x 1K + 2 x 514048 | 2 x CP | 1.000 | 1.000 | 1.040 | 1.560 | 4 | -| 256 x 1K + 1 x 768K | 3 x CP | 1.000 | 1.037 | 1.157 | 2.261 | 10 | -| 32 x 4K + 128K + 256K + 512K | 4 x CP | 1.000 | 1.046 | 1.166 | 2.302 | 10 | -| power-law 1K through 511K | 4 x CP | 1.000 | 1.028 | 1.188 | 2.149 | 15 | - -The important result is not that one grid wins everywhere. Low-fragmentation -`2 x CP` is selected for the original and single-long cases, while irregular -mixtures spend additional boundaries only when they materially improve both -token and causal balance. - -At logical CP64, the planning grid scales below the fixed 16K short-sample -threshold for long sequences. A one-million-token sequence therefore still -selects `2 x CP`, with exact token and causal balance. The original two-long -mixture selects `5 x CP` and reduces modeled causal max/average from 2.007 to -1.072; the power-law mixture also selects `5 x CP` and reduces it from 2.770 -to 1.034. The threshold is used only to preserve genuinely short samples, not -to prevent a dominant long sequence from forming sub-rank zigzag pieces. - -## Four-GPU DSA validation - -The layout-native DSA benchmark executes real deterministic cuDNN index -pooling and DSA selection, including pooled-key all-gather and global block-ID -remap. Each number below is the median critical-path GPU time after two warmups -and three measured iterations. All cases contain 1,048,576 global tokens. -These published measurements used four index heads; current benchmark output -records `index_heads` explicitly because the default model shape is now 64. - -| THD distribution | contiguous | planned | speedup | planned causal max/avg | -|---|---:|---:|---:|---:| -| one 1M sequence | 418.51 ms | 239.16 ms | 42.9% | 1.000 | -| 20 x 1K + 2 x 514048 | 166.77 ms | 109.05 ms | 34.6% | 1.000 | -| 256 x 1K + 1 x 768K | 291.80 ms | 134.35 ms | 54.0% | 1.037 | - -For the original mixture, planned per-rank selection time is approximately -108 ms and remap plus collective time is 0.55--0.96 ms. Under contiguous CP, -the light ranks spend 107--120 ms waiting at the collective while the heavy -ranks finish selection. For the single-long sequence, contiguous candidate -work grows from 8.59 to 60.13 billion across ranks; the planned layout gives -every rank exactly 34.36 billion candidates. This confirms that the speedup is -removal of causal load bubbles rather than a precision or kernel change. - -## Plan contract and integration boundary - -`THDLayoutPlan` records the global spans, owner rank, local `cu_seqlens`, -causal offsets, per-resource loads, and all candidate scores. Planning is -CPU-only and deterministic. It must run once at microbatch ingress, before the -model layer stack, rather than permuting tokens inside every KDA or CSA layer. - -The current KDA runtime still assumes one contiguous global token shard. A -separate lowering step is required before `THDLayoutPlan` becomes its default: - -1. derive every sequence's ordered `(sequence, span ordinal)` chain; -2. merge forward and backward KDA state in sequence order, not physical-rank - order; -3. produce CSA global block IDs and key-remap metadata from the same spans; -4. apply the token permutation once and retain it across all model layers. - -The benchmark adapter already lets DSA consume the planned layout. Keeping the -planner independent from KDA transport is intentional: layout policy and the -collective implementation can be validated and evolved separately. +The candidate count is an integer prefix-sum formula for the complete causal +blocks visible at each query. It includes original-document causal offsets, +not just each fragment's local length. Critical compute is the maximum over +ranks. + +Transport sums actual rows over source/destination edges. `bytes_per_query` +includes the outgoing Q/weights and returning IDs/scores in their real dtypes. +The critical transfer term takes the maximum of: + +- incoming and outgoing per-rank time at explicit intra/inter-node rates; +- the busiest aggregated peer edge, if `peer_gbps` is supplied; +- shared node injection/receive time, if `inter_node_aggregate_gbps` is supplied. + +Add the busiest rank's nonempty peer count times `peer_startup_us`, actual +permutation-copy work times `packing_ns_per_query`, and `exchange_us`. Multiple +fragments on one edge do not pay multiple peer startups. Naturally ordered +source/receive ranges skip `index_select` and carry zero copy cost on that side. +Global identity skips all transport terms. + +Ordered forward exchange, compute and inverse exchange costs add. The optional +`overlap_intra_inter` lets local and inter-node transfer resources overlap +inside collectives; it does not assume compute/communication overlap. Enable +it only after measuring mixed-link traffic. Coefficients are effective eager +implementation costs, not physical link specifications. The payload model is +shape-specific; different head counts, top-K, topology or software require +calibration and held-out validation. + +## Admissible pruning and preparation complexity + +Two lower bounds avoid building candidates that cannot meet the margin: + +- Every partition has the same token/candidate sums and at least one span per + document. Their mean is a compute lower bound. To reduce source rank i's + compute below C, some work must leave it; no moved row can remove more work + than its largest causal position allows. Optimistic outgoing bandwidth prices + these minimum rows. Minimize `C + max(outgoing_bound_i(C))` at the endpoints + and intersections of affine functions. With concurrent intra/inter links, + their capacities add in this optimistic bound. +- Full zigzag has a stronger bound: mean compute plus exact outgoing intra/inter + traffic. Consecutive pieces on the same node form interval runs. Intersecting + source spans with those runs and the owner's two pieces avoids constructing + O(documents * CP) objects. This bound omits incoming and peer bottlenecks and + permutation copies. Local candidates run first to tighten its incumbent. + +Bounds are tested against exhaustive small assignments and concrete zigzag +layouts on contiguous, interleaved and irregular topologies. They cannot exclude +a strictly better modeled zigzag; they make no statement about model error. + +Repeated planning grids are deduplicated. Pinned spans are coalesced once; +owner groups are built in one pass, and diagnostics are computed only for the +selected selection plan. Balanced-token scheduling uses a rank heap for +identical-cost pieces and indexes capacity repair by donor and piece shape. +Source intersections use binary search. DSA window construction maintains an +integer candidate total instead of rescanning all preceding short documents; +window boundaries, scratch requirements and numerical execution are unchanged. + +## Calibrating and measuring + +`benchmarks/calibrate_thd_selection.py` fits nonnegative candidate/query/span +costs from unmodeled selection runs and retains source JSON digests. Pipeline +exchange measurements use differences of maximum cumulative phase endpoints, +so KDA/selection arrival waits are not counted again as network work. + +`benchmarks/bench_thd_transport.py` measures the actual Q/weight and inverse +ID/score exchanges with exact marker checks. Sparse peer pairs, node-local, +inter-node and dense exchanges exercise different message sizes. The 32768-row +and striped fixtures are held out. `--validation-only` measures additional +mixed-link fractions. `benchmarks/fit_thd_transport.py` fits nonnegative resource +costs without using held-out rows in its objective. Ordered fixtures do not +identify copy bandwidth: measure permutation copies separately and supply that +coefficient in the input profile. Unidentifiable node limits must not be +interpreted as measured physical bandwidth. + +The [example profiles](../benchmarks/profiles/README.md) are optional deployment +inputs, not library defaults. The runner's `--suite validation` fixes the 31 +measured THD shapes. Use `summarize_csa_layout_matrix.py` to check complete +output/gradient evidence, compatible configurations, all-rank samples and +preparation accounting before producing a comparison. Preparation includes +common metadata, layout/index/workspace construction and KDA transport buffers. +Report it separately and added to one iteration; amortize only when reuse is real. + +## KDA integration + +The [Kimi CP runtime](KIMI_CP_RUNTIME.md) requires contiguous source ownership. +When every internal shard boundary is a document boundary, all ranks agree that +no recurrence crosses ranks. Native local KDA/autograd then skips summaries and +boundary collectives, with unchanged operands, dtypes and batch-invariance. +One isolated rank cannot bypass exchanges required by other ranks. Manually +constructed plans conservatively default to requiring exchange. + +On the native summary path, backward context is reused only if the summarized +first span covers the complete local shard. A prefix-only context does not +contain the full THD shard's per-token intermediates; full apply prepares its +own intermediates instead. The paired independent-shard benchmark checks the +fast path against forced boundary execution, including weak decay and a +crossing-document control. diff --git a/docs/THD_REDISTRIBUTION.md b/docs/THD_REDISTRIBUTION.md new file mode 100644 index 00000000..ecce6f86 --- /dev/null +++ b/docs/THD_REDISTRIBUTION.md @@ -0,0 +1,99 @@ +# Executing and reversing a THD query layout + +`plan_thd_layout` and `plan_thd_selection` choose global token spans. `THDRedistributionPlan` and +`THDRedistributor`, exported from `cula.distributed`, execute that assignment +from existing contiguous CP shards and restore the original ownership/order. +The planner's modeled score is not an end-to-end latency guarantee; see the +[real CP16/CP8 measurements](THD_SELECTION_RESULTS.md). + +## Contract + +Each process-group rank starts with rows in +`[source_offsets[rank], source_offsets[rank + 1])`. Offsets start at zero and +are nondecreasing; source and target row counts need not be equal. The target +order is the concatenation of that rank's `THDRankLayout.spans`, in the supplied +order. A plan validates full coverage, span ownership, sequence containment, +and absence of gaps/overlap. Source offsets and layouts must be identical on +every peer; rank numbers are **process-group ranks**, not global ranks. + +Preparation builds CPU interval metadata and device row permutations once. +Execution packs rows as necessary, performs a real variable-split +`all_to_all_single`, and unpacks into target span order. `restore` reverses both +the communication splits and the permutations. There is no cast, reduction, +padding, or replicated full-tensor gather in this transport. An identity +layout skips the collective only when **all ranks** are unchanged, and returns +the input tensor itself. Callers must account for this alias before modifying +a returned tensor in place. + +All peers must call the exchanges in the same order with matching trailing +shapes and dtypes. Input tensors must have contiguous rows on the explicitly +selected device; `"cuda"` resolves to the current device during preparation. +An explicit device such as `torch.device('cuda', local_rank)` is also supported. +The caller owns any necessary conversion to contiguous storage. The process +group must outlive every backward using the redistributor. Backward participation +and collective order must also agree across ranks; rank-local conditional losses +must not cause peers to skip a required inverse exchange. + +Prepare on the execution stream, or establish an explicit stream dependency +before using the device indices. Keep collective ordering consistent across +peers. Permutation indices are read-only after preparation; each nonidentity +exchange allocates its own packed/received outputs rather than overwriting +transport storage retained by an earlier autograd invocation. + +## Integration + +```python +from cula.distributed import ( + THDRedistributionPlan, + THDRedistributor, + plan_thd_layout, +) + +# global_cu_seqlens and source_offsets are the same CPU metadata on all peers. +layout = plan_thd_layout(global_cu_seqlens, world_size=cp_world_size) +route = THDRedistributionPlan.build(layout.ranks, source_offsets, cp_rank) +exchange = THDRedistributor(route, local_q.device, group=cp_group) + +target_q = exchange.distribute(local_q) +# The attention call must use this rank's span metadata, original document +# origins, and causal offsets. KV ownership is a separate contract. +target_output = attention_for_spans(target_q, layout.for_rank(cp_rank)) +local_output = exchange.restore(target_output) +``` + +`distribute` backward invokes the inverse exchange; `restore` backward invokes +the forward exchange. First-order autograd is supported; higher-order gradients +are explicitly disabled. The full CSA benchmark also exercises the explicit +inverse APIs because native backward wrappers return gradient tensors directly. + +This does not turn arbitrary attention into distributed attention: global KV +visibility, document isolation, causal masking, KV-gradient reduction and +replicated-parameter reduction remain the integration's responsibility. In +particular, do not pass a shuffled set of recurrent KDA spans into a contiguous +Kimi CP boundary plan. The CP16 experiment keeps KDA and KV/pooling ownership +contiguous, redistributes CSA query-side operands, and restores CSA outputs and +query-side gradients at the layer boundary. + +For the selection-only strategy, use `plan_thd_selection` with calibrated costs +and an explicit row capacity. Route indexer Q/weights to its spans and inverse-route +the selected IDs/scores, then run attention in original source ownership. The +[planner contract](THD_LAYOUT_PLANNER.md#capacity-bounded-selection-planner) +describes this policy and its preparation/reuse budget. + +## Execution and validation scope + +This is an eager PyTorch/NCCL implementation with output/packing allocations. +It does not promise CUDA Graph capture, reusable tensor workspaces, asynchronous +overlap, or a topology-aware communication schedule. Tensor lifetimes can be +material at MLA widths and should be included in memory measurements. + +CPU protocol tests cover unequal counts, reversed target span order, sender +packing, receiver unpacking, inverse counts and identity skipping. Real CP16 +tests on two H20 nodes verify token/channel markers and both autograd directions +for all three layouts, including all 31 measured 1M distributions. Full-tensor CSA +output/gradient comparisons are in the linked report. The CSA harness requires +ratio-4-aligned document/source boundaries. In its default selection scope, +empty selection ranks skip native plan construction and selection kernels but +still participate in all exchanges. Attention stays in nonempty source shards. +The optional query-wide scope still requires nonempty attention query ranks; +these tests do not establish support for unaligned document boundaries. diff --git a/docs/THD_SELECTION_RESULTS.md b/docs/THD_SELECTION_RESULTS.md new file mode 100644 index 00000000..7096e18e --- /dev/null +++ b/docs/THD_SELECTION_RESULTS.md @@ -0,0 +1,163 @@ +# Real-CP THD selection results (2026-11-15) + +This is a **1 KDA + 1 complete CSA operator-chain** forward/backward study. +KDA and CSA use independent synthetic inputs; projections, residuals, MLP, +optimizer and the full training model are excluded. It is not a seven-KDA/one-CSA +model measurement. The 31 shapes contain exactly 1,048,576 tokens each. + +## Setup and accounting + +- H20-3e: two nodes, eight GPUs per node, real CP16. B300: one node, real CP8. +- cuDNN Frontend commit `855ccd77ba9efed22441fd454f7cc0206a068e94`, backend 9.20.0.48, CuTeDSL 4.8.1. +- KDA: H32, K=V=128, batch invariant enabled; CSA MLA-absorb: H64, D576, V512; indexer: H64/D128, Top-K512, ratio-4 pooling. +- Non-deterministic selection/pooling/sparse backward. The transport never changes dtypes; pool gradients retain the per-loss-rank ordering before FP32 accumulation. +- Three warmups and ten timed iterations per policy/shape. Time is the median of each iteration's maximum-rank duration. +- Timed work includes all runtime layouts and activation/gradient communication, pooling, Top-K, sparse attention, detached teacher/KL and all backwards. +- Preparation separately includes common metadata, planning, index/workspace materialization and KDA transport allocation. Fresh-layout cost adds this **once**, without amortization. RNG generation, JIT warmup and numerical oracles are excluded. +- Memory is peak allocated/reserved for the entire measured chain, **including inputs, outputs and gradients**, not workspace-only memory. + +## Results + +C = contiguous; Z = complete per-document zigzag; P = calibrated selection planner. +All table times are milliseconds. The machine-readable [results](results/thd_selection_20261115.json) +include per-policy preparation, memory, errors and original record hashes. + +| Case | Docs | H20 C | H20 Z | H20 P | B300 C | B300 Z | B300 P | +|---|---:|---:|---:|---:|---:|---:|---:| +| 20s2l | 22 | 1770.25 | 1361.15 | 1326.95 | 606.72 | 525.02 | 523.09 | +| count-mixed-1000-1long | 1000 | 1738.30 | 1107.86 | 1078.85 | 564.92 | 480.15 | 466.27 | +| count-mixed-256-1long | 256 | 1933.60 | 1223.23 | 1197.76 | 646.42 | 509.91 | 494.01 | +| count-mixed-512-2long | 512 | 1235.56 | 942.93 | 932.91 | 441.96 | 421.93 | 407.34 | +| count-mixed-512-2long-ascending | 512 | 1240.10 | 947.32 | 938.54 | 466.90 | 434.59 | 421.41 | +| count-mixed-512-2long-descending | 512 | 1241.66 | 936.86 | 933.32 | 466.69 | 430.64 | 412.11 | +| count-mixed-512-4long | 512 | 972.04 | 880.24 | 874.13 | 338.36 | 338.60 | 338.46 | +| count-near-uniform-512 | 512 | 452.80 | 489.87 | 452.81 | 174.68 | 185.55 | 174.55 | +| count-tiny-mix-1000 | 1000 | 804.45 | 819.81 | 803.92 | 277.50 | 288.30 | 277.98 | +| count-tiny-mix-512 | 512 | 685.93 | 716.08 | 685.85 | 252.41 | 260.91 | 252.61 | +| count-uniform-1024 | 1024 | 304.09 | 345.73 | 303.89 | 119.41 | 133.69 | 119.48 | +| count-uniform-256 | 256 | 549.62 | 584.04 | 549.45 | 213.00 | 220.55 | 213.09 | +| jittered-768-s14 | 768 | 358.52 | 397.64 | 358.48 | 139.60 | 152.69 | 139.54 | +| jittered-768-s15 | 768 | 358.78 | 398.11 | 358.70 | 139.20 | 152.08 | 138.97 | +| jittered-768-s16 | 768 | 359.07 | 398.91 | 359.14 | 139.43 | 152.10 | 139.20 | +| lognormal-1000-s14 | 1000 | 685.00 | 700.99 | 684.54 | 242.74 | 255.30 | 242.80 | +| lognormal-1000-s15 | 1000 | 763.70 | 760.71 | 763.37 | 238.08 | 248.49 | 238.69 | +| lognormal-1000-s16 | 1000 | 736.00 | 735.10 | 736.39 | 243.51 | 255.28 | 244.25 | +| lognormal-257-s14 | 257 | 666.27 | 694.94 | 666.14 | 241.62 | 249.37 | 241.79 | +| lognormal-257-s15 | 257 | 786.76 | 774.97 | 786.91 | 265.90 | 270.33 | 266.58 | +| lognormal-257-s16 | 257 | 792.44 | 789.95 | 792.44 | 260.86 | 265.92 | 261.18 | +| tiny-tail-768-s14 | 768 | 1157.98 | 905.56 | 887.67 | 436.30 | 406.08 | 397.62 | +| tiny-tail-768-s15 | 768 | 1348.06 | 970.18 | 959.86 | 498.77 | 423.77 | 416.50 | +| tiny-tail-768-s16 | 768 | 1590.68 | 1054.16 | 1043.18 | 559.72 | 466.85 | 460.22 | +| trimodal-513-s14 | 513 | 1590.94 | 1143.29 | 1123.55 | 528.07 | 482.18 | 471.44 | +| trimodal-513-s15 | 513 | 1565.77 | 1154.85 | 1138.55 | 550.40 | 479.26 | 475.16 | +| trimodal-513-s16 | 513 | 1917.07 | 1225.09 | 1195.03 | 606.03 | 509.75 | 498.81 | +| near-uniform | 64 | 657.86 | 691.02 | 657.87 | 261.57 | 266.84 | 261.59 | +| power-law | 7 | 1788.44 | 1200.61 | 1195.98 | 612.98 | 493.15 | 492.85 | +| single | 1 | 2908.84 | 1934.28 | 1926.76 | 879.36 | 667.50 | 664.84 | +| uniform | 64 | 653.51 | 690.32 | 657.16 | 261.74 | 267.41 | 261.72 | + +| Metric | H20 CP16 | B300 CP8 | +|---|---:|---:| +| Largest P slowdown vs faster C/Z, steady | 1.541% | 0.304% | +| Largest P slowdown vs faster C/Z, prepare + one iteration | 4.038% | 5.188% | +| Best C/P speedup | 1.614x | 1.323x | +| Mean P preparation | 34.118 ms | 18.972 ms | +| Mean extra P vs C preparation / mean P steady time | 3.038% | 2.518% | +| All P preparation / all P steady time | 4.011% | 5.647% | +| contiguous max allocated GiB | 44.91 | 72.95 | +| zigzag max allocated GiB | 46.20 | 73.43 | +| planner max allocated GiB | 46.02 | 75.91 | + +These results do **not** show universal optimality or a strict <3% preparation +guarantee. The complete planner preparation is larger than its net incremental +cost over contiguous, and the fresh-layout metric is more conservative than +steady-state throughput. Preparation was sampled once per case, so host jitter +is visible. Profile margins are explicit: 5% plus 40 ms on H20 and 20 ms on B300. + +## Precision and calibration + +All 186 policy/shape/platform results passed finite-value checks. Forward, inverse +and both autograd row-marker checks were bitwise exact. Every policy used the +same source input fingerprints and compared all 14 output/gradient tensors +against a lossless CPU copy of contiguous results. KDA output and all five KDA +gradients had zero difference. Worst CSA relative RMS was 0.145868% on H20 and +0.161238% on B300; maximum auxiliary KL absolute differences were 3.2014e-10 +and 1.7462e-10. These are auxiliary indexer losses, not training loss results. + +Four additional small FP32 attention/teacher/KL/pooling-gradient fixtures per +platform were tested under all three policies. The reference uses the selected +key set and checks causality, document isolation, uniqueness and tail coverage; +it does not independently prove the selector's discrete ranking. + +Compute coefficients used seven calibration shapes and their C/Z runs. +The other 24 matrix cases were not used for compute fitting. Independent +communication sweeps held out sizes/topologies: absolute percentage error +averaged 7.14% on H20 and 10.53% on B300. Four additional mixed-link H20 +checks had 2.01--3.73% error. These are effective-cost models, not physical +network simulators. No final E2E result is used as a case-specific lookup. + +## Reproduction + +Install the compatible cuDNN build and cuLA's `cudnn-hybrid` extra first. Under +CUDA 13, preserve the `[cu13]` extra when pinning `nvidia-cutlass-dsl`. Use the +appropriate NCCL/rendezvous configuration for the machine; rank-to-node mapping +must match the explicit profile. For the single-node B300 configuration: + +```bash +OMP_NUM_THREADS=4 torchrun --standalone --nproc-per-node=8 -m benchmarks.bench_csa_layout_matrix \ + --suite validation --heads 64 --warmups 3 --repeats 10 \ + --planner-strategy selection \ + --planner-cost-model benchmarks/profiles/thd_selection_b300_cp8.json \ + --output-dir results/b300-cp8 +python -m benchmarks.summarize_csa_layout_matrix results/b300-cp8 --output results/b300-summary.json +``` + +For H20, launch on two eight-GPU nodes with `--nnodes=2`, distinct node ranks +and shared rendezvous; select `thd_selection_h20_cp16.json`. The runner first +executes small FP32 checks, then all 31 cases, keeping contiguous first and +alternating the other two policies. `--list-cases` lists shapes without CUDA. + +## Final source review + +The full matrix above precedes the final CPU metadata cleanup. This cleanup +rejects noninteger transport intervals, removes an obsolete benchmark import, +fixes preparation accounting in the public summarizer and replaces a quadratic +DSA window candidate sum with an integer running total. Planner decisions and +numerical kernels are unchanged. The final branch has separate regression +validation recorded below; the historical matrix is not relabeled as a new run. + +Local CPU preparation microbenchmark (10 measured samples after one warmup): + +| Documents | Old window preparation ms | New ms | Metadata | +|---|---:|---:|---| +| 22 | 0.087 | 0.083 | field-for-field equal | +| 1000 | 11.527 | 2.693 | field-for-field equal | +| 4096 | 153.647 | 7.648 | field-for-field equal | +| 16384 | 2404.650 | 25.638 | field-for-field equal | + +These CPU-only measurements are not GPU or whole-chain speedups. The change +removes repeated scans of already-grouped documents; it does not change scratch +capacity, math precision or kernel launch boundaries. +## Final-branch GPU regression + +On both H20 CP16 and B300 CP8, the final source passed four small FP32 fixtures +and three representative 1M shapes (single long, 1000 documents with one long, +and 1000-document tiny mixture), each under C/Z/P: **42 complete CSA rows**. +The 1M rerun used three warmups and five measurements. A separate six-case KDA +A/B on each platform compared native document-local execution with the forced +boundary route, including weak decay and isolated ranks with a crossing peer: +**24 A/B rows**, with zero difference in output and all five gradients. + +All 181 tested source/config files match the submitted branch. The 62 original +planner decisions still match every measured rank's policy, row count, span +count and remote rows. All **2,232** DSA rank plans across the full 31 x 3 x +(CP16 + CP8) matrix have field-for-field identical window metadata before and +after the integer running-total optimization. The [regression record](results/thd_selection_review_20261115.json) +includes source hashes, correctness maxima and per-case timings. + +337 focused CPU tests and both repository pre-commit hooks passed. Review +covered the complete proposed diff and its distributed/hybrid call paths: +coverage and ownership, collective participation, inverse autograd, storage +lifetimes, context reuse, cost-model bounds and benchmark accounting. No +handwritten GPU kernel, precision mode or math approximation is changed. +This is not a claim of a complete race audit of every unchanged kernel in cuLA. diff --git a/docs/results/thd_selection_20261115.json b/docs/results/thd_selection_20261115.json new file mode 100644 index 00000000..3de510ea --- /dev/null +++ b/docs/results/thd_selection_20261115.json @@ -0,0 +1,6544 @@ +{ + "schema_version": 1, + "date": "2026-11-15", + "cudnn_frontend_commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "cudnn_backend": "9.20.0.48", + "cutlass_dsl": "4.8.1", + "protocol": "3 warmups, 10 measured iterations; median of per-iteration maximum rank time; full prepare counted once", + "scope": "1 KDA + 1 full CSA forward/backward, independent inputs; not a connected model", + "platforms": { + "h20_cp16": { + "world_size": 16, + "cases": { + "1m-20s2l": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1770.2546997070312, + "min_ms": 1766.39697265625, + "max_ms": 1779.2506103515625, + "allocated_gib": 40.48734903335571, + "reserved_gib": 45.47265625, + "prepare_ms": 9.66643076390028, + "layout_prepare_ms": 9.303988888859749, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.999981328845024, + "max_spans": 21, + "raw_file": "layout-1m-20s2l-contiguous.json", + "raw_sha256": "c27f0de10476d5a5227da3f8eb46c815480ae913c329c852efa1f8b8b418602f" + }, + "planner": { + "total_ms": 1326.9475708007812, + "min_ms": 1323.5238037109375, + "max_ms": 1330.18505859375, + "allocated_gib": 40.48831605911255, + "reserved_gib": 45.951171875, + "prepare_ms": 56.893481872975826, + "layout_prepare_ms": 56.678880006074905, + "policy": "selection-local-4x", + "relative_rms": { + "output": 0.0014370549510826182, + "dq": 0.0007328903431475611, + "dkv": 0.0007098334972614795, + "d_index_q": 0.0007886133158625485, + "d_weights": 0.0007656313854345311, + "d_pool_kv": 0.0009375874547660811, + "d_pool_score": 0.0009411841698845073, + "d_ape": 1.331346387614e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.9998921006917953, + "max_spans": 24, + "raw_file": "layout-1m-20s2l-planner.json", + "raw_sha256": "65c946c6186791db5b35e4c309bebf479114decc0c7ce35e7065967232d63621" + }, + "zigzag": { + "total_ms": 1361.1481323242188, + "min_ms": 1357.9793701171875, + "max_ms": 1367.4815673828125, + "allocated_gib": 40.49222230911255, + "reserved_gib": 45.970703125, + "prepare_ms": 10.714996606111526, + "layout_prepare_ms": 10.436772368848324, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014426966180279143, + "dq": 0.000736235596560469, + "dkv": 0.000665001758953214, + "d_index_q": 0.0007290876335645271, + "d_weights": 0.0007622767518445443, + "d_pool_kv": 0.0009443198974116189, + "d_pool_score": 0.0009462030090884896, + "d_ape": 1.3866521682671272e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 8.731149137020111e-11, + "workspace_gib": 1.9999244809150696, + "max_spans": 44, + "raw_file": "layout-1m-20s2l-zigzag.json", + "raw_sha256": "0baff442ce3f471a823476dc4863c99b60004430a19c311cb608e5e7f7e0847e" + } + }, + "speedup_vs_contiguous": 1.3340803650883695, + "gap_to_best_baseline_pct": -2.5126259744439827, + "prepare_plus_iteration_ms": { + "contiguous": 1779.9211304709315, + "planner": 1383.841052673757, + "zigzag": 1371.8631289303303 + }, + "prepare_inclusive_gap_pct": 0.8731136139482176, + "documents": 22, + "lengths_sha256": "1fbc78fcb26714baab3313b04d1a82e7d4c3c5f2a707fddfa67b2c0893379098" + }, + "1m-count-mixed-1000-1long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1738.2982177734375, + "min_ms": 1732.321044921875, + "max_ms": 1749.4837646484375, + "allocated_gib": 40.49317121505737, + "reserved_gib": 45.59375, + "prepare_ms": 10.79770177602768, + "layout_prepare_ms": 10.190024971961975, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999728836119175, + "max_spans": 135, + "raw_file": "layout-1m-count-mixed-1000-1long-contiguous.json", + "raw_sha256": "77a6c060f06eafec547a93790326e091c6aa01ede84b38849cfe169eb8fbf692" + }, + "planner": { + "total_ms": 1078.8523559570312, + "min_ms": 1075.45654296875, + "max_ms": 1081.654541015625, + "allocated_gib": 41.948423862457275, + "reserved_gib": 45.96875, + "prepare_ms": 57.30481201317161, + "layout_prepare_ms": 56.81393697159365, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014131674788891825, + "dq": 0.0007217496110718596, + "dkv": 0.0006838480550948861, + "d_index_q": 0.0007880740154021596, + "d_weights": 0.0008131424597783085, + "d_pool_kv": 0.0009271833921768376, + "d_pool_score": 0.0009146511551639808, + "d_ape": 3.901471804680105e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.999987006187439, + "max_spans": 137, + "raw_file": "layout-1m-count-mixed-1000-1long-planner.json", + "raw_sha256": "5041bcf57be4ba051f162f08ea08577de37d5eaad5833c4dfde3fb4db49149a7" + }, + "zigzag": { + "total_ms": 1107.8590698242188, + "min_ms": 1105.778564453125, + "max_ms": 1109.240234375, + "allocated_gib": 40.492013454437256, + "reserved_gib": 45.97265625, + "prepare_ms": 73.95509001798928, + "layout_prepare_ms": 73.50210502045229, + "policy": "zigzag", + "relative_rms": { + "output": 0.001396905567037877, + "dq": 0.0007126076884560058, + "dkv": 0.000898063044848878, + "d_index_q": 0.000842882708912927, + "d_weights": 0.0008584308868511431, + "d_pool_kv": 0.0009289185753135385, + "d_pool_score": 0.0009154616825916759, + "d_ape": 4.003750630157111e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.999989103525877, + "max_spans": 1200, + "raw_file": "layout-1m-count-mixed-1000-1long-zigzag.json", + "raw_sha256": "2dc533ddba4e51392912145142e4bd7781c883022f478efebf1d799a8faf6f34" + } + }, + "speedup_vs_contiguous": 1.611247552248633, + "gap_to_best_baseline_pct": -2.6182674906285675, + "prepare_plus_iteration_ms": { + "contiguous": 1749.0959195494652, + "planner": 1136.1571679702029, + "zigzag": 1181.814159842208 + }, + "prepare_inclusive_gap_pct": -3.863297075244143, + "documents": 1000, + "lengths_sha256": "9e67a0b0cf2c6f62c886060fcce16cad37d396e458fe8e31e2ea92de7569371d" + }, + "1m-count-mixed-256-1long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1933.6011962890625, + "min_ms": 1923.75244140625, + "max_ms": 1944.3497314453125, + "allocated_gib": 41.22977256774902, + "reserved_gib": 46.46875, + "prepare_ms": 5.177393555641174, + "layout_prepare_ms": 4.873349331319332, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.4999958872795105, + "max_spans": 52, + "raw_file": "layout-1m-count-mixed-256-1long-contiguous.json", + "raw_sha256": "e220a459b7f099102bcbfc2c44d7340a46e640a12a1032c04384e3c9223c7d20" + }, + "planner": { + "total_ms": 1197.7588500976562, + "min_ms": 1195.3802490234375, + "max_ms": 1202.0479736328125, + "allocated_gib": 41.97810459136963, + "reserved_gib": 48.919921875, + "prepare_ms": 52.76484042406082, + "layout_prepare_ms": 52.50781960785389, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014226647261483673, + "dq": 0.0007180340724526626, + "dkv": 0.0008179497186912406, + "d_index_q": 0.0008058696030484662, + "d_weights": 0.0008432608681242646, + "d_pool_kv": 0.0010930844599177977, + "d_pool_score": 0.0010590800398992991, + "d_ape": 7.770387411027729e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 4.999912668019533, + "max_spans": 53, + "raw_file": "layout-1m-count-mixed-256-1long-planner.json", + "raw_sha256": "95643a4a8536f00a772f49252efdd308acb51c87cb2c242a9297e005537d8ef6" + }, + "zigzag": { + "total_ms": 1223.2321166992188, + "min_ms": 1221.118408203125, + "max_ms": 1227.0054931640625, + "allocated_gib": 40.99202299118042, + "reserved_gib": 46.470703125, + "prepare_ms": 22.380927577614784, + "layout_prepare_ms": 22.075112909078598, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013963022789706349, + "dq": 0.0007049773454525711, + "dkv": 0.0007368037351704907, + "d_index_q": 0.0007336936166034252, + "d_weights": 0.0007946033580117142, + "d_pool_kv": 0.0009276963136814727, + "d_pool_score": 0.0009221220113713877, + "d_ape": 7.63173704818653e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 2.499992474913597, + "max_spans": 307, + "raw_file": "layout-1m-count-mixed-256-1long-zigzag.json", + "raw_sha256": "10922039f1b3ddb1b55635eb368d7ebfc2fc730d4cf0feb151c809d59e0f8543" + } + }, + "speedup_vs_contiguous": 1.6143493292755977, + "gap_to_best_baseline_pct": -2.0824556724605814, + "prepare_plus_iteration_ms": { + "contiguous": 1938.7785898447037, + "planner": 1250.523690521717, + "zigzag": 1245.6130442768335 + }, + "prepare_inclusive_gap_pct": 0.3942352938134608, + "documents": 256, + "lengths_sha256": "013b49129cf177ac7e785bf7126e9e6899c7405866ddd6a51c91d4aec66cc87e" + }, + "1m-count-mixed-512-2long-ascending": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1240.0969848632812, + "min_ms": 1234.22412109375, + "max_ms": 1242.7333984375, + "allocated_gib": 40.953940868377686, + "reserved_gib": 44.96875, + "prepare_ms": 11.49603258818388, + "layout_prepare_ms": 10.997926816344261, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999969005584717, + "max_spans": 418, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-contiguous.json", + "raw_sha256": "191f9c9e6f96e2ccf3acd63dfa1bc38bf1bf31f9e710a83fe1d84279e946e8b0" + }, + "planner": { + "total_ms": 938.5447387695312, + "min_ms": 937.6407470703125, + "max_ms": 941.2828979492188, + "allocated_gib": 42.990283489227295, + "reserved_gib": 46.3046875, + "prepare_ms": 46.396294608712196, + "layout_prepare_ms": 45.94092257320881, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014403781516027449, + "dq": 0.0007327477018683716, + "dkv": 0.0007074783258196851, + "d_index_q": 0.0007088233181552034, + "d_weights": 0.0007418597362617036, + "d_pool_kv": 0.0008261977568685192, + "d_pool_score": 0.0008087736658695805, + "d_ape": 6.546711824353377e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999847412109375, + "max_spans": 422, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-planner.json", + "raw_sha256": "4363781658c77cc3b9a7e42a822c60108c59e74f35d4925f6be61fd1b7927d7c" + }, + "zigzag": { + "total_ms": 947.3248901367188, + "min_ms": 946.660888671875, + "max_ms": 949.937255859375, + "allocated_gib": 41.866726875305176, + "reserved_gib": 50.775390625, + "prepare_ms": 58.15050844103098, + "layout_prepare_ms": 57.70234391093254, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014537235383189086, + "dq": 0.0007407989394821003, + "dkv": 0.0007096909729663449, + "d_index_q": 0.0008041080050001198, + "d_weights": 0.0007943771738968879, + "d_pool_kv": 0.0008371011910827963, + "d_pool_score": 0.0008341807726690167, + "d_ape": 6.293301240637564e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999847412109375, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-zigzag.json", + "raw_sha256": "3d28b4576e300c475c485aec206bf644e380d2a754c1cc239669c5e711197243" + } + }, + "speedup_vs_contiguous": 1.3212976788821988, + "gap_to_best_baseline_pct": -0.926836342906634, + "prepare_plus_iteration_ms": { + "contiguous": 1251.5930174514651, + "planner": 984.9410333782434, + "zigzag": 1005.4753985777497 + }, + "prepare_inclusive_gap_pct": -2.042254363314333, + "documents": 512, + "lengths_sha256": "c8ef7a47c8782c941def4feb26285b36f3ad51a989a3c052ec4e3fd7e8677f0e" + }, + "1m-count-mixed-512-2long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1235.557861328125, + "min_ms": 1233.1134033203125, + "max_ms": 1238.40869140625, + "allocated_gib": 39.73763704299927, + "reserved_gib": 44.96875, + "prepare_ms": 6.890648975968361, + "layout_prepare_ms": 6.548076868057251, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999978207051754, + "max_spans": 86, + "raw_file": "layout-1m-count-mixed-512-2long-contiguous.json", + "raw_sha256": "c5be0e549e5a89c44368a0b4a6ad8b7a56d573be301683298146b145455eea1c" + }, + "planner": { + "total_ms": 932.9089050292969, + "min_ms": 932.2576293945312, + "max_ms": 933.833984375, + "allocated_gib": 39.72488594055176, + "reserved_gib": 46.2265625, + "prepare_ms": 43.387351965066046, + "layout_prepare_ms": 43.043665995355695, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.001390238819945953, + "dq": 0.000701639014897124, + "dkv": 0.0006863240111215172, + "d_index_q": 0.000673929731405553, + "d_weights": 0.0007422031462348568, + "d_pool_kv": 0.000996414348174742, + "d_pool_score": 0.0009960602473099074, + "d_ape": 6.394037327728149e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999865889549255, + "max_spans": 90, + "raw_file": "layout-1m-count-mixed-512-2long-planner.json", + "raw_sha256": "f0a97969490f8e8aeabdcd6190e37924ae901254b2fa1f36786ca10bae0367c7" + }, + "zigzag": { + "total_ms": 942.929443359375, + "min_ms": 942.33837890625, + "max_ms": 944.03173828125, + "allocated_gib": 40.74278402328491, + "reserved_gib": 46.474609375, + "prepare_ms": 40.236313827335835, + "layout_prepare_ms": 39.91378378123045, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013902129191113775, + "dq": 0.0007078508527508515, + "dkv": 0.0007465641023219604, + "d_index_q": 0.0007212337867090088, + "d_weights": 0.0007829665474866085, + "d_pool_kv": 0.0010004758288360228, + "d_pool_score": 0.0009971238645788182, + "d_ape": 6.249562389713757e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999849684536457, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-2long-zigzag.json", + "raw_sha256": "333c20f643fc915448950bc524334b6701b9850dbd17fd0212bf99183a9c9804" + } + }, + "speedup_vs_contiguous": 1.3244142645302799, + "gap_to_best_baseline_pct": -1.062702877786692, + "prepare_plus_iteration_ms": { + "contiguous": 1242.4485103040934, + "planner": 976.2962569943629, + "zigzag": 983.1657571867108 + }, + "prepare_inclusive_gap_pct": -0.698712311950811, + "documents": 512, + "lengths_sha256": "21794dd7586d487ed826710eb069c8ec277ff51b5c88db0b0400e835d0419bcd" + }, + "1m-count-mixed-512-2long-descending": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1241.6589965820312, + "min_ms": 1240.9886474609375, + "max_ms": 1247.15087890625, + "allocated_gib": 41.22573900222778, + "reserved_gib": 45.48828125, + "prepare_ms": 12.419818027410656, + "layout_prepare_ms": 11.910784989595413, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999969005584717, + "max_spans": 418, + "raw_file": "layout-1m-count-mixed-512-2long-descending-contiguous.json", + "raw_sha256": "c1bddd9fac6471e12def2d7dc44740ce5235469ee0f34738a9eaf5ca5234372b" + }, + "planner": { + "total_ms": 933.3216857910156, + "min_ms": 932.8864135742188, + "max_ms": 933.894775390625, + "allocated_gib": 43.259296894073486, + "reserved_gib": 51.81640625, + "prepare_ms": 45.23335298290476, + "layout_prepare_ms": 44.762280012946576, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014545159003805394, + "dq": 0.000743494012154302, + "dkv": 0.0007685056085245936, + "d_index_q": 0.0008703250133279459, + "d_weights": 0.000838913170206235, + "d_pool_kv": 0.0008371936081197537, + "d_pool_score": 0.000825633756631383, + "d_ape": 6.513117921279345e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 1.9999847412109375, + "max_spans": 420, + "raw_file": "layout-1m-count-mixed-512-2long-descending-planner.json", + "raw_sha256": "a948e5b646919a94e00f9e31f6a8dd38cf4aa1b30d734c9d86efa949992ed131" + }, + "zigzag": { + "total_ms": 936.8607788085938, + "min_ms": 936.211669921875, + "max_ms": 938.4207763671875, + "allocated_gib": 43.137155055999756, + "reserved_gib": 51.8203125, + "prepare_ms": 58.16895398311317, + "layout_prepare_ms": 57.69163102377206, + "policy": "zigzag", + "relative_rms": { + "output": 0.001422822688413853, + "dq": 0.0007246148440246227, + "dkv": 0.0007254313535724738, + "d_index_q": 0.0007736511212151291, + "d_weights": 0.0007601264812653758, + "d_pool_kv": 0.0008365214053928546, + "d_pool_score": 0.0008298024805049333, + "d_ape": 6.2789767196702126e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 1.9999847412109375, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-2long-descending-zigzag.json", + "raw_sha256": "a831a51932d12b792df73be5d194891b435766a9e9adfbeea5f44922d4d966d0" + } + }, + "speedup_vs_contiguous": 1.3303655272187225, + "gap_to_best_baseline_pct": -0.37776082611535866, + "prepare_plus_iteration_ms": { + "contiguous": 1254.078814609442, + "planner": 978.5550387739204, + "zigzag": 995.0297327917069 + }, + "prepare_inclusive_gap_pct": -1.6556986665679063, + "documents": 512, + "lengths_sha256": "1355e0fb1ef1d83a855305c16e873372467e69d68007d35dcefa354ea7c6ab1b" + }, + "1m-count-mixed-512-4long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 972.0389709472656, + "min_ms": 970.7367553710938, + "max_ms": 973.3665771484375, + "allocated_gib": 39.3654465675354, + "reserved_gib": 45.3046875, + "prepare_ms": 7.157045416533947, + "layout_prepare_ms": 6.765970028936863, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 86, + "raw_file": "layout-1m-count-mixed-512-4long-contiguous.json", + "raw_sha256": "67e72a989c83401b91323ed83782e3cb11b216a0cd580612b06ed511631e9d92" + }, + "planner": { + "total_ms": 874.1318664550781, + "min_ms": 873.6409301757812, + "max_ms": 875.0445556640625, + "allocated_gib": 39.242146015167236, + "reserved_gib": 44.970703125, + "prepare_ms": 53.79614233970642, + "layout_prepare_ms": 53.46741620451212, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0014319279148353801, + "dq": 0.000730531837343021, + "dkv": 0.0006971483885515934, + "d_index_q": 0.0006813065174239298, + "d_weights": 0.0006861706654843519, + "d_pool_kv": 0.0007879833098417588, + "d_pool_score": 0.0007882555784148795, + "d_ape": 6.418386021629022e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.4999895580112934, + "max_spans": 89, + "raw_file": "layout-1m-count-mixed-512-4long-planner.json", + "raw_sha256": "0b4d24ea5cda106a16d3515fb3d8236d1efc50a131d269ebf4d0e9b3b2f22820" + }, + "zigzag": { + "total_ms": 880.24072265625, + "min_ms": 879.207763671875, + "max_ms": 880.8388671875, + "allocated_gib": 39.36966896057129, + "reserved_gib": 45.740234375, + "prepare_ms": 37.095713429152966, + "layout_prepare_ms": 36.7745952680707, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014058193089270753, + "dq": 0.0007201690410107729, + "dkv": 0.0006351635106884441, + "d_index_q": 0.0006657213660369258, + "d_weights": 0.000676296287009572, + "d_pool_kv": 0.0007949862823955126, + "d_pool_score": 0.0007856044854621893, + "d_ape": 6.393762828290671e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49999282509088516, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-4long-zigzag.json", + "raw_sha256": "587fc508900e40c6333631ad80d10af8a5165c2c75601bde78d44a9798f920c9" + } + }, + "speedup_vs_contiguous": 1.1120049597199062, + "gap_to_best_baseline_pct": -0.6939983624862922, + "prepare_plus_iteration_ms": { + "contiguous": 979.1960163637996, + "planner": 927.9280087947845, + "zigzag": 917.336436085403 + }, + "prepare_inclusive_gap_pct": 1.154600677869011, + "documents": 512, + "lengths_sha256": "74f528dba1eacc05b1dd3ae32df6d6e5fcba98ba8abc06fecf0c0df6bf0aa380" + }, + "1m-count-near-uniform-512": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 452.79539489746094, + "min_ms": 452.1741638183594, + "max_ms": 453.1943359375, + "allocated_gib": 37.642160415649414, + "reserved_gib": 44.06640625, + "prepare_ms": 7.008280954323709, + "layout_prepare_ms": 6.630392977967858, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.07122802734375, + "max_spans": 36, + "raw_file": "layout-1m-count-near-uniform-512-contiguous.json", + "raw_sha256": "62d05651861d7981e13bf43c83846a15306b206ca1e1b66a1f00626397fa6739" + }, + "planner": { + "total_ms": 452.81201171875, + "min_ms": 452.55108642578125, + "max_ms": 453.0105895996094, + "allocated_gib": 37.642160415649414, + "reserved_gib": 44.5390625, + "prepare_ms": 7.0143090561032295, + "layout_prepare_ms": 6.682493723928928, + "policy": "contiguous", + "relative_rms": { + "output": 0.000312570186443386, + "dq": 0.00021878066333155848, + "dkv": 0.00017310433315514652, + "d_index_q": 8.758753491107588e-05, + "d_weights": 8.747408107936404e-05, + "d_pool_kv": 0.0001562169465468653, + "d_pool_score": 0.00014843652847624337, + "d_ape": 2.1200156610156e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.07122802734375, + "max_spans": 36, + "raw_file": "layout-1m-count-near-uniform-512-planner.json", + "raw_sha256": "ae47956396ddb81e5bbbd890d2e31d31a5a5cab5e970a179ea2f3b13364d2d05" + }, + "zigzag": { + "total_ms": 489.8685302734375, + "min_ms": 488.3931884765625, + "max_ms": 490.7578430175781, + "allocated_gib": 37.648831844329834, + "reserved_gib": 45.1796875, + "prepare_ms": 64.41162247210741, + "layout_prepare_ms": 64.05988894402981, + "policy": "zigzag", + "relative_rms": { + "output": 0.0003119673592817592, + "dq": 0.00021851863380101095, + "dkv": 0.00017099683323057972, + "d_index_q": 8.753351513712065e-05, + "d_weights": 8.460601101446695e-05, + "d_pool_kv": 0.0001494269842368115, + "d_pool_score": 0.00018111080123773155, + "d_ape": 2.193984997606667e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.06634521484375, + "max_spans": 1024, + "raw_file": "layout-1m-count-near-uniform-512-zigzag.json", + "raw_sha256": "9c0ac05320b645e3d9290cc65c3e6c92fc0b703a622603743c3d114dd6cd3a92" + } + }, + "speedup_vs_contiguous": 0.9999633030466087, + "gap_to_best_baseline_pct": 0.0036698300106996484, + "prepare_plus_iteration_ms": { + "contiguous": 459.80367585178465, + "planner": 459.82632077485323, + "zigzag": 554.2801527455449 + }, + "prepare_inclusive_gap_pct": 0.004924911273618271, + "documents": 512, + "lengths_sha256": "a883d69122302c1f8f9693929fa0ea899a3e5e33acb0a13cd8340e29afc198a0" + }, + "1m-count-tiny-mix-1000": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 804.4497375488281, + "min_ms": 803.3792114257812, + "max_ms": 804.8482666015625, + "allocated_gib": 39.36701440811157, + "reserved_gib": 45.427734375, + "prepare_ms": 11.593663948588073, + "layout_prepare_ms": 11.099680326879025, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 143, + "raw_file": "layout-1m-count-tiny-mix-1000-contiguous.json", + "raw_sha256": "1dc08db67824ae58a274a6c2e945a5729ca8c097cd1d95ab55166978e3cb5181" + }, + "planner": { + "total_ms": 803.9241638183594, + "min_ms": 803.6096801757812, + "max_ms": 804.8082275390625, + "allocated_gib": 39.36701440811157, + "reserved_gib": 46.46875, + "prepare_ms": 12.909071519970894, + "layout_prepare_ms": 12.430059723556042, + "policy": "contiguous", + "relative_rms": { + "output": 0.0009725538862074621, + "dq": 0.0006188448813383469, + "dkv": 0.0003971155353896982, + "d_index_q": 0.0002914177003272136, + "d_weights": 0.0003071750406410824, + "d_pool_kv": 0.0003651129601905051, + "d_pool_score": 0.0003462786951790806, + "d_ape": 3.5353953008116053e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 143, + "raw_file": "layout-1m-count-tiny-mix-1000-planner.json", + "raw_sha256": "05addf8eabd249e3f02186e0478fd744b54c21e61b18c854750e7c80f2ab61fe" + }, + "zigzag": { + "total_ms": 819.8125, + "min_ms": 818.898193359375, + "max_ms": 820.943603515625, + "allocated_gib": 39.35548257827759, + "reserved_gib": 46.896484375, + "prepare_ms": 70.85726968944073, + "layout_prepare_ms": 70.35546749830246, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009806580554309915, + "dq": 0.0006204996956369439, + "dkv": 0.0003984549191234964, + "d_index_q": 0.00029134836348343544, + "d_weights": 0.0003059298731533832, + "d_pool_kv": 0.000370834144304363, + "d_pool_score": 0.000391578141157335, + "d_ape": 3.161593514832567e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49914050847291946, + "max_spans": 1200, + "raw_file": "layout-1m-count-tiny-mix-1000-zigzag.json", + "raw_sha256": "82adb8554d17767de8049dfbb13a48c79d2ff126bdfecfa730ecfba4629cacc8" + } + }, + "speedup_vs_contiguous": 1.0006537603347714, + "gap_to_best_baseline_pct": -0.06533332114324608, + "prepare_plus_iteration_ms": { + "contiguous": 816.0434014974162, + "planner": 816.8332353383303, + "zigzag": 890.6697696894407 + }, + "prepare_inclusive_gap_pct": 0.09678821487493217, + "documents": 1000, + "lengths_sha256": "2043af46235527c07d70653288a5fbff85cbdc1d61c2df251c4b11c9f2173662" + }, + "1m-count-tiny-mix-512": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 685.9302673339844, + "min_ms": 684.9505615234375, + "max_ms": 686.43310546875, + "allocated_gib": 38.420175552368164, + "reserved_gib": 45.21484375, + "prepare_ms": 7.0847609313204885, + "layout_prepare_ms": 6.701943988446146, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.498516321182251, + "max_spans": 59, + "raw_file": "layout-1m-count-tiny-mix-512-contiguous.json", + "raw_sha256": "a09d8c61624f4cc1af71729054dc7ddf2611dac3c7046af0ae8bd417b04ec30c" + }, + "planner": { + "total_ms": 685.8549194335938, + "min_ms": 685.347900390625, + "max_ms": 685.97705078125, + "allocated_gib": 38.420175552368164, + "reserved_gib": 45.21484375, + "prepare_ms": 6.993969029281288, + "layout_prepare_ms": 6.630199961364269, + "policy": "contiguous", + "relative_rms": { + "output": 0.0010325317851200527, + "dq": 0.0006162005011788455, + "dkv": 0.0003776748189111857, + "d_index_q": 0.0002628102942323622, + "d_weights": 0.0002676192491992476, + "d_pool_kv": 0.00033682698470347753, + "d_pool_score": 0.0003439917375613672, + "d_ape": 6.1279133614286e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.498516321182251, + "max_spans": 59, + "raw_file": "layout-1m-count-tiny-mix-512-planner.json", + "raw_sha256": "0add243c5864581013780f0e5889b80b837e719a555a8ac74d61f354649a2e00" + }, + "zigzag": { + "total_ms": 716.0761108398438, + "min_ms": 715.8760986328125, + "max_ms": 717.7543334960938, + "allocated_gib": 38.55094909667969, + "reserved_gib": 45.19921875, + "prepare_ms": 43.37962158024311, + "layout_prepare_ms": 43.03775541484356, + "policy": "zigzag", + "relative_rms": { + "output": 0.0010499063952199626, + "dq": 0.0006258225003131088, + "dkv": 0.00041073803877893905, + "d_index_q": 0.00032662600157353964, + "d_weights": 0.00031260323144411474, + "d_pool_kv": 0.00035676072352841396, + "d_pool_score": 0.0003572713035249323, + "d_ape": 6.468741565403711e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.37377477437257767, + "max_spans": 614, + "raw_file": "layout-1m-count-tiny-mix-512-zigzag.json", + "raw_sha256": "19cccd84d9f2a66d30d27386dcd71840e0b359724ec570a1bf237e3257b1ee70" + } + }, + "speedup_vs_contiguous": 1.0001098598234928, + "gap_to_best_baseline_pct": -0.010984775563771532, + "prepare_plus_iteration_ms": { + "contiguous": 693.0150282653049, + "planner": 692.848888462875, + "zigzag": 759.4557324200869 + }, + "prepare_inclusive_gap_pct": -0.023973477580374603, + "documents": 512, + "lengths_sha256": "eb5e64b29a22782c3e203c9db504474452dd18b36ac3aa9a4ad991a5af9bdbf9" + }, + "1m-count-uniform-1024": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 304.08592224121094, + "min_ms": 303.8802490234375, + "max_ms": 304.4345703125, + "allocated_gib": 38.423105239868164, + "reserved_gib": 45.3671875, + "prepare_ms": 11.142310046125203, + "layout_prepare_ms": 10.64562500687316, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.03118896484375, + "max_spans": 64, + "raw_file": "layout-1m-count-uniform-1024-contiguous.json", + "raw_sha256": "11561bae83a343dadcdb661feaa53ae03cd8f5eadfd0080792d1f6fc43d05d54" + }, + "planner": { + "total_ms": 303.88612365722656, + "min_ms": 303.5679931640625, + "max_ms": 304.477783203125, + "allocated_gib": 38.423166275024414, + "reserved_gib": 45.9609375, + "prepare_ms": 12.489238986745477, + "layout_prepare_ms": 12.01151602435857, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 4.0017354011135795e-05, + "d_index_q": 0.0, + "d_weights": 1.5643278801343326e-05, + "d_pool_kv": 3.5892032839695286e-05, + "d_pool_score": 4.1180529947898683e-05, + "d_ape": 4.6654965940994376e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.03118896484375, + "max_spans": 64, + "raw_file": "layout-1m-count-uniform-1024-planner.json", + "raw_sha256": "27317d3bcce2e740aa3113a446f2d33f3f4c1fff3fa1014172bb3fa0956a3b5b" + }, + "zigzag": { + "total_ms": 345.7340545654297, + "min_ms": 344.55413818359375, + "max_ms": 346.9438171386719, + "allocated_gib": 38.42808675765991, + "reserved_gib": 45.96484375, + "prepare_ms": 129.5411480241455, + "layout_prepare_ms": 129.09127998864278, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 4.131296766393347e-05, + "d_index_q": 0.0, + "d_weights": 1.1611570293793908e-05, + "d_pool_kv": 4.137243430716575e-05, + "d_pool_score": 7.625351996008623e-05, + "d_ape": 5.545519532233894e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.03118896484375, + "max_spans": 2048, + "raw_file": "layout-1m-count-uniform-1024-zigzag.json", + "raw_sha256": "a572931b350c86621dfd0eeb7dc1ca0e4ffab8bc02da97a47fe8195d874f4b1a" + } + }, + "speedup_vs_contiguous": 1.000657478471145, + "gap_to_best_baseline_pct": -0.06570464772318596, + "prepare_plus_iteration_ms": { + "contiguous": 315.22823228733614, + "planner": 316.37536264397204, + "zigzag": 475.2752025895752 + }, + "prepare_inclusive_gap_pct": 0.363904701146911, + "documents": 1024, + "lengths_sha256": "7d1b6ad860e729dfc1e098d7b89b3e6e85ea0e907b90de52c462e82eb70e9471" + }, + "1m-count-uniform-256": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 549.6219177246094, + "min_ms": 549.1296997070312, + "max_ms": 550.0491943359375, + "allocated_gib": 38.51983880996704, + "reserved_gib": 45.65625, + "prepare_ms": 4.39696479588747, + "layout_prepare_ms": 4.108703695237637, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.12493896484375, + "max_spans": 16, + "raw_file": "layout-1m-count-uniform-256-contiguous.json", + "raw_sha256": "bd6a3011bc09519f88cdbb431d094b4dc9d7f52ae5cc585e0538289d59d65a84" + }, + "planner": { + "total_ms": 549.4464111328125, + "min_ms": 549.179443359375, + "max_ms": 549.8413696289062, + "allocated_gib": 38.51983880996704, + "reserved_gib": 45.0625, + "prepare_ms": 4.716673865914345, + "layout_prepare_ms": 4.421164281666279, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006925854248702454, + "dq": 0.0004392233399569253, + "dkv": 0.00029740960412245063, + "d_index_q": 0.00024004049781900123, + "d_weights": 0.00019391788661221848, + "d_pool_kv": 0.00026141680733655204, + "d_pool_score": 0.0002586076167179979, + "d_ape": 4.22166115570864e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.12493896484375, + "max_spans": 16, + "raw_file": "layout-1m-count-uniform-256-planner.json", + "raw_sha256": "d2eb7ac4c21a145d7a446f6436c735525217ccb0764abc7f8deee85e4d728e6e" + }, + "zigzag": { + "total_ms": 584.0359802246094, + "min_ms": 583.1168212890625, + "max_ms": 585.0107421875, + "allocated_gib": 38.524731159210205, + "reserved_gib": 45.958984375, + "prepare_ms": 32.832130789756775, + "layout_prepare_ms": 32.5444545596838, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006908169770485342, + "dq": 0.00043303490110947876, + "dkv": 0.00027639050441138717, + "d_index_q": 0.00015210646909709564, + "d_weights": 0.00015894974404154317, + "d_pool_kv": 0.0002506036255637015, + "d_pool_score": 0.00026886913645408276, + "d_ape": 4.130315630069566e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.12493896484375, + "max_spans": 512, + "raw_file": "layout-1m-count-uniform-256-zigzag.json", + "raw_sha256": "a3209638e2436bb3bd95b42af0c6ce1f771dd7736f682c207a4df18e76810150" + } + }, + "speedup_vs_contiguous": 1.0003194244028912, + "gap_to_best_baseline_pct": -0.03193224035232811, + "prepare_plus_iteration_ms": { + "contiguous": 554.0188825204968, + "planner": 554.1630849987268, + "zigzag": 616.8681110143661 + }, + "prepare_inclusive_gap_pct": 0.026028441047709627, + "documents": 256, + "lengths_sha256": "2b66265ed16d5c91894fb143f8c654f23f801053a71d4fd69181b0642cf716a7" + }, + "1m-heldout-jittered-768-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 358.52392578125, + "min_ms": 357.597412109375, + "max_ms": 358.72125244140625, + "allocated_gib": 38.9788122177124, + "reserved_gib": 45.28515625, + "prepare_ms": 8.966105058789253, + "layout_prepare_ms": 8.518191985785961, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0449279248714447, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-contiguous.json", + "raw_sha256": "5dcb5213bc8f12b186a0ceaeed228456ac87d83946aba7bb5d5dee019ad26ce1" + }, + "planner": { + "total_ms": 358.4803161621094, + "min_ms": 357.7920837402344, + "max_ms": 358.86279296875, + "allocated_gib": 38.9788122177124, + "reserved_gib": 45.28515625, + "prepare_ms": 10.183462873101234, + "layout_prepare_ms": 9.80326347053051, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.363129604384954e-05, + "d_index_q": 0.0, + "d_weights": 1.1220725449324892e-05, + "d_pool_kv": 8.384201051467693e-05, + "d_pool_score": 5.988107855092994e-05, + "d_ape": 5.636189088678152e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0449279248714447, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-planner.json", + "raw_sha256": "7cd682aa27cc1f41a05c9af6ac44e2d9121ab9dec408343086ba737f0cdbfa91" + }, + "zigzag": { + "total_ms": 397.6351623535156, + "min_ms": 396.7493896484375, + "max_ms": 398.86016845703125, + "allocated_gib": 38.98426389694214, + "reserved_gib": 45.98828125, + "prepare_ms": 95.2249551191926, + "layout_prepare_ms": 94.84434872865677, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.3002213991950895e-05, + "d_index_q": 0.0, + "d_weights": 1.3272182524832172e-05, + "d_pool_kv": 8.348898455953135e-05, + "d_pool_score": 4.427979618573066e-05, + "d_ape": 5.096660004587656e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04387390986084938, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-zigzag.json", + "raw_sha256": "d14853409595353c0dafa9ca64ebddcee9ac78ca1611928c3256058091b1ab4d" + } + }, + "speedup_vs_contiguous": 1.0001216513631976, + "gap_to_best_baseline_pct": -0.012163656594355032, + "prepare_plus_iteration_ms": { + "contiguous": 367.49003084003925, + "planner": 368.6637790352106, + "zigzag": 492.8601174727082 + }, + "prepare_inclusive_gap_pct": 0.3193959282346448, + "documents": 768, + "lengths_sha256": "2384da57da625501af834608e8298b60467638272e4c35dc6ea261713c7eb0d3" + }, + "1m-heldout-jittered-768-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 358.77650451660156, + "min_ms": 358.13946533203125, + "max_ms": 359.1441650390625, + "allocated_gib": 39.808849811553955, + "reserved_gib": 45.8671875, + "prepare_ms": 9.590373025275767, + "layout_prepare_ms": 9.206185990478843, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04477034509181976, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-contiguous.json", + "raw_sha256": "789902a0f27ec3d80ade697ee5da1a97a3c9573fee04bc888f5934378afb9322" + }, + "planner": { + "total_ms": 358.70045471191406, + "min_ms": 358.292236328125, + "max_ms": 359.0645751953125, + "allocated_gib": 39.808849811553955, + "reserved_gib": 45.8671875, + "prepare_ms": 10.364365763962269, + "layout_prepare_ms": 9.964034892618656, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.2490964013625554e-05, + "d_index_q": 0.0, + "d_weights": 1.3651717477780841e-05, + "d_pool_kv": 3.4509005688492454e-05, + "d_pool_score": 3.3288530731895434e-05, + "d_ape": 4.501287051876582e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04477034509181976, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-planner.json", + "raw_sha256": "4c51e8a59b01dad3ad4f98ac18943960178bbe4e4f1cd4a2786e503612769735" + }, + "zigzag": { + "total_ms": 398.11195373535156, + "min_ms": 397.47796630859375, + "max_ms": 399.0572509765625, + "allocated_gib": 39.81116342544556, + "reserved_gib": 46.27734375, + "prepare_ms": 97.08238858729601, + "layout_prepare_ms": 96.70192841440439, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 4.378364012653045e-05, + "d_index_q": 0.0, + "d_weights": 1.5808015694390196e-05, + "d_pool_kv": 3.262385514096063e-05, + "d_pool_score": 3.426303838873765e-05, + "d_ape": 4.392719494088674e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.043894730508327484, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-zigzag.json", + "raw_sha256": "b38910fe602a76d46de8d8a34e7db773bcaedb881686cf6b2318b3c2d2dd98d8" + } + }, + "speedup_vs_contiguous": 1.000212014798667, + "gap_to_best_baseline_pct": -0.02119698579202689, + "prepare_plus_iteration_ms": { + "contiguous": 368.36687754187733, + "planner": 369.06482047587633, + "zigzag": 495.19434232264757 + }, + "prepare_inclusive_gap_pct": 0.18946951437555093, + "documents": 768, + "lengths_sha256": "8884d0c226a9c39788069b745f1fa9b8499da77272f0e098c192c933dea50872" + }, + "1m-heldout-jittered-768-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 359.0693359375, + "min_ms": 358.796142578125, + "max_ms": 359.2643737792969, + "allocated_gib": 40.17335319519043, + "reserved_gib": 46.28125, + "prepare_ms": 9.378077986184508, + "layout_prepare_ms": 8.974294003564864, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.044239819049835205, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-contiguous.json", + "raw_sha256": "d11bdff7eb5b9169b7cc30d74ed4caf21b5506ae4d8761eb04c9a6d7769f1274" + }, + "planner": { + "total_ms": 359.144287109375, + "min_ms": 358.93280029296875, + "max_ms": 359.4841613769531, + "allocated_gib": 40.17335319519043, + "reserved_gib": 46.28125, + "prepare_ms": 10.32171497354284, + "layout_prepare_ms": 9.934334957506508, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.280996475029592e-05, + "d_index_q": 0.0, + "d_weights": 1.2683953507657087e-05, + "d_pool_kv": 3.495714158265876e-05, + "d_pool_score": 3.7603865072635344e-05, + "d_ape": 4.496830226084333e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.044239819049835205, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-planner.json", + "raw_sha256": "a543fa3f4100c72630f6afd3572f5c0a197ea70b3b70f3202460326a5fe8acf4" + }, + "zigzag": { + "total_ms": 398.9141540527344, + "min_ms": 397.929931640625, + "max_ms": 399.3319091796875, + "allocated_gib": 40.17674732208252, + "reserved_gib": 46.283203125, + "prepare_ms": 97.00689493911341, + "layout_prepare_ms": 96.62403498077765, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.293799597415976e-05, + "d_index_q": 0.0, + "d_weights": 1.7251727926864844e-05, + "d_pool_kv": 3.9521198405270576e-05, + "d_pool_score": 4.615198594900765e-05, + "d_ape": 5.105090459413078e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04397561401128769, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-zigzag.json", + "raw_sha256": "e84e9c5456c0795d9f4893ed1e9ce4e21ee13233bef2169fbd5128e3f39db425" + } + }, + "speedup_vs_contiguous": 0.9997913062393996, + "gap_to_best_baseline_pct": 0.020873732277726376, + "prepare_plus_iteration_ms": { + "contiguous": 368.4474139236845, + "planner": 369.46600208291784, + "zigzag": 495.9210489918478 + }, + "prepare_inclusive_gap_pct": 0.27645414806583, + "documents": 768, + "lengths_sha256": "484660be7f47772a4ee1c32aa13529e5d4dfa02ba770b46f67c83944d3d16782" + }, + "1m-heldout-lognormal-1000-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 684.9967041015625, + "min_ms": 684.418701171875, + "max_ms": 685.22021484375, + "allocated_gib": 39.45905113220215, + "reserved_gib": 46.2421875, + "prepare_ms": 10.71262452751398, + "layout_prepare_ms": 10.212206281721592, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 114, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-contiguous.json", + "raw_sha256": "fc53cac7b6a0143619cea370851e59a133f109d31f150b72b933cd217c1be2e1" + }, + "planner": { + "total_ms": 684.5411682128906, + "min_ms": 684.1309814453125, + "max_ms": 684.8707275390625, + "allocated_gib": 39.45905113220215, + "reserved_gib": 46.2421875, + "prepare_ms": 12.842495983932167, + "layout_prepare_ms": 12.371807999443263, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006604366951019903, + "dq": 0.0004884184178103446, + "dkv": 0.0002441307659025493, + "d_index_q": 0.00017720154893536717, + "d_weights": 0.0001965275658067716, + "d_pool_kv": 0.0002635452150096701, + "d_pool_score": 0.00025799601845620344, + "d_ape": 3.7463702901400385e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 114, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-planner.json", + "raw_sha256": "56ea84e33677dd134f4c00ed9632d537cc5df9e7dd030f54cd7be1144bed6490" + }, + "zigzag": { + "total_ms": 700.9886169433594, + "min_ms": 700.3733520507812, + "max_ms": 701.34033203125, + "allocated_gib": 39.49775457382202, + "reserved_gib": 46.5, + "prepare_ms": 120.16775365918875, + "layout_prepare_ms": 119.67392452061176, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006636288524415482, + "dq": 0.0004896882885293665, + "dkv": 0.0002431552086838662, + "d_index_q": 0.00017754175506518597, + "d_weights": 0.00019942199427568522, + "d_pool_kv": 0.0002599323951245154, + "d_pool_score": 0.00026026892084907567, + "d_ape": 3.772762659927586e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.3476196713745594, + "max_spans": 1628, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-zigzag.json", + "raw_sha256": "dcc44346b9d207f0c56919c0e0ca3ab35a5842c9b77986c69219d4bb25b3e689" + } + }, + "speedup_vs_contiguous": 1.0006654616403292, + "gap_to_best_baseline_pct": -0.0665019095631103, + "prepare_plus_iteration_ms": { + "contiguous": 695.7093286290765, + "planner": 697.3836641968228, + "zigzag": 821.1563706025481 + }, + "prepare_inclusive_gap_pct": 0.24066596477090307, + "documents": 1000, + "lengths_sha256": "f9552804d64123e178b417a374ec673ead4524cfdca92d036e2c49d2242a4462" + }, + "1m-heldout-lognormal-1000-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 763.6994323730469, + "min_ms": 763.1716918945312, + "max_ms": 764.0468139648438, + "allocated_gib": 40.053749561309814, + "reserved_gib": 46.6796875, + "prepare_ms": 12.699897983111441, + "layout_prepare_ms": 12.214038986712694, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 113, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-contiguous.json", + "raw_sha256": "a508d050d66317a33bed9a5c2ef06bf8b4089f1ebfcfedf36b64c479ab8e8527" + }, + "planner": { + "total_ms": 763.3740234375, + "min_ms": 762.8582153320312, + "max_ms": 763.90478515625, + "allocated_gib": 40.053749561309814, + "reserved_gib": 46.6796875, + "prepare_ms": 42.369666101876646, + "layout_prepare_ms": 41.90250305691734, + "policy": "contiguous", + "relative_rms": { + "output": 0.0008212079038689687, + "dq": 0.0005803233894914287, + "dkv": 0.0003178547583325437, + "d_index_q": 0.00024748702763004843, + "d_weights": 0.0002898346991169899, + "d_pool_kv": 0.00037226213041429183, + "d_pool_score": 0.0003641375152335028, + "d_ape": 3.5695850024897406e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 113, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-planner.json", + "raw_sha256": "9ba3060c3dbb7e99ec6a0205903bd122391f8254146785881fd03798dff85481" + }, + "zigzag": { + "total_ms": 760.7092590332031, + "min_ms": 760.1244506835938, + "max_ms": 761.7604370117188, + "allocated_gib": 40.262939453125, + "reserved_gib": 47.68359375, + "prepare_ms": 111.01808212697506, + "layout_prepare_ms": 110.55053118616343, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008240264857030691, + "dq": 0.000580487816087925, + "dkv": 0.00031653498036455385, + "d_index_q": 0.0002474463983232176, + "d_weights": 0.0002914637802480217, + "d_pool_kv": 0.00036201676107118724, + "d_pool_score": 0.00034605030505217594, + "d_ape": 3.512907903393745e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.4038251228630543, + "max_spans": 1617, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-zigzag.json", + "raw_sha256": "ef3337acf693424381cffe7c0f51de3ee5e65139739c476a8ce96f4e858ccbf6" + } + }, + "speedup_vs_contiguous": 1.0004262771925112, + "gap_to_best_baseline_pct": 0.3502999828980169, + "prepare_plus_iteration_ms": { + "contiguous": 776.3993303561583, + "planner": 805.7436895393766, + "zigzag": 871.7273411601782 + }, + "prepare_inclusive_gap_pct": 3.779544628117737, + "documents": 1000, + "lengths_sha256": "4860b063d4278394f20027f7e1f50de3bf6080e17af8500ea52a852f8f4840bf" + }, + "1m-heldout-lognormal-1000-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 736.0016479492188, + "min_ms": 735.2288818359375, + "max_ms": 737.14697265625, + "allocated_gib": 40.697120666503906, + "reserved_gib": 47.77734375, + "prepare_ms": 11.603533290326595, + "layout_prepare_ms": 11.127131059765816, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 99, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-contiguous.json", + "raw_sha256": "13efa21223b67e20bef071619cac3497a2dbb9ee457fc8770998898b2c556cf8" + }, + "planner": { + "total_ms": 736.3946838378906, + "min_ms": 734.807861328125, + "max_ms": 737.6834716796875, + "allocated_gib": 40.697792053222656, + "reserved_gib": 47.77734375, + "prepare_ms": 12.68920605070889, + "layout_prepare_ms": 12.206138984765857, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006433898524578946, + "dq": 0.0005110624096845733, + "dkv": 0.00026048934040463144, + "d_index_q": 0.00019598883463898154, + "d_weights": 0.000228863130253541, + "d_pool_kv": 0.000301853480114409, + "d_pool_score": 0.000297540264113885, + "d_ape": 4.091797227406095e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 99, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-planner.json", + "raw_sha256": "b4a6625eec2d393f2c092be2cadeb171c27c1f3177e3c92750bde9602a1a1490" + }, + "zigzag": { + "total_ms": 735.1014709472656, + "min_ms": 734.1682739257812, + "max_ms": 735.7579956054688, + "allocated_gib": 41.009629249572754, + "reserved_gib": 47.47265625, + "prepare_ms": 107.5354665517807, + "layout_prepare_ms": 107.07924794405699, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006420663728273947, + "dq": 0.0005102186802942892, + "dkv": 0.00027717264627597377, + "d_index_q": 0.00019555019132033196, + "d_weights": 0.0002284901651517546, + "d_pool_kv": 0.00030743783418931916, + "d_pool_score": 0.0003033758473351343, + "d_ape": 3.978015480011178e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49536293372511864, + "max_spans": 1584, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-zigzag.json", + "raw_sha256": "72dd0d69778869f26fb583147e76cf249ffa64625bf1c376320dbbca473fa59a" + } + }, + "speedup_vs_contiguous": 0.9994662700623754, + "gap_to_best_baseline_pct": 0.17592304487685784, + "prepare_plus_iteration_ms": { + "contiguous": 747.6051812395453, + "planner": 749.0838898885995, + "zigzag": 842.6369374990463 + }, + "prepare_inclusive_gap_pct": 0.19779272350712862, + "documents": 1000, + "lengths_sha256": "a790875510a4369ef17daa8104180cfe9404b90dfa3567b8652ea1435220224c" + }, + "1m-heldout-lognormal-257-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 666.2662658691406, + "min_ms": 665.9510498046875, + "max_ms": 666.6072998046875, + "allocated_gib": 39.30071496963501, + "reserved_gib": 46.08203125, + "prepare_ms": 4.745754005853087, + "layout_prepare_ms": 4.4530140003189445, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.463335856795311, + "max_spans": 26, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-contiguous.json", + "raw_sha256": "4dcabc1f4bce11d18a2d57a37fe93b11f4837c03a49ec63c70a03fae41f78a68" + }, + "planner": { + "total_ms": 666.1408996582031, + "min_ms": 665.9105224609375, + "max_ms": 666.6676635742188, + "allocated_gib": 39.30185651779175, + "reserved_gib": 46.08203125, + "prepare_ms": 4.790247010532767, + "layout_prepare_ms": 4.503867996390909, + "policy": "contiguous", + "relative_rms": { + "output": 0.0009385242732204369, + "dq": 0.0005866022976175567, + "dkv": 0.00033332166942627665, + "d_index_q": 0.0002301383322868088, + "d_weights": 0.0002399194161665796, + "d_pool_kv": 0.0002873366391359497, + "d_pool_score": 0.0002897758095958852, + "d_ape": 4.674192029871579e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.463335856795311, + "max_spans": 26, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-planner.json", + "raw_sha256": "3d2579cce44e52c4cbe0ae529d9f6cb1dc6c2cae81df1d33a5e012991896d356" + }, + "zigzag": { + "total_ms": 694.9446716308594, + "min_ms": 694.22998046875, + "max_ms": 695.3518676757812, + "allocated_gib": 39.167086124420166, + "reserved_gib": 46.083984375, + "prepare_ms": 32.833476550877094, + "layout_prepare_ms": 32.56948012858629, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009470464886583148, + "dq": 0.0005880154208656388, + "dkv": 0.00036135005526755385, + "d_index_q": 0.00022518916873369107, + "d_weights": 0.00024094943810340667, + "d_pool_kv": 0.00029319441343405016, + "d_pool_score": 0.0003009505490939196, + "d_ape": 4.417832424111279e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.3246483616530895, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-zigzag.json", + "raw_sha256": "c9d586768d9e15167c314f37e3321489cba239d2ae0aced4c94349a4e8696776" + } + }, + "speedup_vs_contiguous": 1.0001881977386493, + "gap_to_best_baseline_pct": -0.01881623269248678, + "prepare_plus_iteration_ms": { + "contiguous": 671.0120198749937, + "planner": 670.9311466687359, + "zigzag": 727.7781481817365 + }, + "prepare_inclusive_gap_pct": -0.012052422886987824, + "documents": 257, + "lengths_sha256": "22124dfa90691813499ba5fff3956732dd43551f4504a48fc46bbb35d7aa1141" + }, + "1m-heldout-lognormal-257-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 786.7605895996094, + "min_ms": 786.594482421875, + "max_ms": 787.9214477539062, + "allocated_gib": 39.929911613464355, + "reserved_gib": 46.1015625, + "prepare_ms": 4.555713036097586, + "layout_prepare_ms": 4.276246996596456, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 30, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-contiguous.json", + "raw_sha256": "01d703a50f75290adad1292fa978fa35a81b8ccd5af057a28ed4963da8cabe0f" + }, + "planner": { + "total_ms": 786.9058532714844, + "min_ms": 785.9761352539062, + "max_ms": 787.4791259765625, + "allocated_gib": 39.929789543151855, + "reserved_gib": 46.1015625, + "prepare_ms": 23.62960804020986, + "layout_prepare_ms": 23.33585696760565, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011388051659333431, + "dq": 0.0006564971734834571, + "dkv": 0.0004364387193785559, + "d_index_q": 0.000436640407897889, + "d_weights": 0.000383328993036165, + "d_pool_kv": 0.000393598589728051, + "d_pool_score": 0.00039426697683359133, + "d_ape": 5.08645882566227e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 30, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-planner.json", + "raw_sha256": "13efb3b4d74cf45df412460339bbe0a63c20173fcbbd81e46215bc1fd7d38ca2" + }, + "zigzag": { + "total_ms": 774.9671936035156, + "min_ms": 774.4342651367188, + "max_ms": 775.7385864257812, + "allocated_gib": 39.93263816833496, + "reserved_gib": 47.46875, + "prepare_ms": 33.89150928705931, + "layout_prepare_ms": 33.621191047132015, + "policy": "zigzag", + "relative_rms": { + "output": 0.001143064133573562, + "dq": 0.0006575020348302019, + "dkv": 0.00043892282645587147, + "d_index_q": 0.00035061364043188786, + "d_weights": 0.00038229657460199485, + "d_pool_kv": 0.00039741900440744445, + "d_pool_score": 0.00039195516172039574, + "d_ape": 5.082429890969267e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49996573850512505, + "max_spans": 513, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-zigzag.json", + "raw_sha256": "01378420248c6dd51aa82cdbc474318bd904cf12df08386f5a49b6ad410b73b7" + } + }, + "speedup_vs_contiguous": 0.9998153989180902, + "gap_to_best_baseline_pct": 1.5405374274561456, + "prepare_plus_iteration_ms": { + "contiguous": 791.316302635707, + "planner": 810.5354613116942, + "zigzag": 808.8587028905749 + }, + "prepare_inclusive_gap_pct": 2.428758084721916, + "documents": 257, + "lengths_sha256": "0b8ffc4c0c7294d43fa7ae5f3ee55faaef51c2f526387b1efe2d60aca24ecf20" + }, + "1m-heldout-lognormal-257-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 792.4435119628906, + "min_ms": 792.0433959960938, + "max_ms": 792.8259887695312, + "allocated_gib": 40.349995136260986, + "reserved_gib": 47.30859375, + "prepare_ms": 4.70799405593425, + "layout_prepare_ms": 4.365537024568766, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 28, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-contiguous.json", + "raw_sha256": "a5ddb06a51ad5b0301088d49f0e7592fe7c03e5e1c4bfeef1c8f61f90b7c4426" + }, + "planner": { + "total_ms": 792.43798828125, + "min_ms": 791.5970458984375, + "max_ms": 793.3018188476562, + "allocated_gib": 40.349995136260986, + "reserved_gib": 47.30859375, + "prepare_ms": 4.752930020913482, + "layout_prepare_ms": 4.501574032474309, + "policy": "contiguous", + "relative_rms": { + "output": 0.001227522930499042, + "dq": 0.0006738229944997933, + "dkv": 0.0005094287079412098, + "d_index_q": 0.000377814752024436, + "d_weights": 0.0004027883643553077, + "d_pool_kv": 0.0004238109628606652, + "d_pool_score": 0.0005288219532210354, + "d_ape": 5.122086960896457e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 28, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-planner.json", + "raw_sha256": "198812023d665581c0d6b7835fc3e313c600c914000996a05f6c4dae6ff024bd" + }, + "zigzag": { + "total_ms": 789.9468688964844, + "min_ms": 789.567626953125, + "max_ms": 790.7005615234375, + "allocated_gib": 40.58277988433838, + "reserved_gib": 46.7109375, + "prepare_ms": 33.60013302881271, + "layout_prepare_ms": 33.324415038805455, + "policy": "zigzag", + "relative_rms": { + "output": 0.0012300934864721798, + "dq": 0.0006733776529817055, + "dkv": 0.0005092419932310364, + "d_index_q": 0.0003789284595906472, + "d_weights": 0.0004030942702858978, + "d_pool_kv": 0.00042597496878129035, + "d_pool_score": 0.0005224856786887003, + "d_ape": 4.969292163654653e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.46208810433745384, + "max_spans": 513, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-zigzag.json", + "raw_sha256": "084681bc0cd40e931ce7c1d70dc1712899139a7b97c431ef9feb9066829e0b35" + } + }, + "speedup_vs_contiguous": 1.0000069704907164, + "gap_to_best_baseline_pct": 0.3153527766045272, + "prepare_plus_iteration_ms": { + "contiguous": 797.1515060188249, + "planner": 797.1909183021635, + "zigzag": 823.5470019252971 + }, + "prepare_inclusive_gap_pct": 0.004944139607210296, + "documents": 257, + "lengths_sha256": "1b0bd973931c10a52c40d767fb8a267929d55b8a7e94ed85145c4ab27033efda" + }, + "1m-heldout-tiny-tail-768-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1157.9805297851562, + "min_ms": 1154.6259765625, + "max_ms": 1161.591064453125, + "allocated_gib": 41.0916953086853, + "reserved_gib": 48.49609375, + "prepare_ms": 10.18020004266873, + "layout_prepare_ms": 9.75555699551478, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999936819076538, + "max_spans": 182, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-contiguous.json", + "raw_sha256": "91ed22c0ece3fdda9e0af9b39d07862c32bc0ec7844dbaddac47961b6d34899d" + }, + "planner": { + "total_ms": 887.6731567382812, + "min_ms": 886.7418823242188, + "max_ms": 888.07470703125, + "allocated_gib": 41.17432641983032, + "reserved_gib": 49.134765625, + "prepare_ms": 60.14263443648815, + "layout_prepare_ms": 59.783799573779106, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.001392233060533727, + "dq": 0.0007103945696792637, + "dkv": 0.0006237114201774019, + "d_index_q": 0.0007110733774308313, + "d_weights": 0.0007290097887728568, + "d_pool_kv": 0.0008730318487869236, + "d_pool_score": 0.0008682559631854369, + "d_ape": 6.470404000014752e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 1.8399817384779453, + "max_spans": 186, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-planner.json", + "raw_sha256": "da74a5fecf18cfdd36af78e14717511d3f5a627256de5cf71e13209e75995a9b" + }, + "zigzag": { + "total_ms": 905.5602416992188, + "min_ms": 905.1878051757812, + "max_ms": 906.0325317382812, + "allocated_gib": 42.47430372238159, + "reserved_gib": 46.845703125, + "prepare_ms": 58.144996874034405, + "layout_prepare_ms": 57.731485925614834, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013693331780748614, + "dq": 0.0007012683661039539, + "dkv": 0.0006250740418910341, + "d_index_q": 0.0006958873963356512, + "d_weights": 0.000717607521930308, + "d_pool_kv": 0.0008696228817925042, + "d_pool_score": 0.0008796289615144255, + "d_ape": 6.436690481241068e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 1.9739395715296268, + "max_spans": 894, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-zigzag.json", + "raw_sha256": "18264c92f06f169c8730c50c9612f06ba2c64c3ab8d51799b0fba94dbe51dbbc" + } + }, + "speedup_vs_contiguous": 1.304512275712052, + "gap_to_best_baseline_pct": -1.9752506942413484, + "prepare_plus_iteration_ms": { + "contiguous": 1168.160729827825, + "planner": 947.8157911747694, + "zigzag": 963.7052385732532 + }, + "prepare_inclusive_gap_pct": -1.6487870733179588, + "documents": 768, + "lengths_sha256": "759566912504d164342d1556047348e640f650e2bd969cace033c5f384cd1e9f" + }, + "1m-heldout-tiny-tail-768-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1348.0639038085938, + "min_ms": 1345.923828125, + "max_ms": 1351.31689453125, + "allocated_gib": 41.892484188079834, + "reserved_gib": 47.6171875, + "prepare_ms": 9.628218016587198, + "layout_prepare_ms": 9.187710005789995, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.49999376013875, + "max_spans": 127, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-contiguous.json", + "raw_sha256": "f23e6d23f6186c4dfeeb3396f0e4902d4f550ce4a0fecce0007f4279f8dddd96" + }, + "planner": { + "total_ms": 959.8637390136719, + "min_ms": 959.2150268554688, + "max_ms": 960.6021728515625, + "allocated_gib": 43.086970806121826, + "reserved_gib": 49.92578125, + "prepare_ms": 62.3437559697777, + "layout_prepare_ms": 61.92620098590851, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014097002039817799, + "dq": 0.0007193239552737513, + "dkv": 0.0007043678097197213, + "d_index_q": 0.0008185716297708999, + "d_weights": 0.0008139468703262711, + "d_pool_kv": 0.0009509534037469282, + "d_pool_score": 0.0009260895711974831, + "d_ape": 6.186036509744377e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 2.999843694269657, + "max_spans": 134, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-planner.json", + "raw_sha256": "fed7483a29b50600ee22b7dd13bd9565c6371f32c0b7d9867f7b9e39df31bac3" + }, + "zigzag": { + "total_ms": 970.1784362792969, + "min_ms": 968.9087524414062, + "max_ms": 971.3029174804688, + "allocated_gib": 43.80655336380005, + "reserved_gib": 49.220703125, + "prepare_ms": 52.959115244448185, + "layout_prepare_ms": 52.52271518111229, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014112870474365484, + "dq": 0.0007248772146562982, + "dkv": 0.0007172076699478802, + "d_index_q": 0.0007868982065434454, + "d_weights": 0.0007947178818619413, + "d_pool_kv": 0.0009190684436244239, + "d_pool_score": 0.000898462375877342, + "d_ape": 5.798520906604598e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 2.9999621137976646, + "max_spans": 900, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-zigzag.json", + "raw_sha256": "a5e3c36bc6ff75b804f68f235bbdc01ca47ae018f6e4a2d37f1f1527631586ce" + } + }, + "speedup_vs_contiguous": 1.4044325762256893, + "gap_to_best_baseline_pct": -1.0631752757959245, + "prepare_plus_iteration_ms": { + "contiguous": 1357.692121825181, + "planner": 1022.2074949834496, + "zigzag": 1023.1375515237451 + }, + "prepare_inclusive_gap_pct": -0.09090239517749454, + "documents": 768, + "lengths_sha256": "747a3b346f6fb67d2a1bda28b2c1431fae38be2adb70abab6f64c52541fce124" + }, + "1m-heldout-tiny-tail-768-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1590.6795043945312, + "min_ms": 1584.950927734375, + "max_ms": 1595.4150390625, + "allocated_gib": 42.541175365448, + "reserved_gib": 48.34375, + "prepare_ms": 10.44242299394682, + "layout_prepare_ms": 9.98386600986123, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999874122440815, + "max_spans": 192, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-contiguous.json", + "raw_sha256": "d40a92d0689cf9f689f3c42b6b3bdbd185f0eb0158a1d35fc12a603622060f7a" + }, + "planner": { + "total_ms": 1043.1842651367188, + "min_ms": 1041.0396728515625, + "max_ms": 1050.0198974609375, + "allocated_gib": 46.02216053009033, + "reserved_gib": 52.56640625, + "prepare_ms": 73.80954397376627, + "layout_prepare_ms": 73.403590999078, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0013978769968494877, + "dq": 0.0007126548832640084, + "dkv": 0.0005994592647928282, + "d_index_q": 0.0006814399394547405, + "d_weights": 0.0007170806907142871, + "d_pool_kv": 0.0008913555060706143, + "d_pool_score": 0.0008884188541226811, + "d_ape": 6.953670537953373e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999817803502083, + "max_spans": 194, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-planner.json", + "raw_sha256": "5780aacb4562ec565e2a2a011af8572cd0f482563ff53094b55f7d10c9a5f0f5" + }, + "zigzag": { + "total_ms": 1054.1594848632812, + "min_ms": 1053.013916015625, + "max_ms": 1055.3916015625, + "allocated_gib": 45.27772665023804, + "reserved_gib": 50.9765625, + "prepare_ms": 62.31703143566847, + "layout_prepare_ms": 61.90841551870108, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013928374079413585, + "dq": 0.0007163195568851283, + "dkv": 0.0006183839712719321, + "d_index_q": 0.0007805775784666594, + "d_weights": 0.0007601769275057709, + "d_pool_kv": 0.0008918821804395749, + "d_pool_score": 0.0008784295268673588, + "d_ape": 7.135205817860119e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.9999601282179356, + "max_spans": 892, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-zigzag.json", + "raw_sha256": "f7c6e53ffc1874b39853489c2911068ce53115b45f331148f965adda8dbba092" + } + }, + "speedup_vs_contiguous": 1.5248308065555976, + "gap_to_best_baseline_pct": -1.041134656013265, + "prepare_plus_iteration_ms": { + "contiguous": 1601.121927388478, + "planner": 1116.993809110485, + "zigzag": 1116.4765162989497 + }, + "prepare_inclusive_gap_pct": 0.04633261909081643, + "documents": 768, + "lengths_sha256": "e879dc84790636c8379bc654c1bf98f0f184183c998e485cd8cf5a1df8e1e686" + }, + "1m-heldout-trimodal-513-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1590.93896484375, + "min_ms": 1587.810302734375, + "max_ms": 1597.840576171875, + "allocated_gib": 42.64446830749512, + "reserved_gib": 48.48828125, + "prepare_ms": 7.761615968775004, + "layout_prepare_ms": 7.369155006017536, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.999986007809639, + "max_spans": 158, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-contiguous.json", + "raw_sha256": "b79765dd4a8f05f94432c920fec00478564e03860cc14e4701403733835e6341" + }, + "planner": { + "total_ms": 1123.5488891601562, + "min_ms": 1121.196533203125, + "max_ms": 1124.972412109375, + "allocated_gib": 44.79515075683594, + "reserved_gib": 51.78125, + "prepare_ms": 72.7891989517957, + "layout_prepare_ms": 72.46607198612764, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0014105699808921169, + "dq": 0.0007211853655796001, + "dkv": 0.00076317319296445, + "d_index_q": 0.0008273633359754639, + "d_weights": 0.000839848530016898, + "d_pool_kv": 0.0010063930178328682, + "d_pool_score": 0.0009932757448149157, + "d_ape": 7.00794864677479e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.999897550791502, + "max_spans": 162, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-planner.json", + "raw_sha256": "819ca7d68179708b71824ddac20148db07f896b4f26e0964df375bcb645ad7d6" + }, + "zigzag": { + "total_ms": 1143.2892456054688, + "min_ms": 1139.7706298828125, + "max_ms": 1147.6728515625, + "allocated_gib": 44.624155044555664, + "reserved_gib": 50.60546875, + "prepare_ms": 45.48390599666163, + "layout_prepare_ms": 45.11003097286448, + "policy": "zigzag", + "relative_rms": { + "output": 0.001423907749754376, + "dq": 0.0007267030659625826, + "dkv": 0.0008249438553013025, + "d_index_q": 0.0007623419003029103, + "d_weights": 0.0007973129116156198, + "d_pool_kv": 0.0010020190272064045, + "d_pool_score": 0.0009955530954732558, + "d_ape": 6.664383819341363e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999913454055786, + "max_spans": 616, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-zigzag.json", + "raw_sha256": "28325435bf8b7964b07d4f4716e6f170883b247d4e3cc4408f22a533473c1c24" + } + }, + "speedup_vs_contiguous": 1.4159944264045012, + "gap_to_best_baseline_pct": -1.726628368209504, + "prepare_plus_iteration_ms": { + "contiguous": 1598.700580812525, + "planner": 1196.338088111952, + "zigzag": 1188.7731516021304 + }, + "prepare_inclusive_gap_pct": 0.636365020494134, + "documents": 513, + "lengths_sha256": "3e4264b20b69bf872e0287259db21e68128815011e3a225f7f002b5e26266a7b" + }, + "1m-heldout-trimodal-513-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1565.771240234375, + "min_ms": 1562.936767578125, + "max_ms": 1571.0311279296875, + "allocated_gib": 42.291651248931885, + "reserved_gib": 47.84375, + "prepare_ms": 7.783631037455052, + "layout_prepare_ms": 7.3381830006837845, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999883584678173, + "max_spans": 140, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-contiguous.json", + "raw_sha256": "865ef22abb24e72f9ac9bc8df39d87636f0b8b2c00487563c5ee7d5517132236" + }, + "planner": { + "total_ms": 1138.5536499023438, + "min_ms": 1137.5244140625, + "max_ms": 1141.59814453125, + "allocated_gib": 43.47452640533447, + "reserved_gib": 50.501953125, + "prepare_ms": 65.18473598407581, + "layout_prepare_ms": 64.8464719997719, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0014311802780475353, + "dq": 0.0007300040993747967, + "dkv": 0.0007274725527564618, + "d_index_q": 0.0007625024247133749, + "d_weights": 0.0007744764822715035, + "d_pool_kv": 0.001020879848512074, + "d_pool_score": 0.0009976675758443746, + "d_ape": 7.196140335950982e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 2.9999768882989883, + "max_spans": 144, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-planner.json", + "raw_sha256": "6dca6f101f5597b0bdda06a3fcb03153c2ffa2db0bb9f1d338a6b9df97968853" + }, + "zigzag": { + "total_ms": 1154.8492431640625, + "min_ms": 1152.441650390625, + "max_ms": 1156.752685546875, + "allocated_gib": 44.93105983734131, + "reserved_gib": 50.974609375, + "prepare_ms": 44.415701006073505, + "layout_prepare_ms": 44.05278398189694, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014446751397546181, + "dq": 0.0007365494400272126, + "dkv": 0.0006853989864517794, + "d_index_q": 0.0008626266028998927, + "d_weights": 0.0009449086875199775, + "d_pool_kv": 0.0009599962727863646, + "d_pool_score": 0.0009406345768126211, + "d_ape": 6.54012337861862e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 3.9999019987881184, + "max_spans": 616, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-zigzag.json", + "raw_sha256": "e84e2afa237164871da19461e39333b654d6d8e7d12c39de24ff5e995b28035c" + } + }, + "speedup_vs_contiguous": 1.3752283349745307, + "gap_to_best_baseline_pct": -1.4110580543891582, + "prepare_plus_iteration_ms": { + "contiguous": 1573.55487127183, + "planner": 1203.7383858864196, + "zigzag": 1199.264944170136 + }, + "prepare_inclusive_gap_pct": 0.3730152989153801, + "documents": 513, + "lengths_sha256": "d11985e121377244a83f6693db589940ae4899c1bf48610d0e00341a06f525bd" + }, + "1m-heldout-trimodal-513-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1917.0702514648438, + "min_ms": 1912.1605224609375, + "max_ms": 1927.1201171875, + "allocated_gib": 43.041377544403076, + "reserved_gib": 49.6796875, + "prepare_ms": 13.605939981061965, + "layout_prepare_ms": 13.198564003687352, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.4999979436397552, + "max_spans": 125, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-contiguous.json", + "raw_sha256": "757713ac12769cf75f84e27b9fc0bad211974344e489f5dd20775b2b8514074a" + }, + "planner": { + "total_ms": 1195.0348510742188, + "min_ms": 1192.755615234375, + "max_ms": 1196.207275390625, + "allocated_gib": 45.410380840301514, + "reserved_gib": 49.587890625, + "prepare_ms": 62.280201993416995, + "layout_prepare_ms": 61.91658799070865, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.001453102960327942, + "dq": 0.0007394295376428531, + "dkv": 0.0006929703357612946, + "d_index_q": 0.0009285016240026516, + "d_weights": 0.0008356778914921855, + "d_pool_kv": 0.0009733506318998237, + "d_pool_score": 0.0009755384220336541, + "d_ape": 5.943132432689433e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 4.99997153878212, + "max_spans": 129, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-planner.json", + "raw_sha256": "313bc655ca0f1021f66224cbb9a4a96431c19bb13b3a695417a065b5ed3b5c6a" + }, + "zigzag": { + "total_ms": 1225.0924682617188, + "min_ms": 1222.8372802734375, + "max_ms": 1226.3341064453125, + "allocated_gib": 46.19820690155029, + "reserved_gib": 51.845703125, + "prepare_ms": 48.36773098213598, + "layout_prepare_ms": 47.98452300019562, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014586838423307257, + "dq": 0.000749034668781768, + "dkv": 0.0008060485699208124, + "d_index_q": 0.0008707924287472155, + "d_weights": 0.0008722376455450429, + "d_pool_kv": 0.0013020207402422673, + "d_pool_score": 0.0012741966886479401, + "d_ape": 6.281001290077127e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 4.999932385981083, + "max_spans": 616, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-zigzag.json", + "raw_sha256": "a444f890fab0b9c9e3b8c69cccefebca5a7b9c905700e58d5da5d6b9e9f9e078" + } + }, + "speedup_vs_contiguous": 1.6041961033534597, + "gap_to_best_baseline_pct": -2.453497835159224, + "prepare_plus_iteration_ms": { + "contiguous": 1930.6761914459057, + "planner": 1257.3150530676357, + "zigzag": 1273.4601992438547 + }, + "prepare_inclusive_gap_pct": -1.267817100668367, + "documents": 513, + "lengths_sha256": "30e8a9b80449ecc324bba6134d04db805dac074f3b7693579d6f2afb153135f7" + }, + "1m-near-uniform": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 657.8585815429688, + "min_ms": 657.422119140625, + "max_ms": 658.462158203125, + "allocated_gib": 38.75661849975586, + "reserved_gib": 45.703125, + "prepare_ms": 2.901871979702264, + "layout_prepare_ms": 2.6385170058347285, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.3607950210571289, + "max_spans": 4, + "raw_file": "layout-1m-near-uniform-contiguous.json", + "raw_sha256": "4839fa9922f01fd5680c089e3b287c4e3dfea81d886b85f0b416cfde52d871d7" + }, + "planner": { + "total_ms": 657.8680419921875, + "min_ms": 657.4669189453125, + "max_ms": 658.1314697265625, + "allocated_gib": 38.75661849975586, + "reserved_gib": 45.703125, + "prepare_ms": 2.4814242497086525, + "layout_prepare_ms": 2.266555093228817, + "policy": "contiguous", + "relative_rms": { + "output": 0.0012094462774270827, + "dq": 0.0006690625087143198, + "dkv": 0.00050289701248222, + "d_index_q": 0.00048048109350015526, + "d_weights": 0.0003585406867666776, + "d_pool_kv": 0.000397641522095392, + "d_pool_score": 0.00037759271103066154, + "d_ape": 8.173075437996835e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.3607950210571289, + "max_spans": 4, + "raw_file": "layout-1m-near-uniform-planner.json", + "raw_sha256": "7c0b7122a26409e447f407526a46a39586a9b82b65095a27c193733633e0167c" + }, + "zigzag": { + "total_ms": 691.0231323242188, + "min_ms": 690.33544921875, + "max_ms": 692.6437377929688, + "allocated_gib": 38.89805316925049, + "reserved_gib": 45.986328125, + "prepare_ms": 14.919791370630264, + "layout_prepare_ms": 14.696252532303333, + "policy": "zigzag", + "relative_rms": { + "output": 0.0012156282361116832, + "dq": 0.0006701063181955824, + "dkv": 0.0004998448482890624, + "d_index_q": 0.0003623673282831218, + "d_weights": 0.0003377786590599061, + "d_pool_kv": 0.0003491324172495882, + "d_pool_score": 0.0003654630032104385, + "d_ape": 7.500653892418262e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.4973450005054474, + "max_spans": 128, + "raw_file": "layout-1m-near-uniform-zigzag.json", + "raw_sha256": "8e09ad7be40af8ce81ae3a2b8b76cfdab794b1a3e02270d70ee82dbc5c85ea16" + } + }, + "speedup_vs_contiguous": 0.9999856195336833, + "gap_to_best_baseline_pct": 0.001438067311765323, + "prepare_plus_iteration_ms": { + "contiguous": 660.760453522671, + "planner": 660.3494662418962, + "zigzag": 705.942923694849 + }, + "prepare_inclusive_gap_pct": -0.06219913413155753, + "documents": 64, + "lengths_sha256": "f7b3e262fd896136c35c22e4766cae5a4b416484124973745b17cbc8ceac3850" + }, + "1m-power-law": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1788.4425659179688, + "min_ms": 1784.1978759765625, + "max_ms": 1796.81005859375, + "allocated_gib": 42.91158723831177, + "reserved_gib": 47.84375, + "prepare_ms": 3.3689523115754128, + "layout_prepare_ms": 3.1191427260637283, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999989978969097, + "max_spans": 4, + "raw_file": "layout-1m-power-law-contiguous.json", + "raw_sha256": "ce4146f1ee3890865780f6784326c03488e5affca35da0a224ee79f45d3043fa" + }, + "planner": { + "total_ms": 1195.981201171875, + "min_ms": 1193.8533935546875, + "max_ms": 1202.4031982421875, + "allocated_gib": 42.91142702102661, + "reserved_gib": 47.951171875, + "prepare_ms": 64.25684876739979, + "layout_prepare_ms": 64.0103267505765, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.00141111190665047, + "dq": 0.0007189686564165684, + "dkv": 0.000768608027980026, + "d_index_q": 0.000982448407481547, + "d_weights": 0.0009287907382984111, + "d_pool_kv": 0.0010326563504508956, + "d_pool_score": 0.0010357392149652606, + "d_ape": 1.2755589527582849e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.999922826886177, + "max_spans": 11, + "raw_file": "layout-1m-power-law-planner.json", + "raw_sha256": "ed160493d20ca4d75fedbdf8083b23940a454980ff880cff8ca5c6925f5a428a" + }, + "zigzag": { + "total_ms": 1200.6126708984375, + "min_ms": 1199.58984375, + "max_ms": 1202.20849609375, + "allocated_gib": 42.416353702545166, + "reserved_gib": 47.84765625, + "prepare_ms": 10.712767951190472, + "layout_prepare_ms": 10.503863915801048, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013937638936558827, + "dq": 0.0007097413346802662, + "dkv": 0.0008191421111825079, + "d_index_q": 0.0008236355513542162, + "d_weights": 0.0008566196450316358, + "d_pool_kv": 0.001021948285677823, + "d_pool_score": 0.0010267541408195688, + "d_ape": 1.1541836768590658e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 1.9999937117099762, + "max_spans": 14, + "raw_file": "layout-1m-power-law-zigzag.json", + "raw_sha256": "ecf17cd4721ee756c606f9097f902764309154dcb89f9046d88de584c1b3f6ac" + } + }, + "speedup_vs_contiguous": 1.4953768204429752, + "gap_to_best_baseline_pct": -0.38575885785852027, + "prepare_plus_iteration_ms": { + "contiguous": 1791.8115182295442, + "planner": 1260.2380499392748, + "zigzag": 1211.325438849628 + }, + "prepare_inclusive_gap_pct": 4.03794137569653, + "documents": 7, + "lengths_sha256": "b787057eb34fe32408b2fe31d07c9bc6642c12e889e2e2a42e790f32a7ce58d8" + }, + "1m-single": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 2908.836669921875, + "min_ms": 2895.02099609375, + "max_ms": 2922.881103515625, + "allocated_gib": 44.911484241485596, + "reserved_gib": 49.34375, + "prepare_ms": 5.531359056476504, + "layout_prepare_ms": 5.292983027175069, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999951247125864, + "max_spans": 1, + "raw_file": "layout-1m-single-contiguous.json", + "raw_sha256": "bdbbfaac7bef04b46a9356c86ea45d5723b298e95ccd426c7ea2b748e7af47cd" + }, + "planner": { + "total_ms": 1926.761962890625, + "min_ms": 1924.40087890625, + "max_ms": 1931.9456787109375, + "allocated_gib": 44.916353702545166, + "reserved_gib": 49.97265625, + "prepare_ms": 60.06631493801251, + "layout_prepare_ms": 59.86356601351872, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014038787207327147, + "dq": 0.0007154462314895009, + "dkv": 0.0006833642438317521, + "d_index_q": 0.0007211015679207741, + "d_weights": 0.0008126837022277005, + "d_pool_kv": 0.0009996668460791477, + "d_pool_score": 0.0009953564906126419, + "d_ape": 1.471933089292079e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 3.2014213502407074e-10, + "workspace_gib": 3.999988541007042, + "max_spans": 2, + "raw_file": "layout-1m-single-planner.json", + "raw_sha256": "916b072e7cf7a7079caa13195995c67c587bba2a0b6715a365ff577f812e822f" + }, + "zigzag": { + "total_ms": 1934.275390625, + "min_ms": 1925.960205078125, + "max_ms": 1942.73828125, + "allocated_gib": 44.816391944885254, + "reserved_gib": 49.865234375, + "prepare_ms": 8.57052201172337, + "layout_prepare_ms": 8.357521030120552, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014390151905128753, + "dq": 0.000735736599938945, + "dkv": 0.0006618938605150542, + "d_index_q": 0.000861860733717636, + "d_weights": 0.0008535259313058091, + "d_pool_kv": 0.0010403120797250875, + "d_pool_score": 0.0010515623317991771, + "d_ape": 1.4940338041906002e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.4551915228366852e-10, + "workspace_gib": 3.999988541007042, + "max_spans": 2, + "raw_file": "layout-1m-single-zigzag.json", + "raw_sha256": "93948f98a3acecba7d0deaabfaf4f6c8e589d19e6247ffbf4a55316ad41596e9" + } + }, + "speedup_vs_contiguous": 1.5097021458519413, + "gap_to_best_baseline_pct": -0.3884362987189416, + "prepare_plus_iteration_ms": { + "contiguous": 2914.3680289783515, + "planner": 1986.8282778286375, + "zigzag": 1942.8459126367234 + }, + "prepare_inclusive_gap_pct": 2.263811293826379, + "documents": 1, + "lengths_sha256": "df57f4ef94807b0499d37da5e83d5c5333272a5ebcad42ac3dbec6ac94a30bf2" + }, + "1m-uniform": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 653.5117797851562, + "min_ms": 653.2141723632812, + "max_ms": 658.2883911132812, + "allocated_gib": 38.895822048187256, + "reserved_gib": 45.234375, + "prepare_ms": 3.467416390776634, + "layout_prepare_ms": 3.2254476100206375, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49993896484375, + "max_spans": 4, + "raw_file": "layout-1m-uniform-contiguous.json", + "raw_sha256": "6e865c7f586f9301303f2978436088bbabe9721f239ecfc52fe9935174fcd326" + }, + "planner": { + "total_ms": 657.158935546875, + "min_ms": 656.90869140625, + "max_ms": 657.7572021484375, + "allocated_gib": 38.895822048187256, + "reserved_gib": 45.234375, + "prepare_ms": 2.456046990118921, + "layout_prepare_ms": 2.2205039858818054, + "policy": "contiguous", + "relative_rms": { + "output": 0.001222877084560653, + "dq": 0.0006775747367604897, + "dkv": 0.0005307056800977869, + "d_index_q": 0.00037191161926160743, + "d_weights": 0.00033510128340201917, + "d_pool_kv": 0.000352034253727824, + "d_pool_score": 0.000378997248118142, + "d_ape": 7.817866314212561e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49993896484375, + "max_spans": 4, + "raw_file": "layout-1m-uniform-planner.json", + "raw_sha256": "5a4b2455ed9656af314a3a2f0fc539935d551041e550d382f50644f680352a1f" + }, + "zigzag": { + "total_ms": 690.3211975097656, + "min_ms": 689.617431640625, + "max_ms": 690.9261474609375, + "allocated_gib": 38.90070724487305, + "reserved_gib": 45.986328125, + "prepare_ms": 90.23720677942038, + "layout_prepare_ms": 90.0002857670188, + "policy": "zigzag", + "relative_rms": { + "output": 0.0012208882109481808, + "dq": 0.0006700987030479299, + "dkv": 0.0005020949017264413, + "d_index_q": 0.0003544317603985004, + "d_weights": 0.0003364373695151482, + "d_pool_kv": 0.0003600490455313742, + "d_pool_score": 0.00036589665566245076, + "d_ape": 7.339361484699418e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49993896484375, + "max_spans": 128, + "raw_file": "layout-1m-uniform-zigzag.json", + "raw_sha256": "9e4cd5238a8c4b8f0b44732c138bbe8e870fe81f8ed269602153d5853f90e48a" + } + }, + "speedup_vs_contiguous": 0.9944501161523678, + "gap_to_best_baseline_pct": 0.5580856955505364, + "prepare_plus_iteration_ms": { + "contiguous": 656.9791961759329, + "planner": 659.6149825369939, + "zigzag": 780.558404289186 + }, + "prepare_inclusive_gap_pct": 0.40119784255012725, + "documents": 64, + "lengths_sha256": "dd1b660aba10a40fca5d0224dbbcfdec6e38447a6ae6c9aadae9aa252935fd2e" + } + } + }, + "b300_cp8": { + "world_size": 8, + "cases": { + "1m-20s2l": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 606.7195434570312, + "min_ms": 603.2216186523438, + "max_ms": 607.1049194335938, + "allocated_gib": 68.95169401168823, + "reserved_gib": 71.357421875, + "prepare_ms": 8.157755946740508, + "layout_prepare_ms": 7.820633007213473, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999997615814209, + "max_spans": 21, + "raw_file": "layout-1m-20s2l-contiguous.json", + "raw_sha256": "6e60bf1ead83ddf7d64e883c230ed1e7808be48a09e885988971479496baf23c" + }, + "planner": { + "total_ms": 523.0918579101562, + "min_ms": 521.7444458007812, + "max_ms": 524.9149780273438, + "allocated_gib": 71.21805715560913, + "reserved_gib": 81.74609375, + "prepare_ms": 25.480400072410703, + "layout_prepare_ms": 25.222072028554976, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.00114268760664425, + "dq": 0.0003795518734018198, + "dkv": 0.0004977505257510024, + "d_index_q": 0.0006354303346022228, + "d_weights": 0.0005348152030303236, + "d_pool_kv": 0.0006389127848179412, + "d_pool_score": 0.0006350322727140332, + "d_ape": 1.0398483891015381e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999911297112703, + "max_spans": 24, + "raw_file": "layout-1m-20s2l-planner.json", + "raw_sha256": "0d057d4d9bb3eae1c4ec23e927636eb0251e442a5592de852e5c8bf16c3384eb" + }, + "zigzag": { + "total_ms": 525.017578125, + "min_ms": 523.6755981445312, + "max_ms": 528.3817138671875, + "allocated_gib": 69.14970111846924, + "reserved_gib": 72.060546875, + "prepare_ms": 8.265939075499773, + "layout_prepare_ms": 7.958356058225036, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011220897239465242, + "dq": 0.0003615562187011278, + "dkv": 0.0005358599327513749, + "d_index_q": 0.0006301410117376193, + "d_weights": 0.0005489266029995818, + "d_pool_kv": 0.0006605039288327338, + "d_pool_score": 0.0006744354766970948, + "d_ape": 1.1430774781154517e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.9999856054782867, + "max_spans": 44, + "raw_file": "layout-1m-20s2l-zigzag.json", + "raw_sha256": "231c5cd7327bd55911a5ee839f891ca84e62b0004a55a2886176e160f17e915a" + } + }, + "speedup_vs_contiguous": 1.1598718930189094, + "gap_to_best_baseline_pct": -0.3667915694786972, + "prepare_plus_iteration_ms": { + "contiguous": 614.8772994037718, + "planner": 548.572257982567, + "zigzag": 533.2835172004998 + }, + "prepare_inclusive_gap_pct": 2.8669066807701427, + "documents": 22, + "lengths_sha256": "1fbc78fcb26714baab3313b04d1a82e7d4c3c5f2a707fddfa67b2c0893379098" + }, + "1m-count-mixed-1000-1long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 564.9183959960938, + "min_ms": 562.6470947265625, + "max_ms": 567.5447387695312, + "allocated_gib": 69.90887594223022, + "reserved_gib": 72.322265625, + "prepare_ms": 9.918121038936079, + "layout_prepare_ms": 9.384725010022521, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999988861382008, + "max_spans": 262, + "raw_file": "layout-1m-count-mixed-1000-1long-contiguous.json", + "raw_sha256": "09eddcebc906090d96c39180c67440f1a35211d78349812b6a948624cc0a4ab9" + }, + "planner": { + "total_ms": 466.26866149902344, + "min_ms": 465.1852722167969, + "max_ms": 466.9646301269531, + "allocated_gib": 72.88241815567017, + "reserved_gib": 85.33203125, + "prepare_ms": 37.03177708666772, + "layout_prepare_ms": 36.51334100868553, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0009270768706657761, + "dq": 0.00031215269029789, + "dkv": 0.00046040820270757207, + "d_index_q": 0.0004667214906071487, + "d_weights": 0.00046785144097795687, + "d_pool_kv": 0.0007079749914338807, + "d_pool_score": 0.0006910569760609635, + "d_ape": 3.5272129210754033e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 5.999798700213432, + "max_spans": 264, + "raw_file": "layout-1m-count-mixed-1000-1long-planner.json", + "raw_sha256": "bcb863f548a77b3da2675aa1f1476a4964e4226890939bdeeb4eb9d49a178e92" + }, + "zigzag": { + "total_ms": 480.1496124267578, + "min_ms": 479.004638671875, + "max_ms": 482.7588195800781, + "allocated_gib": 72.79625606536865, + "reserved_gib": 74.830078125, + "prepare_ms": 85.75332898180932, + "layout_prepare_ms": 85.18884400837123, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009499978231453273, + "dq": 0.00035834556965688485, + "dkv": 0.0005074644981803778, + "d_index_q": 0.0006145701352548918, + "d_weights": 0.0005349551113659254, + "d_pool_kv": 0.0006670691629931602, + "d_pool_score": 0.0006652888839264348, + "d_ape": 3.2772513683446074e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 5.999901540577412, + "max_spans": 2000, + "raw_file": "layout-1m-count-mixed-1000-1long-zigzag.json", + "raw_sha256": "8cd57f99a5c1ef4180c616f58a86bcc4b8e82c272b2bfdab18ca97f7d8215054" + } + }, + "speedup_vs_contiguous": 1.211572731866469, + "gap_to_best_baseline_pct": -2.890963684751857, + "prepare_plus_iteration_ms": { + "contiguous": 574.8365170350298, + "planner": 503.30043858569115, + "zigzag": 565.9029414085671 + }, + "prepare_inclusive_gap_pct": -11.062409866090206, + "documents": 1000, + "lengths_sha256": "9e67a0b0cf2c6f62c886060fcce16cad37d396e458fe8e31e2ea92de7569371d" + }, + "1m-count-mixed-256-1long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 646.4150695800781, + "min_ms": 639.7410278320312, + "max_ms": 655.7701416015625, + "allocated_gib": 70.46625900268555, + "reserved_gib": 72.87890625, + "prepare_ms": 6.630808115005493, + "layout_prepare_ms": 6.256341002881527, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 4.999974433332682, + "max_spans": 82, + "raw_file": "layout-1m-count-mixed-256-1long-contiguous.json", + "raw_sha256": "00e0cf7d510294d34b536ba15a711f2c242982c64dae404f8df89b142b09c14b" + }, + "planner": { + "total_ms": 494.0136413574219, + "min_ms": 492.4333190917969, + "max_ms": 495.97784423828125, + "allocated_gib": 74.01397752761841, + "reserved_gib": 85.20703125, + "prepare_ms": 18.05205293931067, + "layout_prepare_ms": 17.699307994917035, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0008858106682604503, + "dq": 0.00045696975731111025, + "dkv": 0.0006497224056940322, + "d_index_q": 0.0006953071842500477, + "d_weights": 0.0006144128886141594, + "d_pool_kv": 0.0008463621505370647, + "d_pool_score": 0.0008385926850077365, + "d_ape": 7.137320680575466e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 7.999998513609171, + "max_spans": 84, + "raw_file": "layout-1m-count-mixed-256-1long-planner.json", + "raw_sha256": "839a52e070e1e1f5f24fbfac53fcd848de76b3cad5141e8afd574f262278abf3" + }, + "zigzag": { + "total_ms": 509.91094970703125, + "min_ms": 507.9485168457031, + "max_ms": 510.84820556640625, + "allocated_gib": 70.80725765228271, + "reserved_gib": 73.716796875, + "prepare_ms": 21.465071011334658, + "layout_prepare_ms": 21.089443005621433, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008941719266047247, + "dq": 0.00045941777187373734, + "dkv": 0.0005930797558927764, + "d_index_q": 0.0006706491690614671, + "d_weights": 0.0005955728663222536, + "d_pool_kv": 0.0008320355873393238, + "d_pool_score": 0.0008225265210542579, + "d_ape": 6.629130868855286e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 4.999996654689312, + "max_spans": 512, + "raw_file": "layout-1m-count-mixed-256-1long-zigzag.json", + "raw_sha256": "8e8b0a3e7aafb5f925681b06acf0c4ede83d0cc9e2f163a550714ec0753f757b" + } + }, + "speedup_vs_contiguous": 1.3084963965851155, + "gap_to_best_baseline_pct": -3.1176636545544145, + "prepare_plus_iteration_ms": { + "contiguous": 653.0458776950836, + "planner": 512.0656942967325, + "zigzag": 531.3760207183659 + }, + "prepare_inclusive_gap_pct": -3.6340229270277913, + "documents": 256, + "lengths_sha256": "013b49129cf177ac7e785bf7126e9e6899c7405866ddd6a51c91d4aec66cc87e" + }, + "1m-count-mixed-512-2long-ascending": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 466.8989715576172, + "min_ms": 466.17083740234375, + "max_ms": 468.3266906738281, + "allocated_gib": 69.80380439758301, + "reserved_gib": 84.18359375, + "prepare_ms": 22.160860942676663, + "layout_prepare_ms": 21.723491954617202, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999922811985016, + "max_spans": 432, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-contiguous.json", + "raw_sha256": "9e87c991926de8af250f166e40a77b1f38cce04235dbcc5d3e3595c64f6c959d" + }, + "planner": { + "total_ms": 421.4059600830078, + "min_ms": 419.9235534667969, + "max_ms": 423.1059875488281, + "allocated_gib": 73.5907506942749, + "reserved_gib": 88.091796875, + "prepare_ms": 26.00089693441987, + "layout_prepare_ms": 25.468059931881726, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0011625036828467048, + "dq": 0.00048191146093208825, + "dkv": 0.0005925134624210264, + "d_index_q": 0.0005005708418548132, + "d_weights": 0.0004610351395769133, + "d_pool_kv": 0.0006021968088168659, + "d_pool_score": 0.0005987209876557188, + "d_ape": 5.704240967664213e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.778191454708576, + "max_spans": 435, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-planner.json", + "raw_sha256": "ea633766e5684664cc4b4390e982912be90d03e5688c48cf1d2c410af16394a7" + }, + "zigzag": { + "total_ms": 434.58563232421875, + "min_ms": 432.279052734375, + "max_ms": 437.4580993652344, + "allocated_gib": 71.56598663330078, + "reserved_gib": 73.9140625, + "prepare_ms": 55.45506288763136, + "layout_prepare_ms": 54.934649961069226, + "policy": "zigzag", + "relative_rms": { + "output": 0.001145584907702924, + "dq": 0.0004733690371043385, + "dkv": 0.0005199240863022013, + "d_index_q": 0.0005926175683857409, + "d_weights": 0.0005237273746309723, + "d_pool_kv": 0.0006443768894823862, + "d_pool_score": 0.0006404921576780437, + "d_ape": 5.340775707345109e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.9998051151633263, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-zigzag.json", + "raw_sha256": "46185171c19639efdff42f646abb228b268b7538d5f55ab168009f12267a3ec6" + } + }, + "speedup_vs_contiguous": 1.1079553109919191, + "gap_to_best_baseline_pct": -3.032698566384806, + "prepare_plus_iteration_ms": { + "contiguous": 489.05983250029385, + "planner": 447.4068570174277, + "zigzag": 490.0406952118501 + }, + "prepare_inclusive_gap_pct": -8.516948789254975, + "documents": 512, + "lengths_sha256": "c8ef7a47c8782c941def4feb26285b36f3ad51a989a3c052ec4e3fd7e8677f0e" + }, + "1m-count-mixed-512-2long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 441.9642791748047, + "min_ms": 439.4814758300781, + "max_ms": 444.86492919921875, + "allocated_gib": 67.59718465805054, + "reserved_gib": 70.009765625, + "prepare_ms": 6.5072940196841955, + "layout_prepare_ms": 6.10797805711627, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999815858900547, + "max_spans": 128, + "raw_file": "layout-1m-count-mixed-512-2long-contiguous.json", + "raw_sha256": "6da77918492109977a8174da04e87b5773ed494d66935b745ac20e5a1651341b" + }, + "planner": { + "total_ms": 407.34478759765625, + "min_ms": 404.76300048828125, + "max_ms": 410.2059326171875, + "allocated_gib": 69.54749727249146, + "reserved_gib": 81.146484375, + "prepare_ms": 32.38745790440589, + "layout_prepare_ms": 32.009240007027984, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0009130715838757859, + "dq": 0.0003953446079186865, + "dkv": 0.0005196069178357984, + "d_index_q": 0.0004475866982705158, + "d_weights": 0.0004382092502930036, + "d_pool_kv": 0.0006609964862132346, + "d_pool_score": 0.0006553244279760129, + "d_ape": 5.201641577139356e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 2.999816305935383, + "max_spans": 132, + "raw_file": "layout-1m-count-mixed-512-2long-planner.json", + "raw_sha256": "684838bb2153ab31197ba4fd95b7a86a2568fc9c8b8a2d26e1440f56b0318aa4" + }, + "zigzag": { + "total_ms": 421.9330749511719, + "min_ms": 420.48248291015625, + "max_ms": 424.0693359375, + "allocated_gib": 68.32417440414429, + "reserved_gib": 71.236328125, + "prepare_ms": 40.53927294444293, + "layout_prepare_ms": 40.1057000271976, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008742053584959008, + "dq": 0.00038871127807538956, + "dkv": 0.0006647450765884712, + "d_index_q": 0.0005054744827599346, + "d_weights": 0.00046845562206280025, + "d_pool_kv": 0.0007495320656450234, + "d_pool_score": 0.0007417334133141131, + "d_ape": 5.2839636515700575e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 2.9998051151633263, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-2long-zigzag.json", + "raw_sha256": "fc7f6bc7a78e939c6e007c5d7d6d844e526b54be053c23ba56a787fa19d3beb4" + } + }, + "speedup_vs_contiguous": 1.084988178641782, + "gap_to_best_baseline_pct": -3.45748845482281, + "prepare_plus_iteration_ms": { + "contiguous": 448.4715731944889, + "planner": 439.73224550206214, + "zigzag": 462.4723478956148 + }, + "prepare_inclusive_gap_pct": -1.9486915592388576, + "documents": 512, + "lengths_sha256": "21794dd7586d487ed826710eb069c8ec277ff51b5c88db0b0400e835d0419bcd" + }, + "1m-count-mixed-512-2long-descending": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 466.68701171875, + "min_ms": 464.418212890625, + "max_ms": 470.1160583496094, + "allocated_gib": 69.80540466308594, + "reserved_gib": 84.1796875, + "prepare_ms": 10.136506985872984, + "layout_prepare_ms": 9.61173593532294, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999922811985016, + "max_spans": 432, + "raw_file": "layout-1m-count-mixed-512-2long-descending-contiguous.json", + "raw_sha256": "a1e5aff7a16615d0379729e41f357707511f7f08670af3e2ae65f0f2f5826a1b" + }, + "planner": { + "total_ms": 412.10789489746094, + "min_ms": 410.8602600097656, + "max_ms": 413.2406005859375, + "allocated_gib": 73.81171321868896, + "reserved_gib": 88.30859375, + "prepare_ms": 35.53258301690221, + "layout_prepare_ms": 34.98343599494547, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0011776489787716086, + "dq": 0.0004764408764781611, + "dkv": 0.0005078191487493045, + "d_index_q": 0.0005257317147264598, + "d_weights": 0.0004880231426663937, + "d_pool_kv": 0.0006782394860014693, + "d_pool_score": 0.0006898723741323015, + "d_ape": 5.235521354946445e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.998665437102318, + "max_spans": 435, + "raw_file": "layout-1m-count-mixed-512-2long-descending-planner.json", + "raw_sha256": "38e8a91ba4d35806c6dff4952478cfee49fe069d45b658e879f0141d21d2ddb6" + }, + "zigzag": { + "total_ms": 430.64112854003906, + "min_ms": 427.5303955078125, + "max_ms": 433.921630859375, + "allocated_gib": 71.83758449554443, + "reserved_gib": 87.01953125, + "prepare_ms": 68.68545711040497, + "layout_prepare_ms": 68.15863598603755, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011748686063494781, + "dq": 0.00047520037171146624, + "dkv": 0.0005415161789643414, + "d_index_q": 0.0005674451612003116, + "d_weights": 0.00048600187424832343, + "d_pool_kv": 0.0005817740228488663, + "d_pool_score": 0.0005603234157210047, + "d_ape": 5.432171671313229e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.9998051151633263, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-2long-descending-zigzag.json", + "raw_sha256": "8055ef0fefe8d48eb4a8e5f0ef8a6632ed7eac920f7deb986023511dc0b57a1b" + } + }, + "speedup_vs_contiguous": 1.132438901309739, + "gap_to_best_baseline_pct": -4.303637626394297, + "prepare_plus_iteration_ms": { + "contiguous": 476.823518704623, + "planner": 447.64047791436315, + "zigzag": 499.32658565044403 + }, + "prepare_inclusive_gap_pct": -6.120302301686131, + "documents": 512, + "lengths_sha256": "1355e0fb1ef1d83a855305c16e873372467e69d68007d35dcefa354ea7c6ab1b" + }, + "1m-count-mixed-512-4long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 338.35548400878906, + "min_ms": 337.0683898925781, + "max_ms": 339.7387390136719, + "allocated_gib": 66.59222507476807, + "reserved_gib": 69.00390625, + "prepare_ms": 8.868525037541986, + "layout_prepare_ms": 8.5072279907763, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 128, + "raw_file": "layout-1m-count-mixed-512-4long-contiguous.json", + "raw_sha256": "2dc7e20d38e92e28ab857def38a7a1baa59daa25e1114930adc86faa455c401a" + }, + "planner": { + "total_ms": 338.45623779296875, + "min_ms": 336.9595031738281, + "max_ms": 341.4984436035156, + "allocated_gib": 66.59222507476807, + "reserved_gib": 69.00390625, + "prepare_ms": 19.420458003878593, + "layout_prepare_ms": 19.02695291209966, + "policy": "contiguous", + "relative_rms": { + "output": 0.001108400347265694, + "dq": 0.00041324557296966425, + "dkv": 0.0004567346009868542, + "d_index_q": 0.0002833367986887155, + "d_weights": 0.00024968591781364885, + "d_pool_kv": 0.00032218307925626635, + "d_pool_score": 0.0003184380093125898, + "d_ape": 5.236217222337733e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 128, + "raw_file": "layout-1m-count-mixed-512-4long-planner.json", + "raw_sha256": "6f84a09f175260bf25c2595c7071ee4c324b11e42965c5d7fedd606ef3207465" + }, + "zigzag": { + "total_ms": 338.5984802246094, + "min_ms": 337.65234375, + "max_ms": 342.5445556640625, + "allocated_gib": 66.80190896987915, + "reserved_gib": 69.716796875, + "prepare_ms": 37.792065064422786, + "layout_prepare_ms": 37.35221491660923, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011215964857729552, + "dq": 0.00041829052447210673, + "dkv": 0.0004561462148145632, + "d_index_q": 0.00033356936908467654, + "d_weights": 0.0002592224993963174, + "d_pool_kv": 0.0003434961985130709, + "d_pool_score": 0.0003701823394787622, + "d_ape": 5.452835218376787e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.49999136850237846, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-4long-zigzag.json", + "raw_sha256": "84ee23ee7a5bc2e3bc5ad747b70f63d8b668af58cd1869aefb51165e623fe8b8" + } + }, + "speedup_vs_contiguous": 0.9997023137028388, + "gap_to_best_baseline_pct": 0.02977749406807284, + "prepare_plus_iteration_ms": { + "contiguous": 347.22400904633105, + "planner": 357.87669579684734, + "zigzag": 376.39054528903216 + }, + "prepare_inclusive_gap_pct": 3.0679579962729076, + "documents": 512, + "lengths_sha256": "74f528dba1eacc05b1dd3ae32df6d6e5fcba98ba8abc06fecf0c0df6bf0aa380" + }, + "1m-count-near-uniform-512": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 174.68292236328125, + "min_ms": 173.8264923095703, + "max_ms": 177.15196228027344, + "allocated_gib": 65.79301404953003, + "reserved_gib": 68.0859375, + "prepare_ms": 6.28169986885041, + "layout_prepare_ms": 5.795156001113355, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.1400146484375, + "max_spans": 70, + "raw_file": "layout-1m-count-near-uniform-512-contiguous.json", + "raw_sha256": "46b66a22406b73c9ade46c9389b8981bb27ec8a8f1bfe6b3e1fa4d17215ad402" + }, + "planner": { + "total_ms": 174.54737854003906, + "min_ms": 173.66610717773438, + "max_ms": 175.65142822265625, + "allocated_gib": 65.79301404953003, + "reserved_gib": 68.0859375, + "prepare_ms": 6.889444077387452, + "layout_prepare_ms": 6.43392198253423, + "policy": "contiguous", + "relative_rms": { + "output": 0.0003621722687541866, + "dq": 0.0001877717766341684, + "dkv": 0.00015745879408379858, + "d_index_q": 5.0233157871382964e-05, + "d_weights": 4.737857482285738e-05, + "d_pool_kv": 0.0001252604413295691, + "d_pool_score": 0.0001366667434959041, + "d_ape": 2.009941592126655e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.1400146484375, + "max_spans": 70, + "raw_file": "layout-1m-count-near-uniform-512-planner.json", + "raw_sha256": "913cceeccdd10800dd4e912a1d564ef8bb61dad70ae81a422a862a906e3a8c9b" + }, + "zigzag": { + "total_ms": 185.5495834350586, + "min_ms": 185.0115203857422, + "max_ms": 186.0485382080078, + "allocated_gib": 65.8086576461792, + "reserved_gib": 68.587890625, + "prepare_ms": 40.83809698931873, + "layout_prepare_ms": 40.41551903355867, + "policy": "zigzag", + "relative_rms": { + "output": 0.00035880839521131567, + "dq": 0.00018598617770927155, + "dkv": 0.00015672983164195206, + "d_index_q": 4.980354544334212e-05, + "d_weights": 4.6031241434262136e-05, + "d_pool_kv": 0.00012127384059513735, + "d_pool_score": 0.0001278846890799026, + "d_ape": 2.0326038418112683e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.1326904296875, + "max_spans": 1024, + "raw_file": "layout-1m-count-near-uniform-512-zigzag.json", + "raw_sha256": "1cc9d1f19a5a539dc40fe26386924bd4ac6896b9d1eca72fe3f782ccd62ae421" + } + }, + "speedup_vs_contiguous": 1.0007765445942294, + "gap_to_best_baseline_pct": -0.07759420406323603, + "prepare_plus_iteration_ms": { + "contiguous": 180.96462223213166, + "planner": 181.43682261742651, + "zigzag": 226.38768042437732 + }, + "prepare_inclusive_gap_pct": 0.26093519245389274, + "documents": 512, + "lengths_sha256": "a883d69122302c1f8f9693929fa0ea899a3e5e33acb0a13cd8340e29afc198a0" + }, + "1m-count-tiny-mix-1000": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 277.5030212402344, + "min_ms": 276.7580871582031, + "max_ms": 278.6227111816406, + "allocated_gib": 67.28772163391113, + "reserved_gib": 69.443359375, + "prepare_ms": 14.695362886413932, + "layout_prepare_ms": 14.116181991994381, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 174, + "raw_file": "layout-1m-count-tiny-mix-1000-contiguous.json", + "raw_sha256": "80d08f5ab5f1e35f442698c25b83751be87ef5ce4224b955890777d7182d1219" + }, + "planner": { + "total_ms": 277.97694396972656, + "min_ms": 276.58367919921875, + "max_ms": 281.081787109375, + "allocated_gib": 67.28772163391113, + "reserved_gib": 69.443359375, + "prepare_ms": 12.267046957276762, + "layout_prepare_ms": 11.748468037694693, + "policy": "contiguous", + "relative_rms": { + "output": 0.0008147533423000863, + "dq": 0.0003699549868243877, + "dkv": 0.00028025793158283253, + "d_index_q": 0.00018789359275136517, + "d_weights": 0.00016652942081256042, + "d_pool_kv": 0.00023713204488080765, + "d_pool_score": 0.00023688382513071412, + "d_ape": 2.7803123769608203e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 174, + "raw_file": "layout-1m-count-tiny-mix-1000-planner.json", + "raw_sha256": "3eac9c25237afd5472f5a365b4f857ee8e947e2718dfb2c3611e8688666e20aa" + }, + "zigzag": { + "total_ms": 288.2996520996094, + "min_ms": 287.6239318847656, + "max_ms": 289.6219177246094, + "allocated_gib": 67.24870872497559, + "reserved_gib": 78.861328125, + "prepare_ms": 67.15863605495542, + "layout_prepare_ms": 66.5940580656752, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008172251772034926, + "dq": 0.00036548175905332905, + "dkv": 0.0002639633962520787, + "d_index_q": 0.00017265337760804814, + "d_weights": 0.00016465928857432128, + "d_pool_kv": 0.00023364198757436886, + "d_pool_score": 0.00023719059082038686, + "d_ape": 2.881065687905103e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49526814743876457, + "max_spans": 2000, + "raw_file": "layout-1m-count-tiny-mix-1000-zigzag.json", + "raw_sha256": "ca6895e139050eaba280929dc0e0e527168019b7b4e68b3b07fbbd3644138080" + } + }, + "speedup_vs_contiguous": 0.9982951005837959, + "gap_to_best_baseline_pct": 0.17078110622872344, + "prepare_plus_iteration_ms": { + "contiguous": 292.1983841266483, + "planner": 290.2439909270033, + "zigzag": 355.4582881545648 + }, + "prepare_inclusive_gap_pct": -0.6688583188050323, + "documents": 1000, + "lengths_sha256": "2043af46235527c07d70653288a5fbff85cbdc1d61c2df251c4b11c9f2173662" + }, + "1m-count-tiny-mix-512": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 252.4097137451172, + "min_ms": 251.78445434570312, + "max_ms": 253.2755126953125, + "allocated_gib": 66.40801906585693, + "reserved_gib": 68.822265625, + "prepare_ms": 11.81349204853177, + "layout_prepare_ms": 11.319566983729601, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49916257709264755, + "max_spans": 92, + "raw_file": "layout-1m-count-tiny-mix-512-contiguous.json", + "raw_sha256": "470b9156895dfef6f001c6566a52ff9fe9156a7aebfe0d9e953cec31588205b4" + }, + "planner": { + "total_ms": 252.61129760742188, + "min_ms": 251.07884216308594, + "max_ms": 253.76771545410156, + "allocated_gib": 66.40801906585693, + "reserved_gib": 68.822265625, + "prepare_ms": 10.621751076541841, + "layout_prepare_ms": 10.238889022730291, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011379571872195018, + "dq": 0.0005142828500427043, + "dkv": 0.00032494544441751173, + "d_index_q": 0.00016323910386101733, + "d_weights": 0.00014454080721691223, + "d_pool_kv": 0.00024713382526628155, + "d_pool_score": 0.00024822032949893725, + "d_ape": 5.576403384703898e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49916257709264755, + "max_spans": 92, + "raw_file": "layout-1m-count-tiny-mix-512-planner.json", + "raw_sha256": "542b74951fd33de1c6536f234290815cf3368ed74b1a311ca939395424d0cce1" + }, + "zigzag": { + "total_ms": 260.91038513183594, + "min_ms": 260.1084899902344, + "max_ms": 261.8586730957031, + "allocated_gib": 66.41342687606812, + "reserved_gib": 69.32421875, + "prepare_ms": 35.87489610072225, + "layout_prepare_ms": 35.48457403667271, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011495718791550967, + "dq": 0.000523029028005613, + "dkv": 0.0003364792880244074, + "d_index_q": 0.00021014620007773496, + "d_weights": 0.0001769941445596639, + "d_pool_kv": 0.0002674930321212269, + "d_pool_score": 0.0002657953944677991, + "d_ape": 5.380611858644655e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.4989374950528145, + "max_spans": 1024, + "raw_file": "layout-1m-count-tiny-mix-512-zigzag.json", + "raw_sha256": "25ff432deedf54ecca7b914719d4cc1415c96127bd9f5acb375435664074a70d" + } + }, + "speedup_vs_contiguous": 0.9992019998146798, + "gap_to_best_baseline_pct": 0.07986374981916455, + "prepare_plus_iteration_ms": { + "contiguous": 264.22320579364896, + "planner": 263.2330486839637, + "zigzag": 296.7852812325582 + }, + "prepare_inclusive_gap_pct": -0.3747426751223859, + "documents": 512, + "lengths_sha256": "eb5e64b29a22782c3e203c9db504474452dd18b36ac3aa9a4ad991a5af9bdbf9" + }, + "1m-count-uniform-1024": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 119.41489791870117, + "min_ms": 119.24073791503906, + "max_ms": 119.6561279296875, + "allocated_gib": 66.34996938705444, + "reserved_gib": 68.70703125, + "prepare_ms": 8.79430107306689, + "layout_prepare_ms": 8.168380009010434, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0623779296875, + "max_spans": 128, + "raw_file": "layout-1m-count-uniform-1024-contiguous.json", + "raw_sha256": "cb2b198cea9cd3cc1c3eaf1a11d285567036530e09b0bb6e49b1e711424adf27" + }, + "planner": { + "total_ms": 119.48390579223633, + "min_ms": 119.34534454345703, + "max_ms": 119.6822738647461, + "allocated_gib": 66.34996938705444, + "reserved_gib": 68.70703125, + "prepare_ms": 11.64705806877464, + "layout_prepare_ms": 11.14371104631573, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.345404721876349e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.14921621395609e-05, + "d_pool_score": 7.218511403644321e-05, + "d_ape": 9.385957743902641e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0623779296875, + "max_spans": 128, + "raw_file": "layout-1m-count-uniform-1024-planner.json", + "raw_sha256": "27c35dcd661256741e720d8db490b9e8e83568b814699ed745b9ab2bbc01fc1d" + }, + "zigzag": { + "total_ms": 133.69087982177734, + "min_ms": 133.57740783691406, + "max_ms": 133.81129455566406, + "allocated_gib": 66.35586500167847, + "reserved_gib": 69.208984375, + "prepare_ms": 96.13504004664719, + "layout_prepare_ms": 95.60391097329557, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.0615702213174114e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.488063479448549e-05, + "d_pool_score": 7.075341797691834e-05, + "d_ape": 9.443483387805362e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0623779296875, + "max_spans": 2048, + "raw_file": "layout-1m-count-uniform-1024-zigzag.json", + "raw_sha256": "0dfd44fe1fc27144d82b9d43d49510d9504ce3ee909f4a66712e0e47b687967a" + } + }, + "speedup_vs_contiguous": 0.9994224504708178, + "gap_to_best_baseline_pct": 0.0577883285401537, + "prepare_plus_iteration_ms": { + "contiguous": 128.20919899176806, + "planner": 131.13096386101097, + "zigzag": 229.82591986842453 + }, + "prepare_inclusive_gap_pct": 2.278904237932644, + "documents": 1024, + "lengths_sha256": "7d1b6ad860e729dfc1e098d7b89b3e6e85ea0e907b90de52c462e82eb70e9471" + }, + "1m-count-uniform-256": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 212.9967041015625, + "min_ms": 212.8686065673828, + "max_ms": 213.72950744628906, + "allocated_gib": 65.50574588775635, + "reserved_gib": 67.923828125, + "prepare_ms": 3.961967071518302, + "layout_prepare_ms": 3.575184033252299, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.2498779296875, + "max_spans": 32, + "raw_file": "layout-1m-count-uniform-256-contiguous.json", + "raw_sha256": "06822967569423fd2d1a139db17c3413ae46207130da1cfde995a274d5d45d78" + }, + "planner": { + "total_ms": 213.0918197631836, + "min_ms": 212.87930297851562, + "max_ms": 213.86585998535156, + "allocated_gib": 65.50574588775635, + "reserved_gib": 67.923828125, + "prepare_ms": 4.589442978613079, + "layout_prepare_ms": 4.242056980729103, + "policy": "contiguous", + "relative_rms": { + "output": 0.0008983001869912695, + "dq": 0.0004206017803911536, + "dkv": 0.00029550489189777144, + "d_index_q": 0.00012622909156508592, + "d_weights": 0.00011351108154107496, + "d_pool_kv": 0.00020215864138665445, + "d_pool_score": 0.0002058701846442034, + "d_ape": 3.364103427754296e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.2498779296875, + "max_spans": 32, + "raw_file": "layout-1m-count-uniform-256-planner.json", + "raw_sha256": "f2b892081f4956336269dce71261727b62a18bd3c2a902b38e6846a72bd0e6c7" + }, + "zigzag": { + "total_ms": 220.54678344726562, + "min_ms": 220.18325805664062, + "max_ms": 221.15260314941406, + "allocated_gib": 65.51161479949951, + "reserved_gib": 68.42578125, + "prepare_ms": 22.930420003831387, + "layout_prepare_ms": 22.559862001799047, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009009673640706196, + "dq": 0.0004183338551494578, + "dkv": 0.0002940296337405208, + "d_index_q": 0.00010926371136128527, + "d_weights": 9.464661783500973e-05, + "d_pool_kv": 0.00020544086367790907, + "d_pool_score": 0.00021230314881310683, + "d_ape": 3.6113839509353313e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.2498779296875, + "max_spans": 512, + "raw_file": "layout-1m-count-uniform-256-zigzag.json", + "raw_sha256": "bebe21ddd1346db24c9d611e1a1a05d400b8ec7f65ace3032d45430e6888efbf" + } + }, + "speedup_vs_contiguous": 0.9995536400143056, + "gap_to_best_baseline_pct": 0.044655931190251685, + "prepare_plus_iteration_ms": { + "contiguous": 216.9586711730808, + "planner": 217.68126274179667, + "zigzag": 243.477203451097 + }, + "prepare_inclusive_gap_pct": 0.3330549384400472, + "documents": 256, + "lengths_sha256": "2b66265ed16d5c91894fb143f8c654f23f801053a71d4fd69181b0642cf716a7" + }, + "1m-heldout-jittered-768-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 139.59954833984375, + "min_ms": 139.31805419921875, + "max_ms": 140.8841552734375, + "allocated_gib": 66.06588745117188, + "reserved_gib": 68.396484375, + "prepare_ms": 7.3116159765049815, + "layout_prepare_ms": 6.80255691986531, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08821307122707367, + "max_spans": 99, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-contiguous.json", + "raw_sha256": "c61c210bfbe957150e8897a19dae3ec5aefa62d1b97f3dbc0c3c74a13421bbf6" + }, + "planner": { + "total_ms": 139.54326629638672, + "min_ms": 139.3474884033203, + "max_ms": 139.70681762695312, + "allocated_gib": 66.06588745117188, + "reserved_gib": 68.396484375, + "prepare_ms": 9.15238680317998, + "layout_prepare_ms": 8.70554894208908, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.534552172078891e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 8.384488122997325e-05, + "d_pool_score": 0.00012568695483731863, + "d_ape": 9.973954559775284e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08821307122707367, + "max_spans": 99, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-planner.json", + "raw_sha256": "84b9744b6aa02b91671350f67939997a9415d18cd770df78214da079e1e72be8" + }, + "zigzag": { + "total_ms": 152.68809509277344, + "min_ms": 152.55538940429688, + "max_ms": 153.68234252929688, + "allocated_gib": 66.0734510421753, + "reserved_gib": 68.90234375, + "prepare_ms": 64.65917301829904, + "layout_prepare_ms": 64.22794598620385, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.467940785081395e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.491533441383689e-05, + "d_pool_score": 6.582463735603153e-05, + "d_ape": 8.561384532654269e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08652793243527412, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-zigzag.json", + "raw_sha256": "8a93d233b6c7707cf3595479a5e33dfa2e64b450083591db987e93dc77794549" + } + }, + "speedup_vs_contiguous": 1.0004033304146507, + "gap_to_best_baseline_pct": -0.040316780481275405, + "prepare_plus_iteration_ms": { + "contiguous": 146.91116431634873, + "planner": 148.6956530995667, + "zigzag": 217.34726811107248 + }, + "prepare_inclusive_gap_pct": 1.2146720036711134, + "documents": 768, + "lengths_sha256": "2384da57da625501af834608e8298b60467638272e4c35dc6ea261713c7eb0d3" + }, + "1m-heldout-jittered-768-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 139.19557189941406, + "min_ms": 138.859619140625, + "max_ms": 140.72189331054688, + "allocated_gib": 66.05843544006348, + "reserved_gib": 68.38671875, + "prepare_ms": 7.30161985848099, + "layout_prepare_ms": 6.851075915619731, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08720123767852783, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-contiguous.json", + "raw_sha256": "281825c0dfd3827c5cc80ef8b905d5396badd31ecbada0b521049755e9fae486" + }, + "planner": { + "total_ms": 138.96669006347656, + "min_ms": 138.5364532470703, + "max_ms": 139.24237060546875, + "allocated_gib": 66.05843544006348, + "reserved_gib": 68.38671875, + "prepare_ms": 9.032243047840893, + "layout_prepare_ms": 8.597164996899664, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.083760511669527e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.48209891306436e-05, + "d_pool_score": 6.628093591786182e-05, + "d_ape": 8.97144166007357e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08720123767852783, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-planner.json", + "raw_sha256": "a2229866121576881acd3547ce8ad2ba875e0536e87568b6d4b5b6425ad2d497" + }, + "zigzag": { + "total_ms": 152.07752227783203, + "min_ms": 151.6907501220703, + "max_ms": 152.71705627441406, + "allocated_gib": 66.06595468521118, + "reserved_gib": 68.890625, + "prepare_ms": 67.99968285486102, + "layout_prepare_ms": 67.59183399844915, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.278026323711552e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 7.544563700271164e-05, + "d_pool_score": 7.063303606390711e-05, + "d_ape": 9.006565520482359e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0866801105439663, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-zigzag.json", + "raw_sha256": "5ef0a181cfe1c6fafec89e9ac1539b46788a6868f57d4dd07a2bee0ff28bb0cd" + } + }, + "speedup_vs_contiguous": 1.0016470266063973, + "gap_to_best_baseline_pct": -0.16443183702919972, + "prepare_plus_iteration_ms": { + "contiguous": 146.49719175789505, + "planner": 147.99893311131746, + "zigzag": 220.07720513269305 + }, + "prepare_inclusive_gap_pct": 1.0250990721407227, + "documents": 768, + "lengths_sha256": "8884d0c226a9c39788069b745f1fa9b8499da77272f0e098c192c933dea50872" + }, + "1m-heldout-jittered-768-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 139.4260025024414, + "min_ms": 138.7571258544922, + "max_ms": 140.87948608398438, + "allocated_gib": 66.05876111984253, + "reserved_gib": 68.38671875, + "prepare_ms": 7.246268913149834, + "layout_prepare_ms": 6.824985961429775, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08736230432987213, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-contiguous.json", + "raw_sha256": "880cadfe68a2931bee30456d4c858043bad4546227d18fe035b62b8ea3f8b7ec" + }, + "planner": { + "total_ms": 139.19524383544922, + "min_ms": 138.77737426757812, + "max_ms": 140.00262451171875, + "allocated_gib": 66.05876111984253, + "reserved_gib": 68.38671875, + "prepare_ms": 9.026253945194185, + "layout_prepare_ms": 8.594213984906673, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.93177721660562e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.695678676610094e-05, + "d_pool_score": 5.4488619542700796e-05, + "d_ape": 8.187076335588852e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08736230432987213, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-planner.json", + "raw_sha256": "840688ba09cf768b72512d250f48fe5d9f33bd023b6ee5b7103d19f9b8335802" + }, + "zigzag": { + "total_ms": 152.10254669189453, + "min_ms": 151.87277221679688, + "max_ms": 152.91253662109375, + "allocated_gib": 66.06567144393921, + "reserved_gib": 68.890625, + "prepare_ms": 71.22155895922333, + "layout_prepare_ms": 70.7582599716261, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.1343651448698765e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.659607648562089e-05, + "d_pool_score": 5.1206900409261796e-05, + "d_ape": 8.198515312622374e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08655709028244019, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-zigzag.json", + "raw_sha256": "3aebfa0afb6dc800f3e22106a7f55c050272ff51d1bce0726e29d3fee735c395" + } + }, + "speedup_vs_contiguous": 1.0016578056881382, + "gap_to_best_baseline_pct": -0.16550619170778713, + "prepare_plus_iteration_ms": { + "contiguous": 146.67227141559124, + "planner": 148.2214977806434, + "zigzag": 223.32410565111786 + }, + "prepare_inclusive_gap_pct": 1.0562503396858602, + "documents": 768, + "lengths_sha256": "484660be7f47772a4ee1c32aa13529e5d4dfa02ba770b46f67c83944d3d16782" + }, + "1m-heldout-lognormal-1000-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 242.74365997314453, + "min_ms": 241.8260498046875, + "max_ms": 243.4641571044922, + "allocated_gib": 67.14972496032715, + "reserved_gib": 69.392578125, + "prepare_ms": 23.308494011871517, + "layout_prepare_ms": 22.75374298915267, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 186, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-contiguous.json", + "raw_sha256": "c72415467035a1b2320593612913f0d9156c3b9f87dfd0b2bab580825435d73d" + }, + "planner": { + "total_ms": 242.79873657226562, + "min_ms": 242.02297973632812, + "max_ms": 243.5925750732422, + "allocated_gib": 67.14972496032715, + "reserved_gib": 69.392578125, + "prepare_ms": 18.588110920973122, + "layout_prepare_ms": 17.979708965867758, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006639286414964582, + "dq": 0.0003421409964573097, + "dkv": 0.00019721484937837802, + "d_index_q": 0.00011000092475129282, + "d_weights": 0.0001068132442141927, + "d_pool_kv": 0.00018391022308396582, + "d_pool_score": 0.000179099258457748, + "d_ape": 3.408956283145262e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 186, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-planner.json", + "raw_sha256": "31fe420fe090d09c73296c247e5e8a6243701726a0ea69e87df03d963bc8a78d" + }, + "zigzag": { + "total_ms": 255.3045883178711, + "min_ms": 253.9938507080078, + "max_ms": 256.24114990234375, + "allocated_gib": 67.42135286331177, + "reserved_gib": 79.7109375, + "prepare_ms": 73.20481608621776, + "layout_prepare_ms": 72.67806399613619, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006729130295918793, + "dq": 0.000346805284697457, + "dkv": 0.00019871221130998912, + "d_index_q": 0.00011022225577992002, + "d_weights": 0.00010803303612996262, + "d_pool_kv": 0.00018681248298492773, + "d_pool_score": 0.00017741006842812262, + "d_ape": 3.0188399219256512e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49985016882419586, + "max_spans": 1790, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-zigzag.json", + "raw_sha256": "9021d15661632a997b26c2d03add080ed503ddb322430f01d16390e8079eedc8" + } + }, + "speedup_vs_contiguous": 0.9997731594492679, + "gap_to_best_baseline_pct": 0.02268920190426904, + "prepare_plus_iteration_ms": { + "contiguous": 266.05215398501605, + "planner": 261.38684749323875, + "zigzag": 328.50940440408885 + }, + "prepare_inclusive_gap_pct": -1.7535308103688751, + "documents": 1000, + "lengths_sha256": "f9552804d64123e178b417a374ec673ead4524cfdca92d036e2c49d2242a4462" + }, + "1m-heldout-lognormal-1000-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 238.08277130126953, + "min_ms": 237.04464721679688, + "max_ms": 240.6115264892578, + "allocated_gib": 67.31743335723877, + "reserved_gib": 69.40234375, + "prepare_ms": 10.576636996120214, + "layout_prepare_ms": 10.025152005255222, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 205, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-contiguous.json", + "raw_sha256": "73d12f484cc6a6c5781d76c844be01adbcc0bb83c62e714ff358b96cdf452c3f" + }, + "planner": { + "total_ms": 238.69235229492188, + "min_ms": 236.03648376464844, + "max_ms": 239.77203369140625, + "allocated_gib": 67.31743335723877, + "reserved_gib": 69.40234375, + "prepare_ms": 14.750760048627853, + "layout_prepare_ms": 14.257603092119098, + "policy": "contiguous", + "relative_rms": { + "output": 0.000627483968379822, + "dq": 0.0003440353669091771, + "dkv": 0.00019056774527328763, + "d_index_q": 0.00012198431218257316, + "d_weights": 0.00012043062385770405, + "d_pool_kv": 0.00018010804743631497, + "d_pool_score": 0.00017470367552733856, + "d_ape": 3.126033191671862e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 205, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-planner.json", + "raw_sha256": "268cbe29bd9f8bbfb74658d4ec75f8412f70a70d7a8090fd2764979bf02fba83" + }, + "zigzag": { + "total_ms": 248.48849487304688, + "min_ms": 246.69786071777344, + "max_ms": 250.46841430664062, + "allocated_gib": 67.61479187011719, + "reserved_gib": 69.91015625, + "prepare_ms": 69.25003707874566, + "layout_prepare_ms": 68.7437430024147, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006277650198511363, + "dq": 0.00034722313626910604, + "dkv": 0.00019309980305575396, + "d_index_q": 0.00011586721853121804, + "d_weights": 0.00011320829699045079, + "d_pool_kv": 0.00019337419199732237, + "d_pool_score": 0.0001837468057090465, + "d_ape": 3.3393516739202175e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49977222830057144, + "max_spans": 1756, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-zigzag.json", + "raw_sha256": "462ccac477b21d252cce91a052b7ebeedc598fc61058bf33fe9395890da44b82" + } + }, + "speedup_vs_contiguous": 0.9974461645386142, + "gap_to_best_baseline_pct": 0.25603742359037174, + "prepare_plus_iteration_ms": { + "contiguous": 248.65940829738975, + "planner": 253.44311234354973, + "zigzag": 317.73853195179254 + }, + "prepare_inclusive_gap_pct": 1.9237977275482043, + "documents": 1000, + "lengths_sha256": "4860b063d4278394f20027f7e1f50de3bf6080e17af8500ea52a852f8f4840bf" + }, + "1m-heldout-lognormal-1000-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 243.51073455810547, + "min_ms": 242.13845825195312, + "max_ms": 245.1307830810547, + "allocated_gib": 67.10111045837402, + "reserved_gib": 69.26171875, + "prepare_ms": 9.780772146768868, + "layout_prepare_ms": 9.23799304291606, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 165, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-contiguous.json", + "raw_sha256": "85762a7b6a5337bbff4590396f9f87426434440d07a330772a0b23ac03910407" + }, + "planner": { + "total_ms": 244.25122833251953, + "min_ms": 242.3041229248047, + "max_ms": 245.93923950195312, + "allocated_gib": 67.10111045837402, + "reserved_gib": 69.26171875, + "prepare_ms": 11.766831041313708, + "layout_prepare_ms": 11.266377987340093, + "policy": "contiguous", + "relative_rms": { + "output": 0.0005700379404832603, + "dq": 0.000319132883698665, + "dkv": 0.00017456473994610376, + "d_index_q": 0.00011017834715051893, + "d_weights": 9.883468029904095e-05, + "d_pool_kv": 0.00017233705959233462, + "d_pool_score": 0.00017515985135813642, + "d_ape": 3.255570361937096e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 165, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-planner.json", + "raw_sha256": "066d43150dee8501d7f08bc63475b30c1f33486a98044180dc550a0c599da0f0" + }, + "zigzag": { + "total_ms": 255.27643585205078, + "min_ms": 254.18067932128906, + "max_ms": 256.8161315917969, + "allocated_gib": 67.19186353683472, + "reserved_gib": 69.892578125, + "prepare_ms": 65.90253801550716, + "layout_prepare_ms": 65.40064502041787, + "policy": "zigzag", + "relative_rms": { + "output": 0.0005658249552824931, + "dq": 0.00032052362678565844, + "dkv": 0.00017358126670537923, + "d_index_q": 0.00010731651292044749, + "d_weights": 9.582209699794192e-05, + "d_pool_kv": 0.00017146293516180915, + "d_pool_score": 0.0001825117182559775, + "d_ape": 3.296560839970807e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49838995188474655, + "max_spans": 1743, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-zigzag.json", + "raw_sha256": "d898c9cf7a64ce0e3dc8bbc7f34d48e71c1bac6dcccc8b6008a9ecd44aeb7b41" + } + }, + "speedup_vs_contiguous": 0.99696831094988, + "gap_to_best_baseline_pct": 0.30409081380244896, + "prepare_plus_iteration_ms": { + "contiguous": 253.29150670487434, + "planner": 256.01805937383324, + "zigzag": 321.17897386755794 + }, + "prepare_inclusive_gap_pct": 1.076448517531925, + "documents": 1000, + "lengths_sha256": "a790875510a4369ef17daa8104180cfe9404b90dfa3567b8652ea1435220224c" + }, + "1m-heldout-lognormal-257-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 241.6246566772461, + "min_ms": 240.7863311767578, + "max_ms": 242.99119567871094, + "allocated_gib": 65.84781551361084, + "reserved_gib": 68.2734375, + "prepare_ms": 4.298857995308936, + "layout_prepare_ms": 3.948985948227346, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49603627994656563, + "max_spans": 41, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-contiguous.json", + "raw_sha256": "9f80e24a6b4b20a2697560effb1c4ce1ea3384ddd71ec8131fd60f255b096e43" + }, + "planner": { + "total_ms": 241.78618621826172, + "min_ms": 241.03724670410156, + "max_ms": 242.47116088867188, + "allocated_gib": 65.84781551361084, + "reserved_gib": 68.2734375, + "prepare_ms": 16.895215027034283, + "layout_prepare_ms": 16.556818038225174, + "policy": "contiguous", + "relative_rms": { + "output": 0.001082461469725319, + "dq": 0.00047458198967384186, + "dkv": 0.00032668096235836366, + "d_index_q": 0.00016693420591685536, + "d_weights": 0.0001401149820797931, + "d_pool_kv": 0.0002270806698028004, + "d_pool_score": 0.0002237423415161784, + "d_ape": 4.043788132397096e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49603627994656563, + "max_spans": 41, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-planner.json", + "raw_sha256": "2627c6242e0e931d47e104db31fdd7f534a82a2e567533cd631d8a1ebca7aaca" + }, + "zigzag": { + "total_ms": 249.37068939208984, + "min_ms": 248.5408935546875, + "max_ms": 250.9279327392578, + "allocated_gib": 65.8637466430664, + "reserved_gib": 68.78125, + "prepare_ms": 22.184235975146294, + "layout_prepare_ms": 21.849395940080285, + "policy": "zigzag", + "relative_rms": { + "output": 0.001088782543496487, + "dq": 0.0004730404072428335, + "dkv": 0.00031546559266904786, + "d_index_q": 0.00016690660493783074, + "d_weights": 0.00014110205381527205, + "d_pool_kv": 0.0002280621593994692, + "d_pool_score": 0.0002186074713902558, + "d_ape": 3.871855939807412e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49925995245575905, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-zigzag.json", + "raw_sha256": "77d01364b5020f7f3f07344f74d5e5c49a00d484dbec869d72f66e5812e4068b" + } + }, + "speedup_vs_contiguous": 0.9993319323012531, + "gap_to_best_baseline_pct": 0.06685143115645609, + "prepare_plus_iteration_ms": { + "contiguous": 245.92351467255503, + "planner": 258.681401245296, + "zigzag": 271.55492536723614 + }, + "prepare_inclusive_gap_pct": 5.187745706110292, + "documents": 257, + "lengths_sha256": "22124dfa90691813499ba5fff3956732dd43551f4504a48fc46bbb35d7aa1141" + }, + "1m-heldout-lognormal-257-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 265.89527893066406, + "min_ms": 264.75982666015625, + "max_ms": 268.54608154296875, + "allocated_gib": 65.86178588867188, + "reserved_gib": 68.294921875, + "prepare_ms": 4.706783103756607, + "layout_prepare_ms": 4.385833046399057, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 43, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-contiguous.json", + "raw_sha256": "0e96976fed2028414081cc39e060cf53f598d02e636f8fa3f134d0f65c0451cf" + }, + "planner": { + "total_ms": 266.5758819580078, + "min_ms": 264.8830261230469, + "max_ms": 267.7908020019531, + "allocated_gib": 65.86178588867188, + "reserved_gib": 68.294921875, + "prepare_ms": 5.949159967713058, + "layout_prepare_ms": 5.652374937199056, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011651272922486198, + "dq": 0.00046214190754927277, + "dkv": 0.00033944054288377243, + "d_index_q": 0.00018021932141442865, + "d_weights": 0.00016083575052146454, + "d_pool_kv": 0.00026245941371384453, + "d_pool_score": 0.00027060036694570604, + "d_ape": 3.819325031365223e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 43, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-planner.json", + "raw_sha256": "a6a02d026ba3e2910be4548d73f88cfb1ead501e4292eb973941dfac8667150b" + }, + "zigzag": { + "total_ms": 270.3296203613281, + "min_ms": 269.5667724609375, + "max_ms": 272.5, + "allocated_gib": 65.86807107925415, + "reserved_gib": 68.802734375, + "prepare_ms": 29.988013091497123, + "layout_prepare_ms": 29.641884961165488, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011900939859345021, + "dq": 0.0004752948227896761, + "dkv": 0.0003530750774232262, + "d_index_q": 0.0001982857983837737, + "d_weights": 0.00016784374447724294, + "d_pool_kv": 0.00026278078754386075, + "d_pool_score": 0.0002623528065003457, + "d_ape": 3.922395870611972e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.4999861344695091, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-zigzag.json", + "raw_sha256": "3f4742ba6a41a71ac42df4190de2bedd3f97ce2acfdcd640f44e3148b04dd391" + } + }, + "speedup_vs_contiguous": 0.9974468694529126, + "gap_to_best_baseline_pct": 0.25596657077961993, + "prepare_plus_iteration_ms": { + "contiguous": 270.60206203442067, + "planner": 272.52504192572087, + "zigzag": 300.31763345282525 + }, + "prepare_inclusive_gap_pct": 0.710630169202342, + "documents": 257, + "lengths_sha256": "0b8ffc4c0c7294d43fa7ae5f3ee55faaef51c2f526387b1efe2d60aca24ecf20" + }, + "1m-heldout-lognormal-257-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 260.8553009033203, + "min_ms": 259.1219482421875, + "max_ms": 261.8494567871094, + "allocated_gib": 65.91541194915771, + "reserved_gib": 68.26171875, + "prepare_ms": 5.178843857720494, + "layout_prepare_ms": 4.816529923118651, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 54, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-contiguous.json", + "raw_sha256": "195dddebf12e65361a3e617322fbf574c7c9c3eed81643a4e319b1fd093c307d" + }, + "planner": { + "total_ms": 261.18247985839844, + "min_ms": 260.1310119628906, + "max_ms": 263.29669189453125, + "allocated_gib": 65.91541194915771, + "reserved_gib": 68.26171875, + "prepare_ms": 5.219785147346556, + "layout_prepare_ms": 4.866166040301323, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011647173346943536, + "dq": 0.00047342276965134394, + "dkv": 0.00036562031615802835, + "d_index_q": 0.00021551581252850696, + "d_weights": 0.00016625701885295734, + "d_pool_kv": 0.0002632888196715603, + "d_pool_score": 0.000255773612521125, + "d_ape": 3.971134246049848e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 54, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-planner.json", + "raw_sha256": "8d936534214d4549c1acb1225671cf5c54859c2056d5b334d7cc1ad492298d73" + }, + "zigzag": { + "total_ms": 265.91778564453125, + "min_ms": 263.9945983886719, + "max_ms": 267.6018371582031, + "allocated_gib": 65.97511625289917, + "reserved_gib": 68.794921875, + "prepare_ms": 21.29810012411326, + "layout_prepare_ms": 20.93889401294291, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011734780735060887, + "dq": 0.00047267021844590634, + "dkv": 0.0003651486194391658, + "d_index_q": 0.00019081633609087836, + "d_weights": 0.0001568154855097009, + "d_pool_kv": 0.0002506288295072141, + "d_pool_score": 0.0002447311671719562, + "d_ape": 3.5682579411177634e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.4963202215731144, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-zigzag.json", + "raw_sha256": "abea4ee11e6a7073940a93bf00e5942008bb99412477d1c56c274fe4044782f4" + } + }, + "speedup_vs_contiguous": 0.998747316606935, + "gap_to_best_baseline_pct": 0.12542545769440583, + "prepare_plus_iteration_ms": { + "contiguous": 266.0341447610408, + "planner": 266.402265005745, + "zigzag": 287.2158857686445 + }, + "prepare_inclusive_gap_pct": 0.13837330731918396, + "documents": 257, + "lengths_sha256": "1b0bd973931c10a52c40d767fb8a267929d55b8a7e94ed85145c4ab27033efda" + }, + "1m-heldout-tiny-tail-768-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 436.30291748046875, + "min_ms": 433.5298767089844, + "max_ms": 439.7943115234375, + "allocated_gib": 68.20519065856934, + "reserved_gib": 70.546875, + "prepare_ms": 10.106927016749978, + "layout_prepare_ms": 9.562587947584689, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.0, + "max_spans": 263, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-contiguous.json", + "raw_sha256": "8a8ffeec6eec8a264804f99b37d66ec9a4b009a10313c251fc50f63ae72185af" + }, + "planner": { + "total_ms": 397.61912536621094, + "min_ms": 393.9225769042969, + "max_ms": 398.3644714355469, + "allocated_gib": 69.3592586517334, + "reserved_gib": 82.36328125, + "prepare_ms": 42.66655596438795, + "layout_prepare_ms": 42.218683985993266, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0008605439165391654, + "dq": 0.0003796395940608871, + "dkv": 0.0004054932800869999, + "d_index_q": 0.00046272970168726135, + "d_weights": 0.0004308179609189755, + "d_pool_kv": 0.0006452094646867585, + "d_pool_score": 0.0006358833131884315, + "d_ape": 5.545222900579305e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 2.9356987215578556, + "max_spans": 267, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-planner.json", + "raw_sha256": "784f57c9706e909b3737814020733d26552a436f7be44459872241000b3725c8" + }, + "zigzag": { + "total_ms": 406.08197021484375, + "min_ms": 404.5019226074219, + "max_ms": 407.4826965332031, + "allocated_gib": 70.75117206573486, + "reserved_gib": 73.427734375, + "prepare_ms": 48.15629299264401, + "layout_prepare_ms": 47.64960997272283, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008973106239127031, + "dq": 0.00038915186357727963, + "dkv": 0.0005022528545868634, + "d_index_q": 0.0005194953800697627, + "d_weights": 0.0004678227992383376, + "d_pool_kv": 0.0006610885127278837, + "d_pool_score": 0.0006449522590483258, + "d_ape": 5.407920341001154e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 2.9998814575374126, + "max_spans": 1260, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-zigzag.json", + "raw_sha256": "7d6cbc1cb6b8c8507589e766a4faf98800549fc3398bf84b122e09175a2c8179" + } + }, + "speedup_vs_contiguous": 1.0972885599469837, + "gap_to_best_baseline_pct": -2.0840237856793786, + "prepare_plus_iteration_ms": { + "contiguous": 446.4098444972187, + "planner": 440.2856813305989, + "zigzag": 454.23826320748776 + }, + "prepare_inclusive_gap_pct": -1.3718700969772124, + "documents": 768, + "lengths_sha256": "759566912504d164342d1556047348e640f650e2bd969cace033c5f384cd1e9f" + }, + "1m-heldout-tiny-tail-768-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 498.76747131347656, + "min_ms": 496.61907958984375, + "max_ms": 499.63848876953125, + "allocated_gib": 68.21028566360474, + "reserved_gib": 70.623046875, + "prepare_ms": 9.636490955017507, + "layout_prepare_ms": 9.107938036322594, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.99999725446105, + "max_spans": 176, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-contiguous.json", + "raw_sha256": "751af802d0941f5e890120e8eb2cb43fbcdd5b47b8f92461fdb1fa7de6da76f2" + }, + "planner": { + "total_ms": 416.4952392578125, + "min_ms": 412.5445251464844, + "max_ms": 419.02020263671875, + "allocated_gib": 71.1473913192749, + "reserved_gib": 80.89453125, + "prepare_ms": 29.679763014428318, + "layout_prepare_ms": 29.224156984128058, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0007987615295327556, + "dq": 0.00039969776861594446, + "dkv": 0.0005472686255923894, + "d_index_q": 0.0004013407150718083, + "d_weights": 0.00039543349130716937, + "d_pool_kv": 0.0006895992619430324, + "d_pool_score": 0.0006990540356347693, + "d_ape": 5.209086564608632e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 4.499835789203644, + "max_spans": 177, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-planner.json", + "raw_sha256": "30332df67d347bcfa79a52ba3f9690159d7c4efc61181cd7792ad349b4bf31bd" + }, + "zigzag": { + "total_ms": 423.7662811279297, + "min_ms": 422.471435546875, + "max_ms": 425.5033874511719, + "allocated_gib": 71.05869436264038, + "reserved_gib": 73.974609375, + "prepare_ms": 48.98834810592234, + "layout_prepare_ms": 48.50035498384386, + "policy": "zigzag", + "relative_rms": { + "output": 0.0007981808335954349, + "dq": 0.00040108914815335767, + "dkv": 0.0005455319088188014, + "d_index_q": 0.00040511993368696884, + "d_weights": 0.00040129581062005136, + "d_pool_kv": 0.000683351457010383, + "d_pool_score": 0.0006834509121711517, + "d_ape": 5.314517142166228e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 4.499952055513859, + "max_spans": 1233, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-zigzag.json", + "raw_sha256": "6a2beb285d81a216825fd74f33c3b1cc372c41ee64d142f80226c6e7de0b5a5b" + } + }, + "speedup_vs_contiguous": 1.1975346277721488, + "gap_to_best_baseline_pct": -1.7158141631193558, + "prepare_plus_iteration_ms": { + "contiguous": 508.40396226849407, + "planner": 446.1750022722408, + "zigzag": 472.75462923385203 + }, + "prepare_inclusive_gap_pct": -5.622288036541545, + "documents": 768, + "lengths_sha256": "747a3b346f6fb67d2a1bda28b2c1431fae38be2adb70abab6f64c52541fce124" + }, + "1m-heldout-tiny-tail-768-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 559.7161865234375, + "min_ms": 555.4302978515625, + "max_ms": 560.9325561523438, + "allocated_gib": 69.61964559555054, + "reserved_gib": 72.03125, + "prepare_ms": 9.729802957735956, + "layout_prepare_ms": 9.215633966960013, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999971453100443, + "max_spans": 225, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-contiguous.json", + "raw_sha256": "34fb5f5d012997932088d2de86f6c14a8a809f74ed42e02a14fa7eb70040d2f8" + }, + "planner": { + "total_ms": 460.2198486328125, + "min_ms": 457.4324951171875, + "max_ms": 461.8628234863281, + "allocated_gib": 71.91085004806519, + "reserved_gib": 84.1875, + "prepare_ms": 33.673338941298425, + "layout_prepare_ms": 33.20295491721481, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0009838151945294016, + "dq": 0.0003731228063005086, + "dkv": 0.0004893858809695903, + "d_index_q": 0.000563171314835403, + "d_weights": 0.000552436597671842, + "d_pool_kv": 0.0006584843852002015, + "d_pool_score": 0.000660263037667876, + "d_ape": 6.329523728349438e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 5.999660342931747, + "max_spans": 226, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-planner.json", + "raw_sha256": "87ded17b209d211147131a5fc1862e21ff1f67971640652625b880fb27055cb5" + }, + "zigzag": { + "total_ms": 466.84815979003906, + "min_ms": 465.43475341796875, + "max_ms": 468.9653625488281, + "allocated_gib": 72.83784294128418, + "reserved_gib": 82.203125, + "prepare_ms": 49.553396995179355, + "layout_prepare_ms": 49.08921697642654, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009946083346030021, + "dq": 0.00038457455793733337, + "dkv": 0.0005425483112289231, + "d_index_q": 0.0006348606318300278, + "d_weights": 0.000538693682633851, + "d_pool_kv": 0.0006613934471291266, + "d_pool_score": 0.0006581440262921791, + "d_ape": 5.958685410178528e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 5.999810948967934, + "max_spans": 1248, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-zigzag.json", + "raw_sha256": "fd5cbe18c33e009862cd9cdab3a5a6d72d1be1f2db6e9201e6cd706880b2efbb" + } + }, + "speedup_vs_contiguous": 1.2161930611776122, + "gap_to_best_baseline_pct": -1.419800210888178, + "prepare_plus_iteration_ms": { + "contiguous": 569.4459894811735, + "planner": 493.8931875741109, + "zigzag": 516.4015567852184 + }, + "prepare_inclusive_gap_pct": -4.358695072731777, + "documents": 768, + "lengths_sha256": "e879dc84790636c8379bc654c1bf98f0f184183c998e485cd8cf5a1df8e1e686" + }, + "1m-heldout-trimodal-513-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 528.0744018554688, + "min_ms": 522.5753784179688, + "max_ms": 529.0509643554688, + "allocated_gib": 70.32668733596802, + "reserved_gib": 72.73828125, + "prepare_ms": 8.3013930125162, + "layout_prepare_ms": 7.790984003804624, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999376945197582, + "max_spans": 233, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-contiguous.json", + "raw_sha256": "8a3fc72747754bc7bf98ae1deb8ea22ee20008aa3b6703f2d087437b21883ccb" + }, + "planner": { + "total_ms": 471.4351348876953, + "min_ms": 470.94830322265625, + "max_ms": 474.5873107910156, + "allocated_gib": 73.74629783630371, + "reserved_gib": 87.171875, + "prepare_ms": 44.60881382692605, + "layout_prepare_ms": 44.13713398389518, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.000961612511327622, + "dq": 0.0003228331744803191, + "dkv": 0.000675315032828408, + "d_index_q": 0.0006175052651110469, + "d_weights": 0.0005102943346481259, + "d_pool_kv": 0.0007424000145656556, + "d_pool_score": 0.0007213206164765067, + "d_ape": 5.763866208010157e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999752186238766, + "max_spans": 237, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-planner.json", + "raw_sha256": "5441b436adfd01760c603c170f255191312cae6630d20e5ac4b7616c6600aa09" + }, + "zigzag": { + "total_ms": 482.18006896972656, + "min_ms": 480.6382141113281, + "max_ms": 484.48101806640625, + "allocated_gib": 73.42837762832642, + "reserved_gib": 75.80078125, + "prepare_ms": 34.889695001766086, + "layout_prepare_ms": 34.403237979859114, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009026663219808964, + "dq": 0.00029800123046863397, + "dkv": 0.0005882285438040862, + "d_index_q": 0.0004945744903112981, + "d_weights": 0.0004758399315457316, + "d_pool_kv": 0.0006676745367674404, + "d_pool_score": 0.0006603297039606205, + "d_ape": 5.349247680815768e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 5.999862506985664, + "max_spans": 835, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-zigzag.json", + "raw_sha256": "5f55ccd78b5bdd22737e144e4239d13ccba51f553282e2a54b8e49fbc808dce0" + } + }, + "speedup_vs_contiguous": 1.1201422269497712, + "gap_to_best_baseline_pct": -2.2284069320803623, + "prepare_plus_iteration_ms": { + "contiguous": 536.375794867985, + "planner": 516.0439487146214, + "zigzag": 517.0697639714926 + }, + "prepare_inclusive_gap_pct": -0.19839010678022184, + "documents": 513, + "lengths_sha256": "3e4264b20b69bf872e0287259db21e68128815011e3a225f7f002b5e26266a7b" + }, + "1m-heldout-trimodal-513-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 550.4004211425781, + "min_ms": 547.5628051757812, + "max_ms": 553.02490234375, + "allocated_gib": 69.8761420249939, + "reserved_gib": 72.076171875, + "prepare_ms": 33.551717875525355, + "layout_prepare_ms": 33.11699989717454, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999942742288113, + "max_spans": 260, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-contiguous.json", + "raw_sha256": "3dee930bb2ff3e79a9e051ed87ccaa077dee76d566fb3f1bb0d6e015c51dd7b4" + }, + "planner": { + "total_ms": 475.1640930175781, + "min_ms": 471.80645751953125, + "max_ms": 477.1839904785156, + "allocated_gib": 74.11927461624146, + "reserved_gib": 85.970703125, + "prepare_ms": 28.17985403817147, + "layout_prepare_ms": 27.757827076129615, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0009194823642314863, + "dq": 0.0002948452198108362, + "dkv": 0.00043585622989332116, + "d_index_q": 0.0007520857577959455, + "d_weights": 0.0005999834445308111, + "d_pool_kv": 0.0006288663704566335, + "d_pool_score": 0.0006258680549696073, + "d_ape": 5.887180290091932e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999993029981852, + "max_spans": 265, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-planner.json", + "raw_sha256": "924d83bdefd5a1ee078e10521aee743d31584790a0f0ae9d1f6e24569a585005" + }, + "zigzag": { + "total_ms": 479.2586669921875, + "min_ms": 477.9535827636719, + "max_ms": 481.8824768066406, + "allocated_gib": 71.65527057647705, + "reserved_gib": 74.568359375, + "prepare_ms": 35.18300596624613, + "layout_prepare_ms": 34.71819101832807, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008696921284109005, + "dq": 0.0003016373452874105, + "dkv": 0.0005305068105551274, + "d_index_q": 0.0006150898262228985, + "d_weights": 0.0005590287009485252, + "d_pool_kv": 0.0007163439171067582, + "d_pool_score": 0.0007367160338217451, + "d_ape": 5.756793869761055e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999988615512848, + "max_spans": 817, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-zigzag.json", + "raw_sha256": "27be3e9d0a28de7930f03c3651bd2d9858cf830d7ba19b8ff0487e2815828021" + } + }, + "speedup_vs_contiguous": 1.1583375706005141, + "gap_to_best_baseline_pct": -0.8543557491211939, + "prepare_plus_iteration_ms": { + "contiguous": 583.9521390181035, + "planner": 503.3439470557496, + "zigzag": 514.4416729584336 + }, + "prepare_inclusive_gap_pct": -2.1572369592190332, + "documents": 513, + "lengths_sha256": "d11985e121377244a83f6693db589940ae4899c1bf48610d0e00341a06f525bd" + }, + "1m-heldout-trimodal-513-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 606.0254821777344, + "min_ms": 603.9866943359375, + "max_ms": 608.9718017578125, + "allocated_gib": 71.13527822494507, + "reserved_gib": 73.546875, + "prepare_ms": 20.425522001460195, + "layout_prepare_ms": 20.0147470459342, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 4.999916918575764, + "max_spans": 240, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-contiguous.json", + "raw_sha256": "e787b7d95ec05a91a114359d1ab494d627217bdd711ce034667b52a13bc9434b" + }, + "planner": { + "total_ms": 498.80828857421875, + "min_ms": 495.22216796875, + "max_ms": 500.7232971191406, + "allocated_gib": 75.91035795211792, + "reserved_gib": 85.07421875, + "prepare_ms": 28.75487704295665, + "layout_prepare_ms": 28.326248051598668, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0009195983309898245, + "dq": 0.0003224729936723083, + "dkv": 0.0005545372026871616, + "d_index_q": 0.0004788406854099131, + "d_weights": 0.0004916242112471586, + "d_pool_kv": 0.0007596727225373355, + "d_pool_score": 0.0007537076166265762, + "d_ape": 5.542803593958258e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 7.999753892421722, + "max_spans": 243, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-planner.json", + "raw_sha256": "a9d6fb26a731bfdea6c8af827da8d3fb53f15d16fd7edcc6b1d54c02c71082c5" + }, + "zigzag": { + "total_ms": 509.7503662109375, + "min_ms": 507.0311584472656, + "max_ms": 511.49639892578125, + "allocated_gib": 72.71457099914551, + "reserved_gib": 75.626953125, + "prepare_ms": 38.119480945169926, + "layout_prepare_ms": 37.66501497011632, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009651923554078656, + "dq": 0.0003559762003050189, + "dkv": 0.0004964050388006907, + "d_index_q": 0.0006082251240356637, + "d_weights": 0.0005475287924962972, + "d_pool_kv": 0.0007222207414950428, + "d_pool_score": 0.0007242622401541285, + "d_ape": 5.483300095764783e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 7.499938875436783, + "max_spans": 847, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-zigzag.json", + "raw_sha256": "a7707b4d3b17d281581ddee91ba773fdbb2646e2054a68873f5bb0aebb316db2" + } + }, + "speedup_vs_contiguous": 1.2149466960743227, + "gap_to_best_baseline_pct": -2.146556110994702, + "prepare_plus_iteration_ms": { + "contiguous": 626.4510041791946, + "planner": 527.5631656171754, + "zigzag": 547.8698471561074 + }, + "prepare_inclusive_gap_pct": -3.706479129001228, + "documents": 513, + "lengths_sha256": "30e8a9b80449ecc324bba6134d04db805dac074f3b7693579d6f2afb153135f7" + }, + "1m-near-uniform": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 261.5732116699219, + "min_ms": 261.14141845703125, + "max_ms": 262.1058654785156, + "allocated_gib": 65.3901014328003, + "reserved_gib": 67.666015625, + "prepare_ms": 3.4267649753019214, + "layout_prepare_ms": 3.1357220141217113, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.3920431137084961, + "max_spans": 8, + "raw_file": "layout-1m-near-uniform-contiguous.json", + "raw_sha256": "0341b198790d2b51903794067c2aecc73345790bc795019ab793fd43fa3e4991" + }, + "planner": { + "total_ms": 261.5891571044922, + "min_ms": 261.0076599121094, + "max_ms": 262.2134704589844, + "allocated_gib": 65.3901014328003, + "reserved_gib": 67.666015625, + "prepare_ms": 3.440934931859374, + "layout_prepare_ms": 3.165671951137483, + "policy": "contiguous", + "relative_rms": { + "output": 0.0015901644171629613, + "dq": 0.0005902133591829151, + "dkv": 0.00047656249471409655, + "d_index_q": 0.00026291229043044637, + "d_weights": 0.000199747845851943, + "d_pool_kv": 0.00029679011974003083, + "d_pool_score": 0.00028698171976176143, + "d_ape": 6.356026201118735e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 0.3920431137084961, + "max_spans": 8, + "raw_file": "layout-1m-near-uniform-planner.json", + "raw_sha256": "9798272f912bf142eb9127388a691e04d11e776a1c7423d3f2f5f9df8619fac8" + }, + "zigzag": { + "total_ms": 266.84141540527344, + "min_ms": 266.42767333984375, + "max_ms": 267.36077880859375, + "allocated_gib": 65.50305652618408, + "reserved_gib": 68.41796875, + "prepare_ms": 10.221436969004571, + "layout_prepare_ms": 9.95832693297416, + "policy": "zigzag", + "relative_rms": { + "output": 0.0016123834602235457, + "dq": 0.000596027757416673, + "dkv": 0.0004744482659872333, + "d_index_q": 0.0003024936753378096, + "d_weights": 0.00022482807025107514, + "d_pool_kv": 0.00029945938494957544, + "d_pool_score": 0.0002934684618935451, + "d_ape": 6.307271516024825e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.4991369843482971, + "max_spans": 128, + "raw_file": "layout-1m-near-uniform-zigzag.json", + "raw_sha256": "3f1624f6a8368598fe04d11064f2689db4a6f15a9c78f9e45f8bdcb05df1a07c" + } + }, + "speedup_vs_contiguous": 0.9999390439774078, + "gap_to_best_baseline_pct": 0.006095973845532399, + "prepare_plus_iteration_ms": { + "contiguous": 264.9999766452238, + "planner": 265.03009203635156, + "zigzag": 277.062852374278 + }, + "prepare_inclusive_gap_pct": 0.011364299540339573, + "documents": 64, + "lengths_sha256": "f7b3e262fd896136c35c22e4766cae5a4b416484124973745b17cbc8ceac3850" + }, + "1m-power-law": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 612.9796752929688, + "min_ms": 610.664306640625, + "max_ms": 614.4771118164062, + "allocated_gib": 68.94974088668823, + "reserved_gib": 71.298828125, + "prepare_ms": 4.962046048603952, + "layout_prepare_ms": 4.639261984266341, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999988242983818, + "max_spans": 5, + "raw_file": "layout-1m-power-law-contiguous.json", + "raw_sha256": "6429bf4696405989be5de571ee173b6ca4acde3df0aaaa4c786cf941014001e3" + }, + "planner": { + "total_ms": 492.8457794189453, + "min_ms": 490.5483093261719, + "max_ms": 493.98095703125, + "allocated_gib": 71.04951000213623, + "reserved_gib": 81.416015625, + "prepare_ms": 17.59058702737093, + "layout_prepare_ms": 17.31892500538379, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0013432263284307595, + "dq": 0.00048809261707209576, + "dkv": 0.0005241199919794483, + "d_index_q": 0.0005870922763788422, + "d_weights": 0.0005419402939198647, + "d_pool_kv": 0.0006357484486274489, + "d_pool_score": 0.0006410094965115505, + "d_ape": 1.1075007383756194e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 8.731149137020111e-11, + "workspace_gib": 5.999798443168402, + "max_spans": 11, + "raw_file": "layout-1m-power-law-planner.json", + "raw_sha256": "a6abe46aecd4b94aff2226f824224590238014f009c4258ae1b44969ec53371a" + }, + "zigzag": { + "total_ms": 493.15296936035156, + "min_ms": 491.9137878417969, + "max_ms": 497.9830627441406, + "allocated_gib": 68.97773599624634, + "reserved_gib": 71.888671875, + "prepare_ms": 7.186377886682749, + "layout_prepare_ms": 6.911287899129093, + "policy": "zigzag", + "relative_rms": { + "output": 0.001332787454565865, + "dq": 0.0004750203554706029, + "dkv": 0.0005353831251106813, + "d_index_q": 0.0005565436942807204, + "d_weights": 0.0005130840726434935, + "d_pool_kv": 0.0006657684779061981, + "d_pool_score": 0.0006483417130847936, + "d_ape": 1.1188895558466485e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.9999998174607754, + "max_spans": 14, + "raw_file": "layout-1m-power-law-zigzag.json", + "raw_sha256": "2e25f55ea691125c1c3adb2d2e7bf27af6d94db7c98b2aa0f4070eef7dd2aaaa" + } + }, + "speedup_vs_contiguous": 1.24375555374677, + "gap_to_best_baseline_pct": -0.062291005122550036, + "prepare_plus_iteration_ms": { + "contiguous": 617.9417213415727, + "planner": 510.43636644631624, + "zigzag": 500.3393472470343 + }, + "prepare_inclusive_gap_pct": 2.018034211148434, + "documents": 7, + "lengths_sha256": "b787057eb34fe32408b2fe31d07c9bc6642c12e889e2e2a42e790f32a7ce58d8" + }, + "1m-single": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 879.3570251464844, + "min_ms": 876.3743286132812, + "max_ms": 884.391357421875, + "allocated_gib": 72.9497447013855, + "reserved_gib": 75.298828125, + "prepare_ms": 6.599797983653843, + "layout_prepare_ms": 6.2927789986133575, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 7.9999911189079285, + "max_spans": 1, + "raw_file": "layout-1m-single-contiguous.json", + "raw_sha256": "de07cbd27df21fd95be7f54d41bd223ef072cd85db0e59944fbc0776603a1610" + }, + "planner": { + "total_ms": 664.8384704589844, + "min_ms": 663.4234008789062, + "max_ms": 669.0645141601562, + "allocated_gib": 72.95559072494507, + "reserved_gib": 75.802734375, + "prepare_ms": 15.023829997517169, + "layout_prepare_ms": 14.757112949155271, + "policy": "selection-local-2x", + "relative_rms": { + "output": 0.0011529975036186093, + "dq": 0.0003767490804819508, + "dkv": 0.0005946098839724117, + "d_index_q": 0.0006048463254406639, + "d_weights": 0.0005030932802858819, + "d_pool_kv": 0.0008104081461652777, + "d_pool_score": 0.0007695389939561969, + "d_ape": 1.2836426006061245e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.9103830456733704e-11, + "workspace_gib": 7.9999911189079285, + "max_spans": 2, + "raw_file": "layout-1m-single-planner.json", + "raw_sha256": "dface57613b8b8bdec4b67f29b41e3a4814f217e27e436fbd73114e1b6f7db98" + }, + "zigzag": { + "total_ms": 667.4992980957031, + "min_ms": 662.85498046875, + "max_ms": 671.0496826171875, + "allocated_gib": 72.95510244369507, + "reserved_gib": 75.80078125, + "prepare_ms": 6.756177055649459, + "layout_prepare_ms": 6.48829503916204, + "policy": "zigzag", + "relative_rms": { + "output": 0.001182160242463775, + "dq": 0.0004037382914331386, + "dkv": 0.0005202209349556262, + "d_index_q": 0.0006295794340313956, + "d_weights": 0.0005169888293507331, + "d_pool_kv": 0.0007736514408215355, + "d_pool_score": 0.0007362455606161046, + "d_ape": 1.444483038268409e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 7.9999911189079285, + "max_spans": 2, + "raw_file": "layout-1m-single-zigzag.json", + "raw_sha256": "ae689331fefbba2baca40beac27367e73c7758b5e7833cc80dfe10fe1c55d506" + } + }, + "speedup_vs_contiguous": 1.3226626680303306, + "gap_to_best_baseline_pct": -0.3986262823511222, + "prepare_plus_iteration_ms": { + "contiguous": 885.9568231301382, + "planner": 679.8623004565015, + "zigzag": 674.2554751513526 + }, + "prepare_inclusive_gap_pct": 0.8315579942291818, + "documents": 1, + "lengths_sha256": "df57f4ef94807b0499d37da5e83d5c5333272a5ebcad42ac3dbec6ac94a30bf2" + }, + "1m-uniform": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 261.7406768798828, + "min_ms": 261.2946472167969, + "max_ms": 262.6141052246094, + "allocated_gib": 65.49799585342407, + "reserved_gib": 67.916015625, + "prepare_ms": 21.517536020837724, + "layout_prepare_ms": 21.237667999230325, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49993896484375, + "max_spans": 8, + "raw_file": "layout-1m-uniform-contiguous.json", + "raw_sha256": "1be5b4895931639b8ffdae843a7316effeb9f792a8fdb2cd7344db3d5933f273" + }, + "planner": { + "total_ms": 261.7190704345703, + "min_ms": 261.2510070800781, + "max_ms": 262.2694091796875, + "allocated_gib": 65.49799585342407, + "reserved_gib": 67.916015625, + "prepare_ms": 4.200693103484809, + "layout_prepare_ms": 3.913125954568386, + "policy": "contiguous", + "relative_rms": { + "output": 0.0015967309781703524, + "dq": 0.0006004709784135087, + "dkv": 0.0005059608205209192, + "d_index_q": 0.00029251105906248924, + "d_weights": 0.00022732957796148034, + "d_pool_kv": 0.00029126673820099435, + "d_pool_score": 0.0002934495651972289, + "d_ape": 6.876366322072345e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 0.49993896484375, + "max_spans": 8, + "raw_file": "layout-1m-uniform-planner.json", + "raw_sha256": "0c3dee88bee4c9bd715e8e224758cbef62730da61683099646ce4092be28962f" + }, + "zigzag": { + "total_ms": 267.4135284423828, + "min_ms": 266.8036193847656, + "max_ms": 267.57830810546875, + "allocated_gib": 65.5038571357727, + "reserved_gib": 68.41796875, + "prepare_ms": 9.938867064192891, + "layout_prepare_ms": 9.64121101424098, + "policy": "zigzag", + "relative_rms": { + "output": 0.0016095293393893419, + "dq": 0.0005985745463675235, + "dkv": 0.0004895944404512913, + "d_index_q": 0.0002713512548711526, + "d_weights": 0.0002148084702065561, + "d_pool_kv": 0.0002933053408000937, + "d_pool_score": 0.00029111625679209704, + "d_ape": 6.765475676396151e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49993896484375, + "max_spans": 128, + "raw_file": "layout-1m-uniform-zigzag.json", + "raw_sha256": "763cd6a9c47b94fed97a8ac9aac15ece2cc739a8a320973118e87c63096ddd2d" + } + }, + "speedup_vs_contiguous": 1.0000825558690722, + "gap_to_best_baseline_pct": -0.00825490541632723, + "prepare_plus_iteration_ms": { + "contiguous": 283.25821290072054, + "planner": 265.9197635380551, + "zigzag": 277.3523955065757 + }, + "prepare_inclusive_gap_pct": -4.122059933046273, + "documents": 64, + "lengths_sha256": "dd1b660aba10a40fca5d0224dbbcfdec6e38447a6ae6c9aadae9aa252935fd2e" + } + } + } + }, + "measured_source_sha256": { + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "9100c6359659a85d024340878a7ff1c395ba20b51ecf2668d308a9066304a231", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "0fe29b40204b5f59976517353c63a5ed5e2ea1033be1bb58b68aeb1c703a3aa3", + "cula/distributed/thd_planner.py": "6b10315a430627461d5ecb7941b8bd1e7bf5411cf4afa3057317070fda80867c", + "cula/hybrid/__init__.py": "74934cf8db62dd1f237a30ee6aca283aa7a38483c5ed921da61aee09ea7ebbf5", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "6b4a64ed35b2b70e41fc978fb4db3a8fd60b85fe33160c5d4fa02afb0ca4b52e", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "benchmarks/bench_full_csa_cp.py": "39bdd101b52253d9da338b94f259cbde4a0f1f07dda9d137231c2b1dfe653bfa" + } +} diff --git a/docs/results/thd_selection_review_20261115.json b/docs/results/thd_selection_review_20261115.json new file mode 100644 index 00000000..7c8e97a4 --- /dev/null +++ b/docs/results/thd_selection_review_20261115.json @@ -0,0 +1,1038 @@ +{ + "cpu_tests_passed": 337, + "dsa_rank_plans_metadata_equal": 2232, + "profiles_and_planner_decisions_unchanged_cases": 62, + "source_sha256": { + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "2acb5c13043c7776ce6b029620310335f2bbb032ce58118b39fe8db23dfb70d2", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "a7d06bb6f9cd7c14c2ee792baf64e094a35e745f81a22c948d6b8608ed16ac1f", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "20ce22c2a71a747492004a459379f2d902220e37fab79ba05f7889ccc18f3c0a", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/profiles/README.md": "4b15ea28c35d6dbcb65fb4be02d233fd82a93bedfecacbc9df08926ac290f0a6", + "benchmarks/profiles/thd_selection_b300_cp8.json": "d166704b094b2c6aef277290e0d77672b4ea06cbd8f24bdd110ebb316fe1eb72", + "benchmarks/profiles/thd_selection_h20_cp16.json": "63839a024e0120c256576a159e555441c1f4850524af07d10ea0b1cc5fc774de", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "058a536d36c0117f70ebf6309d290909a537bfa7e816d9cf04ea58b2495c8d0f", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032", + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "990bd5ed2a5fb559ad65020d140356d995f83ff84c4363148f24800dcf122a6a", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "9100c6359659a85d024340878a7ff1c395ba20b51ecf2668d308a9066304a231", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "b923570e51067143cf99919ca3b11e56b47308dd7f44ba6d3daa9fc34d7a0e65", + "cula/distributed/thd_planner.py": "6b10315a430627461d5ecb7941b8bd1e7bf5411cf4afa3057317070fda80867c", + "cula/hybrid/__init__.py": "74934cf8db62dd1f237a30ee6aca283aa7a38483c5ed921da61aee09ea7ebbf5", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "pyproject.toml": "56a112afeea1bfd5f334853f3f34e92afe3dd26ad025a73ada3075bb1aff7f25", + "scripts/modal_validate.py": "048d89dc199da77a4a38b390fe6f02275ea4ec4123a8cb05fe9778fa23bd773a", + "setup.py": "6b65a6be394a519d1a1f79db18c60a20479e6065c9fe4c0f9a27474d8f58d6b3", + "tests/__init__.py": "0f0f8fcb798517a7b8cf062cf3bcbcc7f3b245a3431590f85a40b4150e6aa857", + "tests/conftest.py": "ce2c8122dc096d503d5904e612d4656f592293d96456955856899acfd2199127", + "tests/lightning_attn_reference.py": "5072c4000e3e51a6887e24b5820e8364d8f0ca5216f282506f11cc366321f29d", + "tests/test_chunk_wy_dqkg_sm90.py": "b3275ae3e718ceace6a315dceefd19c4bc0386dd8a3f453ae1b40af4791d8f17", + "tests/test_compare_with_fla.py": "a32febc8fc73cab1e751cd2daa9ee31ae022136b8cf37613836cdf72f416512c", + "tests/test_csa_matrix_report.py": "35b201871efed3e85ec540724d2f8e9f24b71eae0a471143183ccde2777f1411", + "tests/test_cutedsl_compat.py": "1abe8b60c888f1bce045c4b881fb201976b3c80004f32ed3c1a519fb4cb7f573", + "tests/test_cutedsl_mbarrier_init.py": "e408a02c3d1f547576277f7d73e019f951c54202c69354ed81083f946f7008d6", + "tests/test_full_csa_cp_metadata.py": "c44aca48dff4765fb850b6203cccb9269e203fffc9d87b0635ca5a23db411e48", + "tests/test_hybrid_distributed_pool.py": "071ece1d2f11154500486c5948d11e51d5f53bdcc561c76066841cacb2af5fd2", + "tests/test_hybrid_dsa_cudnn.py": "25d14e60911a4dfee55cf561c39db180bfbb59eb3996cc94bd16067a6d5d4602", + "tests/test_hybrid_dsa_helpers.py": "642c55c8536011c8afa65cc101d55c2b0c56968cef30c91c2e42e719e7154d6c", + "tests/test_hybrid_dsa_plan.py": "647102932147049e6569e4896e094c867de6211124548d97d98d704e9464d0ed", + "tests/test_hybrid_index_pool_helpers.py": "d3eb6cc43cd023872039e94c3928b4f253a51e6582a75d5c07519d79130fe6da", + "tests/test_hybrid_index_pool_plan.py": "aa47c241c3c55e386da4f716ab7b83626e170b7a9c3dd81020bdba2916444f2f", + "tests/test_kda_conv_mtp.py": "484743cdc3b7c50d4a196caaa2c00119f3f1631124b5d5999f0c1d200310f053", + "tests/test_kda_conv_mtp_kvbuffer.py": "6fa813cf8f9c25caa79756dd2b58070036c83610313db4529bf3dfd202df2527", + "tests/test_kda_decode.py": "fcdaca68b9773dea3a8822ec571daa7a6898b993ec78fab259a3969a02276afc", + "tests/test_kda_decode_mtp.py": "45224002328dbace1a9250f4944c7e5c91f6804efd26f221ca278c42f293be9c", + "tests/test_kda_flashkda_arch.py": "7d0b125df694743b97429434537608cd09ce288897cba75cdd2489fea2e27374", + "tests/test_kda_flashkda_pipeline.py": "4598953061b94e593da29775b85692632e41558f39fb0916e5bb2d3ceab3e9c6", + "tests/test_kda_flashkda_sm100.py": "427cf2782a86b2143b2315b31e77dc94d4495f762d05ce688606f515e01a8599", + "tests/test_kda_packed_decode.py": "1eed4f70b8ca7152a7205a12faaeec8e9c605a5790b215e8016e447cde029242", + "tests/test_kda_prefill_api.py": "10c913455885e763c4b64f64b222aba01a4ad853d24f2a5293854c83d5f76b85", + "tests/test_kda_sm100_csrc_golden.py": "e0b97255a6b76430cd432bbe2a97dcae225885d2a5bf6f464ae9691a569e7e29", + "tests/test_kda_sm100_full_port_cutedsl.py": "b7ff7a5b1b05c48be0861bf0cfac36dc0c53383fd514f3676a66ab74ef1f68ac", + "tests/test_kda_sm100_intra_fused_cutedsl.py": "7bdccf9e9213ce59b85e011517a53a789634fa3ecdf614c8624f4d15c591fcc6", + "tests/test_kda_sm100_intracard_cp.py": "ac55728c850d864514956fab7b9f6a030ed36c10a24a6645ceb418a573ef83e0", + "tests/test_kda_sm100_output.py": "986b89eb1654907df89d683e06552dd377766dbccebfeeb0a1e9efe845095ab0", + "tests/test_kda_sm100_recompute_wu_cutedsl.py": "a436cc24f8bb00693f4efb93d631bfe43fc598e127012f5c9ec17454df976397", + "tests/test_kda_sm100_recurrence.py": "f129df6556f6ee3717a62d7a384263726f2ae99e7993fd0883a3fd47c3f781e6", + "tests/test_kda_sm90_intracard_cp.py": "a29ba5ed1c5e7c20a44fbfbb4863dc6b5d084a4194370d9c1616d318e797e6b6", + "tests/test_kda_sm90_prefill_vs_fla.py": "c592157326564935b2db6bf270f8b6f3c464170a2bc0a24b7624bb881665d255", + "tests/test_kimi_cp_distributed.py": "24ec2a82ad164c38365cba0840935ec0262659bc2a9b6c931dbe48ef5649684c", + "tests/test_kimi_cp_helpers.py": "c2374b4267b86be6003c1687b1b787c153fac5214a7485ba11850d78a92f116e", + "tests/test_kimi_cp_plan.py": "facddb9d2471b7c5c172c4eeea89febebb34e2f9192116c361b3a3f7b8e4181e", + "tests/test_la_decode_mtp.py": "0af705c288e6c176a610c0da6344f414bfe79f8752571e2816683e69ca840319", + "tests/test_la_kvbuffer.py": "33aec4bd91f031b8fc55f7917e3fee916a9c987728e617ee18c36766999fdb8d", + "tests/test_large_cp_emulation.py": "37e5e0eebf108e9673442b15fa6f2893d010ac65242a0ac76fdd65cc475fd174", + "tests/test_large_cp_query_layout.py": "ba5504cc75c7bab381ba9918dd6179abcd38d9030df5558ca4f602322a3ca6d6", + "tests/test_lightning_attn_prefill_dispatch.py": "002ca5602944b82083fb5a8c0cce45ab7f7dc4661efde9a6beca6d232fb708cc", + "tests/test_lightning_attn_prefill_sm90.py": "7817d21b004690f54fc75ae45fb4bf4aaed2a0d85b445e27608a456cbbf4f7d7", + "tests/test_lightning_decode.py": "58d8d89dd162d2d9ed16256fddef92cddfc0b37954d79acb58bae321a45bd82b", + "tests/test_lightning_decode_pool.py": "faf013566520c5869eafbf576437c08e3e85921a075c6ff8b9698b5b37dcea04", + "tests/test_lightning_sm100_prefill.py": "93bd5e250dc9b5db4cf7bbdc6bae07c87c20157d5e6ac388afcc5e3be6c829c3", + "tests/test_ptx_umma_masked.py": "f88ca7d76598396d147a58e23d535bcc80bcdb3aa52ca15b5f1a8d1e3a1c9082", + "tests/test_ptx_umma_ws.py": "6043d672a5456cd039d4bdec5425dd434a15c21f13ab8e0003ab56d44291b568", + "tests/test_standalone_kernel_surface.py": "7d06007483aaa21bba94e9c49353cbc3ddf868a75bfa3a5b7b28fd51fd78b60e", + "tests/test_test_suite_profile.py": "cbabf13caa933a1e4259796a947874280774b28f0883738c63d0636c5fa1ecab", + "tests/test_thd_1m_cases.py": "8f49b87c0f75fdedb359ba382752b124e3c9c4f6c6aac14c69c9678b6d3f5a08", + "tests/test_thd_cost_fit.py": "80811592b6e22b4c0ace12b68b081345c23a5288fcadef4a6482b9ffa16a84b1", + "tests/test_thd_planner.py": "b78eaad978f14b420b29ac4805eaaba968771030e45fd949cc1f23b81f36b500", + "tests/test_thd_redistribute.py": "6967330ec4b8e86273de0ff2069f3e6d648832404745d528b0b9f990f14bdceb" + }, + "platforms": { + "h20_cp16": { + "full_csa_rows": 21, + "fp32_check_rows": 12, + "one_m_cases": { + "1m-count-mixed-1000-1long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-mixed-1000-1long-contiguous.json", + "total_ms": 1734.75927734375, + "min_ms": 1730.8006591796875, + "max_ms": 1741.20263671875, + "allocated_gib": 40.67960977554321, + "reserved_gib": 45.763671875, + "prepare_ms": 18.482470943126827, + "layout_prepare_ms": 17.99330097856, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999728836119175, + "max_spans": 135 + }, + "planner": { + "source": "layout-1m-count-mixed-1000-1long-planner.json", + "total_ms": 1079.9918212890625, + "min_ms": 1078.9337158203125, + "max_ms": 1080.5509033203125, + "allocated_gib": 42.25836515426636, + "reserved_gib": 46.767578125, + "prepare_ms": 67.1296549262479, + "layout_prepare_ms": 66.67199596995488, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0013937532701230287, + "dq": 0.0007082964070750158, + "dkv": 0.0006137254351491774, + "d_index_q": 0.000786539812462648, + "d_weights": 0.0008088782633845682, + "d_pool_kv": 0.0009303882391162907, + "d_pool_score": 0.0009171219579661385, + "d_ape": 4.0186864686063254e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999987006187439, + "max_spans": 137 + }, + "zigzag": { + "source": "layout-1m-count-mixed-1000-1long-zigzag.json", + "total_ms": 1110.866943359375, + "min_ms": 1110.0362548828125, + "max_ms": 1112.42529296875, + "allocated_gib": 41.40707540512085, + "reserved_gib": 46.76953125, + "prepare_ms": 71.14337000530213, + "layout_prepare_ms": 70.65627403790131, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014044624923471708, + "dq": 0.000718481723676461, + "dkv": 0.0006745502854727569, + "d_index_q": 0.0008122306113631543, + "d_weights": 0.0008234815695709059, + "d_pool_kv": 0.0009008851498893785, + "d_pool_score": 0.000896099670380151, + "d_ape": 4.149431394764015e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.999989103525877, + "max_spans": 1200 + } + }, + "speedup_vs_contiguous": 1.6062707542295707, + "gap_to_best_baseline_pct": -2.779371755985738, + "prepare_plus_iteration_ms": { + "contiguous": 1753.2417482868768, + "planner": 1147.1214762153104, + "zigzag": 1182.0103133646771 + }, + "prepare_inclusive_gap_pct": -2.951652515624259, + "documents": 1000 + }, + "1m-count-tiny-mix-1000": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-tiny-mix-1000-contiguous.json", + "total_ms": 806.0984497070312, + "min_ms": 805.9192504882812, + "max_ms": 806.1605834960938, + "allocated_gib": 39.01279067993164, + "reserved_gib": 46.146484375, + "prepare_ms": 27.179943048395216, + "layout_prepare_ms": 26.643655030056834, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 143 + }, + "planner": { + "source": "layout-1m-count-tiny-mix-1000-planner.json", + "total_ms": 805.8177490234375, + "min_ms": 805.7346801757812, + "max_ms": 806.04150390625, + "allocated_gib": 39.0129189491272, + "reserved_gib": 44.91015625, + "prepare_ms": 13.915454037487507, + "layout_prepare_ms": 13.422106392681599, + "policy": "contiguous", + "relative_rms": { + "output": 0.0009711554023225683, + "dq": 0.0006152729484808843, + "dkv": 0.0003926991846874368, + "d_index_q": 0.0003016586499429551, + "d_weights": 0.0003208988010868512, + "d_pool_kv": 0.0003709002742156663, + "d_pool_score": 0.000379869535872596, + "d_ape": 3.3729356601707806e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 143 + }, + "zigzag": { + "source": "layout-1m-count-tiny-mix-1000-zigzag.json", + "total_ms": 821.2994995117188, + "min_ms": 821.0466918945312, + "max_ms": 822.2781372070312, + "allocated_gib": 39.00111246109009, + "reserved_gib": 46.18359375, + "prepare_ms": 76.13682397641242, + "layout_prepare_ms": 75.66057000076398, + "policy": "zigzag", + "relative_rms": { + "output": 0.0010180829441058961, + "dq": 0.0006456383045368285, + "dkv": 0.0004906538276004508, + "d_index_q": 0.00034542113019621524, + "d_weights": 0.00033085709677908745, + "d_pool_kv": 0.00037554118525190026, + "d_pool_score": 0.0003727287946029054, + "d_ape": 3.2555742636422667e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49914050847291946, + "max_spans": 1200 + } + }, + "speedup_vs_contiguous": 1.000348342641911, + "gap_to_best_baseline_pct": -0.034822134156908824, + "prepare_plus_iteration_ms": { + "contiguous": 833.2783927554265, + "planner": 819.733203060925, + "zigzag": 897.4363234881312 + }, + "prepare_inclusive_gap_pct": -1.6255299324048456, + "documents": 1000 + }, + "1m-single": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-single-contiguous.json", + "total_ms": 2905.626953125, + "min_ms": 2882.656982421875, + "max_ms": 2919.6826171875, + "allocated_gib": 42.58390665054321, + "reserved_gib": 47.701171875, + "prepare_ms": 6.134580064099282, + "layout_prepare_ms": 5.915455054491758, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999951247125864, + "max_spans": 1 + }, + "planner": { + "source": "layout-1m-single-planner.json", + "total_ms": 1930.8380126953125, + "min_ms": 1929.918701171875, + "max_ms": 1935.875732421875, + "allocated_gib": 43.17422676086426, + "reserved_gib": 48.498046875, + "prepare_ms": 64.26966283470392, + "layout_prepare_ms": 64.06422145664692, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014571483324829387, + "dq": 0.0007421278526508277, + "dkv": 0.0007551298904303639, + "d_index_q": 0.0007261477208478219, + "d_weights": 0.0007914909302235379, + "d_pool_kv": 0.0009610520429536651, + "d_pool_score": 0.0009602856176819985, + "d_ape": 1.3991300052915012e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.999988541007042, + "max_spans": 2 + }, + "zigzag": { + "source": "layout-1m-single-zigzag.json", + "total_ms": 1944.2569580078125, + "min_ms": 1939.1024169921875, + "max_ms": 1955.9830322265625, + "allocated_gib": 43.17422676086426, + "reserved_gib": 48.5, + "prepare_ms": 7.630876265466213, + "layout_prepare_ms": 7.41471815854311, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014060094451772223, + "dq": 0.0007140955156081367, + "dkv": 0.0006730968914330185, + "d_index_q": 0.000726804709767014, + "d_weights": 0.0007835076200786753, + "d_pool_kv": 0.00101384945694068, + "d_pool_score": 0.0010062370245195436, + "d_ape": 1.4595408260036373e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 8.731149137020111e-11, + "workspace_gib": 3.999988541007042, + "max_spans": 2 + } + }, + "speedup_vs_contiguous": 1.5048527810310464, + "gap_to_best_baseline_pct": -0.6901837361173557, + "prepare_plus_iteration_ms": { + "contiguous": 2911.7615331890993, + "planner": 1995.1076755300164, + "zigzag": 1951.8878342732787 + }, + "prepare_inclusive_gap_pct": 2.214258447531603, + "documents": 1 + } + }, + "max_relative_rms": { + "output": 0.0022360824205777032, + "dq": 0.0011669460245709953, + "dkv": 0.002013003506138166, + "d_index_q": 0.0026207664468348288, + "d_weights": 0.0019498450002418547, + "d_pool_kv": 0.0035533125482489977, + "d_pool_score": 0.0029468591104983063, + "d_ape": 4.616907922474155e-05, + "kda_output": 0, + "kda_dq": 0, + "kda_dk": 0, + "kda_dv": 0, + "kda_dg": 0, + "kda_dbeta": 0 + }, + "loss_abs_diff": 5.820766091346741e-10, + "independent_kda": [ + { + "case": "uniform-1024", + "label": "baseline", + "passed": true, + "total_ms": 30.399328231811523 + }, + { + "case": "uniform-1024", + "label": "candidate", + "passed": true, + "total_ms": 26.572128295898438 + }, + { + "case": "uniform-4096", + "label": "baseline", + "passed": true, + "total_ms": 35.26870346069336 + }, + { + "case": "uniform-4096", + "label": "candidate", + "passed": true, + "total_ms": 25.992992401123047 + }, + { + "case": "uniform-16384", + "label": "baseline", + "passed": true, + "total_ms": 58.847904205322266 + }, + { + "case": "uniform-16384", + "label": "candidate", + "passed": true, + "total_ms": 29.499359130859375 + }, + { + "case": "near-uniform", + "label": "baseline", + "passed": true, + "total_ms": 58.68083190917969 + }, + { + "case": "near-uniform", + "label": "candidate", + "passed": true, + "total_ms": 29.76799964904785 + }, + { + "case": "weak-decay-uniform", + "label": "baseline", + "passed": true, + "total_ms": 58.779518127441406 + }, + { + "case": "weak-decay-uniform", + "label": "candidate", + "passed": true, + "total_ms": 29.494943618774414 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "baseline", + "passed": true, + "total_ms": 111.36412811279297 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "candidate", + "passed": true, + "total_ms": 111.15647888183594 + } + ], + "raw_sha256": { + "layout-check-single-contiguous.json": "df4f9940c230c68ca647c70e636a8259f30b3630fa56f4ff8505e59eeb7fb713", + "layout-check-uniform-zigzag.json": "12951e3aa159381afd92fe6a34f398118ced9484f0d70b663dba6c8da869c165", + "layout-check-dense-zigzag.json": "2753074b3e030efce01c3cf5690f0ac79107321a5efcf26346c5b92eabc0242e", + "layout-check-mixed-planner.json": "b2b48f44dabbe1078a343704c104182875b4eabb67a46ac256baeb0ec13777a4", + "layout-1m-single-planner.json": "e547b82aa37d1dce0353e67acd524f02f238d7e7e20290426541875e0f40b540", + "layout-1m-count-mixed-1000-1long-zigzag.json": "811050d5906a45c3bd89e7a54a498116d97f7fa16faaa760079cab881e140c65", + "layout-check-mixed-zigzag.json": "a292ec0bee1f427930f1a8195491e5c333897f72f57cc98b35211b8a27af0fdd", + "layout-1m-count-mixed-1000-1long-contiguous.json": "583a4487a52242e40ba34476924a962d7ec56289821da15fdc0af2cba0380ce1", + "layout-check-single-zigzag.json": "7c72292b8584f667c3446d0725953f23b967b13903d5b5dafa21aa8f1dee4a4e", + "layout-1m-count-tiny-mix-1000-contiguous.json": "7c973181c8902c9891993cf2b8237872ce0c7f880adc7a724ed0ad1ed840d7b7", + "layout-check-single-planner.json": "8fabf01ebe04935b40a5a9455abf5f982f5565fdcb5c4b34c4cde9501ebb9f3a", + "layout-check-uniform-planner.json": "74fc7347994c5a6c98065a84c0fee011c5894727cf0ac9fc916f15847410068f", + "layout-check-mixed-contiguous.json": "6aa870d6ce4d270dd740fa5dd6f2fdfbe7dd0b0320d20e1f7a9c8be07f5a4a1a", + "layout-1m-single-contiguous.json": "21b0740f9da731c6c51354ef8d5eaf0e6716cf79293d073156444bba35c073da", + "layout-check-dense-planner.json": "8b9df01222bf46328824a7856df56889f1c281e8da5aceca66b0740450edd097", + "layout-1m-single-zigzag.json": "5e2adbed587139640c0236552a46902e790c8b4fea51fe1098e0575b9e44b6aa", + "layout-1m-count-tiny-mix-1000-planner.json": "4fc9ef2dfbdf4d73aade67b6227b2a2a37774556a1befc4ce78ee98a6a6a0ed3", + "layout-1m-count-tiny-mix-1000-zigzag.json": "33231fee890bc3d94bfd53dfd4b1f5068c10e19faebfed835382f503f53353d7", + "layout-check-uniform-contiguous.json": "6f21a533c62df42232b337f1fd4424386813480793c57c7bc7de6af109157957", + "layout-check-dense-contiguous.json": "910a2defc4a846c6bd43dfa3c6aef7ea6d0d6ef51745fdeea99b4be572cec5f4", + "layout-1m-count-mixed-1000-1long-planner.json": "3c64c6cc910cc4111ef9f4533bf235fcf5aac8a18b32e4cdd5d8f9f100c18cec" + }, + "independent_raw_sha256": "3829d509f0bf33a97df9065301cd294d52e2dbf0323df0124586080e5b7818d7" + }, + "b300_cp8": { + "full_csa_rows": 21, + "fp32_check_rows": 12, + "one_m_cases": { + "1m-count-mixed-1000-1long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-mixed-1000-1long-contiguous.json", + "total_ms": 563.7484130859375, + "min_ms": 561.4152221679688, + "max_ms": 566.3046264648438, + "allocated_gib": 69.97186422348022, + "reserved_gib": 72.40625, + "prepare_ms": 11.447699856944382, + "layout_prepare_ms": 10.858646943233907, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999988861382008, + "max_spans": 262 + }, + "planner": { + "source": "layout-1m-count-mixed-1000-1long-planner.json", + "total_ms": 465.6362609863281, + "min_ms": 464.8326110839844, + "max_ms": 466.11572265625, + "allocated_gib": 72.94540643692017, + "reserved_gib": 85.4140625, + "prepare_ms": 37.901079980656505, + "layout_prepare_ms": 37.406648974865675, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.000981929418930544, + "dq": 0.0003586859933571614, + "dkv": 0.0005251233580841019, + "d_index_q": 0.000768516565343547, + "d_weights": 0.0005601425977036474, + "d_pool_kv": 0.00075289034085658, + "d_pool_score": 0.0007494019256045927, + "d_ape": 3.2458577600061467e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 5.999798700213432, + "max_spans": 264 + }, + "zigzag": { + "source": "layout-1m-count-mixed-1000-1long-zigzag.json", + "total_ms": 480.0238342285156, + "min_ms": 479.3189392089844, + "max_ms": 482.1390686035156, + "allocated_gib": 72.85924434661865, + "reserved_gib": 74.9140625, + "prepare_ms": 68.89537198003381, + "layout_prepare_ms": 68.42616200447083, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009359406560165561, + "dq": 0.0003445985393205624, + "dkv": 0.0005055089868584756, + "d_index_q": 0.0007650331133222328, + "d_weights": 0.0005585781202017105, + "d_pool_kv": 0.0007500151256218873, + "d_pool_score": 0.0007479169339269727, + "d_ape": 3.378714550586395e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999901540577412, + "max_spans": 2000 + } + }, + "speedup_vs_contiguous": 1.2107055663830488, + "gap_to_best_baseline_pct": -2.9972622641354696, + "prepare_plus_iteration_ms": { + "contiguous": 575.1961129428819, + "planner": 503.53734096698463, + "zigzag": 548.9192062085494 + }, + "prepare_inclusive_gap_pct": -8.267494510717299, + "documents": 1000 + }, + "1m-count-tiny-mix-1000": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-tiny-mix-1000-contiguous.json", + "total_ms": 276.4977722167969, + "min_ms": 276.32672119140625, + "max_ms": 277.500244140625, + "allocated_gib": 67.35119819641113, + "reserved_gib": 69.52734375, + "prepare_ms": 14.639637200161815, + "layout_prepare_ms": 14.148701098747551, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 174 + }, + "planner": { + "source": "layout-1m-count-tiny-mix-1000-planner.json", + "total_ms": 276.5791931152344, + "min_ms": 275.75604248046875, + "max_ms": 278.41815185546875, + "allocated_gib": 67.35119819641113, + "reserved_gib": 69.52734375, + "prepare_ms": 13.63393000792712, + "layout_prepare_ms": 13.120088959112763, + "policy": "contiguous", + "relative_rms": { + "output": 0.000816856606217396, + "dq": 0.0003701320592240906, + "dkv": 0.00028171521311393853, + "d_index_q": 0.0001906295405525901, + "d_weights": 0.00016998027710324372, + "d_pool_kv": 0.00024510550786153437, + "d_pool_score": 0.00023626867066475846, + "d_ape": 2.881648668361276e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 174 + }, + "zigzag": { + "source": "layout-1m-count-tiny-mix-1000-zigzag.json", + "total_ms": 288.1846923828125, + "min_ms": 286.8497314453125, + "max_ms": 289.92547607421875, + "allocated_gib": 67.31179094314575, + "reserved_gib": 78.9453125, + "prepare_ms": 76.26319001428783, + "layout_prepare_ms": 75.76619903557003, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008172618750855098, + "dq": 0.0003660674276407079, + "dkv": 0.0002623909333939321, + "d_index_q": 0.00018360667662851773, + "d_weights": 0.00016406571075909486, + "d_pool_kv": 0.0002447536048006018, + "d_pool_score": 0.00023747323117413518, + "d_ape": 2.8442642578901785e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.49526814743876457, + "max_spans": 2000 + } + }, + "speedup_vs_contiguous": 0.99970561452031, + "gap_to_best_baseline_pct": 0.02944721680204765, + "prepare_plus_iteration_ms": { + "contiguous": 291.1374094169587, + "planner": 290.2131231231615, + "zigzag": 364.44788239710033 + }, + "prepare_inclusive_gap_pct": -0.31747424545962577, + "documents": 1000 + }, + "1m-single": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-single-contiguous.json", + "total_ms": 878.70654296875, + "min_ms": 874.7012329101562, + "max_ms": 881.0527954101562, + "allocated_gib": 73.0117564201355, + "reserved_gib": 75.380859375, + "prepare_ms": 7.158453925512731, + "layout_prepare_ms": 6.8812749814242125, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 7.9999911189079285, + "max_spans": 1 + }, + "planner": { + "source": "layout-1m-single-planner.json", + "total_ms": 664.1754760742188, + "min_ms": 663.5114135742188, + "max_ms": 668.2277221679688, + "allocated_gib": 73.01857900619507, + "reserved_gib": 75.88671875, + "prepare_ms": 15.7493349397555, + "layout_prepare_ms": 15.474541927687824, + "policy": "selection-local-2x", + "relative_rms": { + "output": 0.001168400329939734, + "dq": 0.00039265724673352774, + "dkv": 0.0005302362191601861, + "d_index_q": 0.0007076503704418045, + "d_weights": 0.0005489574816612602, + "d_pool_kv": 0.0007111586908343255, + "d_pool_score": 0.0006894097289940324, + "d_ape": 1.3500283722646205e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.9103830456733704e-11, + "workspace_gib": 7.9999911189079285, + "max_spans": 2 + }, + "zigzag": { + "source": "layout-1m-single-zigzag.json", + "total_ms": 668.0455932617188, + "min_ms": 664.5176391601562, + "max_ms": 668.7427978515625, + "allocated_gib": 73.01857900619507, + "reserved_gib": 75.884765625, + "prepare_ms": 12.595297070220113, + "layout_prepare_ms": 12.315480038523674, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011720185988330905, + "dq": 0.00039552385403338535, + "dkv": 0.0005946938023163722, + "d_index_q": 0.0006038215373079134, + "d_weights": 0.0005245172798592142, + "d_pool_kv": 0.000809623655331361, + "d_pool_score": 0.0007646019067327281, + "d_ape": 1.3008000488275949e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 7.9999911189079285, + "max_spans": 2 + } + }, + "speedup_vs_contiguous": 1.3230035956198996, + "gap_to_best_baseline_pct": -0.5793193198991409, + "prepare_plus_iteration_ms": { + "contiguous": 885.8649968942627, + "planner": 679.9248110139742, + "zigzag": 680.6408903319389 + }, + "prepare_inclusive_gap_pct": -0.10520662630413291, + "documents": 1 + } + }, + "max_relative_rms": { + "output": 0.0024348185275864243, + "dq": 0.0007232451638847554, + "dkv": 0.0005946938023163722, + "d_index_q": 0.000768516565343547, + "d_weights": 0.0005601425977036474, + "d_pool_kv": 0.000809623655331361, + "d_pool_score": 0.0007646019067327281, + "d_ape": 3.0188638995342922e-05, + "kda_output": 0, + "kda_dq": 0, + "kda_dk": 0, + "kda_dv": 0, + "kda_dg": 0, + "kda_dbeta": 0 + }, + "loss_abs_diff": 1.5133991837501526e-09, + "independent_kda": [ + { + "case": "uniform-1024", + "label": "baseline", + "passed": true, + "total_ms": 13.65231990814209 + }, + { + "case": "uniform-1024", + "label": "candidate", + "passed": true, + "total_ms": 12.879487991333008 + }, + { + "case": "uniform-4096", + "label": "baseline", + "passed": true, + "total_ms": 14.166272163391113 + }, + { + "case": "uniform-4096", + "label": "candidate", + "passed": true, + "total_ms": 12.644831657409668 + }, + { + "case": "uniform-16384", + "label": "baseline", + "passed": true, + "total_ms": 26.2923526763916 + }, + { + "case": "uniform-16384", + "label": "candidate", + "passed": true, + "total_ms": 23.619583129882812 + }, + { + "case": "near-uniform", + "label": "baseline", + "passed": true, + "total_ms": 25.801279067993164 + }, + { + "case": "near-uniform", + "label": "candidate", + "passed": true, + "total_ms": 23.4333438873291 + }, + { + "case": "weak-decay-uniform", + "label": "baseline", + "passed": true, + "total_ms": 26.233535766601562 + }, + { + "case": "weak-decay-uniform", + "label": "candidate", + "passed": true, + "total_ms": 23.613311767578125 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "baseline", + "passed": true, + "total_ms": 29.493568420410156 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "candidate", + "passed": true, + "total_ms": 29.489408493041992 + } + ], + "raw_sha256": { + "layout-check-single-contiguous.json": "1fe908c09dff05415909118c66af592465e8f91343458d52a983c1c7b13ce871", + "layout-check-uniform-zigzag.json": "3285988d7171e294fe04d1dcb5478bd7075ca1ecd3767243b171f11f63ef8eb7", + "layout-check-dense-zigzag.json": "d23fb5db1ec3e09cbd0188e0b551d71c5fb4c00c7c0fbd745bff637740f4bc8a", + "layout-check-mixed-planner.json": "d846a3cbaff139fb00a4925639ad41892c8309870a9db51528a4de303e5eab5a", + "layout-1m-single-planner.json": "1dd73977e2e22a6c2a0e43d9834e3cf1054faa5ee24fe0a0c6e30d89093c2a37", + "layout-1m-count-mixed-1000-1long-zigzag.json": "c49c0de6b7a58a18a0829a98f80b560c80a4d61dec734e70eb208d51caee5b8b", + "layout-check-mixed-zigzag.json": "571f0e879504feb4dba87b75352e2e147b75054ff654ec94829d844bce2bfca2", + "layout-1m-count-mixed-1000-1long-contiguous.json": "1746f1715f6c65066db2ba233c76d34eb5088e1c26c5fb129a83c6791ab6f893", + "layout-check-single-zigzag.json": "2448feea4f6eb7c30e71d9901d301149ca38a716a578cd78842da88e446210bb", + "layout-1m-count-tiny-mix-1000-contiguous.json": "676fe0e3c57bf5f367a318771a62766a3c26711e34075d652bc1c26e4a2320ef", + "layout-check-single-planner.json": "c405c4b8908c997d16a6663e8b01e7d07c4307ec52d0845fe529f892e3304469", + "layout-check-uniform-planner.json": "ca3388e5689d4dbeacc218952621ba9e12a128664f7e56853ae3e00f94befdd4", + "layout-check-mixed-contiguous.json": "86fda36489cd9e03c72137363ee8fe90eb2cca8a823c9031656ccaca5bff74ab", + "layout-1m-single-contiguous.json": "f72c50eb44db7e50c7e09635063c63c9b97363764b0c7cc76b9afc66d6297916", + "layout-check-dense-planner.json": "4c159ab030611c0c9a29b81a10d46e1cde31cd14a8ff324b2a6836f7519c8fa5", + "layout-1m-single-zigzag.json": "0e50108c69a225344769d79dacaba4e9f70db7e1c8d2bd0c05a8e660bffc8384", + "layout-1m-count-tiny-mix-1000-planner.json": "83b4f0d29314fbaeb5884ea3e8d564c5ecb844ccf99c156c9157d29c284842cb", + "layout-1m-count-tiny-mix-1000-zigzag.json": "2a258d9126aafcb5c4bd0a6982e60c3f5c2b7834f7643b459c35013af991fe88", + "layout-check-uniform-contiguous.json": "96a42331a4114a8a30f2cc8294f1987dfab63b4b477281081b0c013cc63ce4c4", + "layout-check-dense-contiguous.json": "1074540bc05ba6fde4c79146e380af1ef0375be014bae0bd094fd9843bfba259", + "layout-1m-count-mixed-1000-1long-planner.json": "09f3b10bad5248e5df5039485a16f5189aea02aa1ca3d18c2a621ff587129f2a" + }, + "independent_raw_sha256": "67bebb11c6aac209a0cdae42d493378fada7952f4df806fdd3007a735cf9cf10" + } + } +} diff --git a/tests/test_csa_matrix_report.py b/tests/test_csa_matrix_report.py new file mode 100644 index 00000000..98604098 --- /dev/null +++ b/tests/test_csa_matrix_report.py @@ -0,0 +1,107 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest + +from benchmarks.summarize_csa_layout_matrix import summarize + + +def _matrix(tmp_path): + tensors = [ + "output", + "dq", + "dkv", + "d_index_q", + "d_weights", + "d_pool_kv", + "d_pool_score", + "d_ape", + "kda_output", + "kda_dq", + "kda_dk", + "kda_dv", + "kda_dg", + "kda_dbeta", + ] + for layout, latency, setup in [("contiguous", 10, 1), ("zigzag", 8, 3), ("planner", 7, 5)]: + rows = [ + dict( + rank=rank, + finite=True, + input_match=True, + redistribution_bitwise=True, + samples=[dict(total_ms=latency + rank, peak_allocated_gib=1, peak_reserved_gib=2)], + layout_prepare_ms=setup, + common_prepare_ms=2, + kda_prepare_ms=3, + total_prepare_ms=setup + 5, + selected_policy=layout, + selection_workspace_bytes=0, + selection_spans=1, + layout_vs_contiguous=( + None if layout == "contiguous" else {k: dict(finite=True, relative_rms=0, max_abs=0) for k in tensors} + ), + loss_abs_diff=None if layout == "contiguous" else 0, + ) + for rank in range(2) + ] + data = dict(world_size=2, config=dict(layout=layout, repeats=1, sequence_lengths=[1 << 20], with_kda=True), ranks=rows) + (tmp_path / f"layout-1m-fixture-{layout}.json").write_text(json.dumps(data)) + + +def test_report_distinguishes_steady_and_preparation_inclusive_cost(tmp_path): + _matrix(tmp_path) + case = summarize(tmp_path)["1m-fixture"] + assert case["speedup_vs_contiguous"] == 11 / 8 + assert case["prepare_plus_iteration_ms"] == dict(contiguous=17, zigzag=17, planner=18) + assert case["prepare_inclusive_gap_pct"] == pytest.approx(100 * (18 / 17 - 1)) + + +@pytest.mark.parametrize( + "broken", + [ + "missing_all", + "missing_dg", + "bad_loss", + "nan_loss", + "missing_rank", + "nan_time", + "negative_memory", + "heads", + "kda_batch_invariant", + "kda_gate_scale", + "candidate_mib", + "nan_prepare", + "inconsistent_prepare", + ], +) +def test_report_rejects_incomplete_precision_evidence(tmp_path, broken): + _matrix(tmp_path) + path = tmp_path / "layout-1m-fixture-planner.json" + data = json.loads(path.read_text()) + row = data["ranks"][0] + if broken == "missing_all": + row["layout_vs_contiguous"] = None + elif broken == "missing_dg": + del row["layout_vs_contiguous"]["kda_dg"] + elif broken == "bad_loss": + row["loss_abs_diff"] = 1 + elif broken == "nan_loss": + row["loss_abs_diff"] = float("nan") + elif broken == "missing_rank": + data["ranks"].pop() + elif broken == "nan_time": + row["samples"][0]["total_ms"] = float("nan") + elif broken == "negative_memory": + row["samples"][0]["peak_allocated_gib"] = -1 + elif broken in ("heads", "kda_batch_invariant", "kda_gate_scale", "candidate_mib"): + data["config"][broken] = 999 + elif broken == "nan_prepare": + row["total_prepare_ms"] = float("nan") + elif broken == "inconsistent_prepare": + row["total_prepare_ms"] += 1 + path.write_text(json.dumps(data)) + with pytest.raises(ValueError): + summarize(tmp_path) diff --git a/tests/test_full_csa_cp_metadata.py b/tests/test_full_csa_cp_metadata.py new file mode 100644 index 00000000..26415442 --- /dev/null +++ b/tests/test_full_csa_cp_metadata.py @@ -0,0 +1,59 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks of the experimental full-CSA benchmark's addressing contract.""" + +import pytest +import torch + +from benchmarks.bench_full_csa_cp import expand_blocks, query_metadata +from cula.distributed import THDRankLayout, THDSpan + + +@pytest.mark.parametrize("visible", [1, 2, 3, 4, 5, 6, 7, 8, 17]) +@pytest.mark.parametrize("origin", [0, 128, 1024]) +def test_complete_blocks_and_causal_tail(visible, origin): + count = min(visible // 4, 4) + ids = torch.full((1, 8), -1, dtype=torch.int32) + # Valid entries intentionally have holes; ids/scores must stay paired. + ids[0, 1 : 2 * count : 2] = torch.arange(count) + scores = ids.float() + 0.25 + sorted_ids, sorted_scores, members, tokens, lengths = expand_blocks( + ids, scores, torch.tensor([origin]), torch.tensor([visible]) + ) + actual = tokens[0, : lengths.item()].tolist() + expected = list(range(origin, origin + 4 * count)) + expected += list(range(origin + visible // 4 * 4, origin + visible)) + assert actual == expected + assert len(actual) == len(set(actual)) + assert torch.all(tokens[0, lengths.item() :] == -1) + assert torch.all(members[0, 4 * count :] == -1) + assert torch.equal(sorted_scores, sorted_ids.float() + 0.25) + + +def test_all_selected_blocks_leave_room_for_three_tail_tokens(): + ids = torch.arange(128, dtype=torch.int32)[None] + _, _, _, tokens, lengths = expand_blocks(ids, ids.float(), torch.tensor([2048]), torch.tensor([515])) + assert lengths.item() == 515 + assert tokens.shape[1] % 128 == 0 + assert torch.equal(tokens[0, :515], torch.arange(2048, 2048 + 515)) + assert torch.all(tokens[0, 515:] == -1) + + +@pytest.mark.parametrize("spans", [(), ((128, 160, 64, 256),), ((512, 520, 256, 1024), (64, 128, 0, 256))]) +def test_bulk_metadata_preserves_addresses_including_reordered_and_empty_spans(spans): + layout = THDRankLayout(0, tuple(THDSpan(i, a, z, start, end, 0) for i, (a, z, start, end) in enumerate(spans))) + origin, visible, rows = query_metadata(layout, "cpu") + expected = [(start, t - start + 1, t) for a, z, start, end in spans for t in range(a, z)] + assert torch.stack((origin, visible, rows), dim=1).tolist() == [list(row) for row in expected] + assert origin.dtype == visible.dtype == rows.dtype == torch.int32 + assert all(x.is_contiguous() for x in (origin, visible, rows)) + + +def test_bulk_metadata_is_prepared_on_cpu_under_a_different_default_device(): + layout = THDRankLayout(0, (THDSpan(0, 4, 8, 0, 16, 0),)) + with torch.device("meta"): + origin, visible, rows = query_metadata(layout, "cpu") + assert origin.tolist() == [0] * 4 + assert visible.tolist() == [5, 6, 7, 8] + assert rows.tolist() == [4, 5, 6, 7] diff --git a/tests/test_thd_1m_cases.py b/tests/test_thd_1m_cases.py new file mode 100644 index 00000000..7b78a1e1 --- /dev/null +++ b/tests/test_thd_1m_cases.py @@ -0,0 +1,91 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +from collections import Counter + +import pytest + +from benchmarks.bench_csa_layout_matrix import performance_cases +from benchmarks.thd_1m_cases import TOTAL_TOKENS, counted_cases, heldout_cases, manifest, statistics_cases, summarize + + +@pytest.mark.parametrize("case", statistics_cases(), ids=lambda case: case.name) +def test_exact_packing_and_provenance(case): + assert sum(case.lengths) == TOTAL_TOKENS + assert all(isinstance(n, int) and n > 0 and n % 4 == 0 for n in case.lengths) + for length, count in Counter(case.long_documents).items(): + assert case.lengths.count(length) == count + assert 0 <= case.final_document_truncation < 16384 + stats = summarize(case) + assert sum(b["documents"] for b in stats["token_length_histogram"]) == len(case.lengths) + assert sum(b["tokens"] for b in stats["token_length_histogram"]) == TOTAL_TOKENS + + +def test_reproducible_cases_and_order_only_controls(): + cases = statistics_cases() + assert cases == statistics_cases() + assert len({case.name for case in cases}) == len(cases) == 18 + lookup = {case.name: case for case in cases} + for base in ("1m-stat-mix80", "1m-stat-mix80-2x256k"): + for order in ("ascending", "descending", "alternating"): + candidate = lookup[f"{base}-{order}"].lengths + assert Counter(candidate) == Counter(lookup[base].lengths) + assert candidate != lookup[base].lengths + assert lookup[base + "-ascending"].lengths == tuple(sorted(lookup[base].lengths)) + assert lookup[base + "-descending"].lengths == tuple(sorted(lookup[base].lengths, reverse=True)) + assert lookup["1m-stat-mix80"].lengths != statistics_cases(seed=123)[2].lengths + + +def test_matrix_keeps_baseline_and_manifest_matches_execution(): + baseline = performance_cases() + assert len(baseline) == 5 + assert [c[0] for c in baseline] == ["1m-single", "1m-uniform", "1m-near-uniform", "1m-20s2l", "1m-power-law"] + cases = performance_cases("statistics", seed=123) + saved = manifest(seed=123)["cases"] + assert [(name, lengths) for name, lengths, _ in cases] == [(c["name"], list(c["lengths"])) for c in saved] + assert len(performance_cases("all")) == 39 + with pytest.raises(ValueError, match="unknown suite"): + performance_cases("typo") + + +@pytest.mark.parametrize("case", counted_cases(), ids=lambda case: case.name) +def test_fixed_document_counts(case): + assert sum(case.lengths) == TOTAL_TOKENS + assert 256 <= len(case.lengths) <= 1024 + assert all(n > 0 and n % 4 == 0 for n in case.lengths) + assert len(case.lengths) == int(next(s for s in case.name.split("-") if s in {"256", "512", "1000", "1024"})) + + +def test_counted_manifest_matches_the_actual_matrix(): + saved = manifest(seed=123, suite="counted")["cases"] + cases = performance_cases("counted", seed=123) + assert [(name, lengths) for name, lengths, _ in cases] == [(c["name"], list(c["lengths"])) for c in saved] + + +def test_heldout_lengths_vary_without_changing_budget_or_document_counts(): + first, second = heldout_cases(20261114), heldout_cases(20261115) + assert first == heldout_cases(20261114) + assert len(first) == 5 + assert {len(c.lengths) for c in first} == {257, 1000, 768, 513} + for a, b in zip(first, second): + assert a.name == b.name + assert len(a.lengths) == len(b.lengths) + assert sorted(a.lengths) != sorted(b.lengths) + for case in (a, b): + assert sum(case.lengths) == TOTAL_TOKENS + assert all(n > 0 and n % 4 == 0 for n in case.lengths) + saved = manifest(20261114, "heldout")["cases"] + actual = performance_cases("heldout", 20261114) + assert [(name, lengths) for name, lengths, _ in actual] == [(c["name"], list(c["lengths"])) for c in saved] + + +def test_validation_suite_has_31_distinct_exact_1m_shapes(): + from benchmarks.bench_csa_layout_matrix import performance_cases + + cases = performance_cases("validation") + assert len(cases) == len({name for name, _, _ in cases}) == 31 + assert all(sum(lengths) == 1 << 20 and all(n > 0 and n % 4 == 0 for n in lengths) for _, lengths, _ in cases) + for seed in (20261114, 20261115, 20261116): + observed = {name: lengths for name, lengths, _ in cases} + for case in heldout_cases(seed): + assert observed[f"{case.name}-seed{seed}"] == list(case.lengths) diff --git a/tests/test_thd_cost_fit.py b/tests/test_thd_cost_fit.py new file mode 100644 index 00000000..74dc833c --- /dev/null +++ b/tests/test_thd_cost_fit.py @@ -0,0 +1,136 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from benchmarks.bench_thd_transport import transport_layout +from benchmarks.calibrate_thd_selection import nonnegative_fit, routing_critical_ms +from cula.distributed import THDRedistributionPlan + + +def test_nonnegative_fit_recovers_independent_costs(): + x = [[1, 0, 0], [0, 1, 0], [0, 0, 1], [2, 3, 4]] + weights, rmse = nonnegative_fit(x, [2, 5, 7, 47]) + assert weights == pytest.approx([2, 5, 7]) + assert rmse == pytest.approx(0, abs=1e-10) + + +def test_fit_drops_negative_cost_instead_of_cancelling_compute(): + weights, _ = nonnegative_fit([[1, 0, 0], [0, 1, 0], [0, 0, 1]], [2, -5, 7]) + assert weights == pytest.approx([2, 0, 7]) + + +@pytest.mark.parametrize("features,targets", [([], []), ([[1, 2, 3]], []), ([[1, 2]], [3]), ([[float("nan"), 1, 2]], [3])]) +def test_fit_rejects_incomplete_or_nonfinite_measurements(features, targets): + with pytest.raises(ValueError): + nonnegative_fit(features, targets) + + +def test_routing_fit_excludes_kda_and_selection_arrival_waits(): + def rank(kda, query, selection, restore): + return dict( + kda_fwd_ms=kda, + query_redistribute_ms=query, + pool_fwd_ms=1, + pool_gather_ms=2, + selection_key_layout_ms=0.5, + selection_ms=selection, + selection_restore_ms=restore, + ) + + # Rank 0 waits 80 ms before query exchange and 60 ms before restore. + # Both exchanges take 5/3 ms after the last rank arrives, respectively. + a, b = rank(20, 85, 10, 63), rank(100, 5, 70, 3) + assert routing_critical_ms([a, b]) == pytest.approx(8.5) + # Shifting the slow phase earlier on the other rank leaves transfer cost + # unchanged. Neither the original waiting times nor rank identity matter. + assert routing_critical_ms([b, a]) == pytest.approx(8.5) + assert routing_critical_ms([rank(100, 5, 70, 3)] * 2) == pytest.approx(8.5) + + +def test_routing_fit_rejects_invalid_phase_measurements(): + with pytest.raises(ValueError): + routing_critical_ms([]) + + +@pytest.mark.parametrize("kind", ["intra", "intra_dense", "inter", "inter_dense", "striped"]) +@pytest.mark.parametrize("world,per_node", [(4, 2), (8, 4), (16, 8)]) +def test_transport_calibration_exercises_the_declared_links(kind, world, per_node): + rows = 128 + ranks = transport_layout(world, rows, per_node, kind) + for rank in range(world): + p = THDRedistributionPlan.build(ranks, tuple(i * rows for i in range(world + 1)), rank) + assert p.target_rows == p.source_rows == rows + assert sum(p.send_counts) == sum(p.receive_counts) == rows + assert not p.identity + peers = [i for i, count in enumerate(p.send_counts) if count] + if kind.startswith("intra"): + assert all(i // per_node == rank // per_node for i in peers) + assert (rank in peers) == (kind == "intra_dense") + elif kind.startswith("inter"): + assert all(i // per_node != rank // per_node for i in peers) + else: + assert peers == list(range(world)) + assert len(set(p.send_counts)) == 1 + + +@pytest.mark.parametrize("world,per_node", [(4, 2), (8, 8), (16, 8)]) +@pytest.mark.parametrize("kind", ["intra_pair", "intra", "intra_dense", "striped"]) +@pytest.mark.parametrize("overlap", [False, True]) +def test_transport_fit_prediction_uses_the_same_cost_model_as_planning(world, per_node, kind, overlap): + from benchmarks.fit_thd_transport import predict + from cula.distributed import THDSelectionCostModel + + rows = 128 + layouts = transport_layout(world, rows, per_node, kind) + plans = [THDRedistributionPlan.build(layouts, tuple(i * rows for i in range(world + 1)), r) for r in range(world)] + intra_out = [0] * world + inter_out = [0] * world + intra_in = [0] * world + inter_in = [0] * world + node_out = [0] * (world // per_node) + node_in = node_out.copy() + degrees_out = [0] * world + degrees_in = [0] * world + edge = [] + for s, p in enumerate(plans): + for t, n in enumerate(p.send_counts): + if s == t or not n: + continue + edge.append(n) + degrees_out[s] += 1 + degrees_in[t] += 1 + if s // per_node == t // per_node: + intra_out[s] += n + intra_in[t] += n + else: + inter_out[s] += n + inter_in[t] += n + node_out[s // per_node] += n + node_in[t // per_node] += n + f = dict( + rank_intra_rows=intra_out + intra_in, + rank_inter_rows=inter_out + inter_in, + peer_rows=max(edge), + node_rows=max(node_out + node_in), + peer_degree=max(degrees_out + degrees_in), + copies=0, + ) + coeff = [1, 3, 4, 2, 0.5, 0.007, 0.01] + factor = 65536 * 20608 / 1e6 + m = THDSelectionCostModel( + candidate_ns=0, + query_ns=0, + span_us=0, + bytes_per_query=20608, + rank_node_ids=tuple(r // per_node for r in range(world)), + intra_node_gbps=factor / coeff[0], + inter_node_gbps=factor / coeff[1], + peer_gbps=factor / coeff[2], + inter_node_aggregate_gbps=factor / coeff[3], + packing_ns_per_query=coeff[4] * 1e6 / 65536, + peer_startup_us=coeff[5] * 1000, + exchange_us=coeff[6] * 1000, + overlap_intra_inter=overlap, + ) + assert predict(f, coeff, overlap) == pytest.approx(m.estimate_ms(layouts, block_size=4)) diff --git a/tests/test_thd_planner.py b/tests/test_thd_planner.py index 963cd543..04587a09 100644 --- a/tests/test_thd_planner.py +++ b/tests/test_thd_planner.py @@ -3,10 +3,22 @@ import random from collections import defaultdict +from itertools import permutations, product import pytest -from cula.distributed.thd_planner import THDPlannerConfig, plan_thd_layout +from cula.distributed.thd_planner import ( + THDPlannerConfig, + THDRankLayout, + THDSelectionCostModel, + THDSpan, + _contiguous_items, + _localize_spans, + _minimum_assignment, + _transfer_lower_bound, + plan_thd_layout, + plan_thd_selection, +) def _boundaries(lengths): @@ -114,3 +126,423 @@ def test_random_thd_plans_preserve_coverage_and_never_lose_to_contiguous(): _assert_exact_cover(plan, boundaries) assert plan.metrics.score <= dict(plan.evaluated_scores)["contiguous"] assert max(plan.metrics.token_loads) - min(plan.metrics.token_loads) <= plan.block_size + + +def _cost_model(**overrides): + values = dict( + candidate_ns=0.1, + query_ns=1.0, + span_us=1.0, + bytes_per_query=20608, + intra_node_gbps=200.0, + inter_node_gbps=20.0, + packing_ns_per_query=20.0, + rank_node_ids=(0, 0, 1, 1), + exchange_us=10.0, + ) + return THDSelectionCostModel(**(values | overrides)) + + +def test_calibrated_model_rejects_migration_when_network_cost_dominates(): + lengths = [1 << 20] + cheap = plan_thd_layout(_boundaries(lengths), world_size=4, cost_model=_cost_model()) + expensive = plan_thd_layout( + _boundaries(lengths), + world_size=4, + cost_model=_cost_model(intra_node_gbps=0.001, inter_node_gbps=0.001), + ) + assert cheap.policy != "contiguous" + assert expensive.policy == "contiguous" + assert cheap.score_unit == expensive.score_unit == "ms" + _assert_exact_cover(cheap, _boundaries(lengths)) + assert cheap.metrics.score < dict(cheap.evaluated_scores)["contiguous"] + + +def test_identity_cost_contains_no_transport_and_margin_can_keep_it(): + boundaries = _boundaries([1024] * 1024) + plan = plan_thd_layout(boundaries, world_size=4, cost_model=_cost_model()) + assert plan.policy == "contiguous" + # One document's exact complete-block candidate count: sum(floor(i/4)). + candidates = sum(i // 4 for i in range(1, 1025)) * 256 + expected = candidates * 0.1 / 1e6 + 262144 / 1e6 + 256 / 1e3 + assert plan.metrics.score == pytest.approx(expected) + assert len(plan.evaluated_scores) == 1 # Proven lower bound prunes all migrations. + conservative = plan_thd_layout([0, 1 << 20], world_size=4, cost_model=_cost_model(min_gain_fraction=0.99)) + assert conservative.policy == "contiguous" + + +@pytest.mark.parametrize( + "overrides", + [ + {"candidate_ns": float("nan")}, + {"inter_node_gbps": 0}, + {"packing_ns_per_query": -1}, + {"min_gain_fraction": 1}, + {"bytes_per_query": True}, + {"rank_node_ids": (0, False)}, + {"peer_gbps": 0}, + {"peer_gbps": float("inf")}, + {"peer_gbps": True}, + {"min_gain_ms": -1}, + {"min_gain_ms": float("nan")}, + {"min_gain_ms": True}, + ], +) +def test_cost_model_rejects_invalid_profiles(overrides): + with pytest.raises((ValueError, TypeError)): + _cost_model(**overrides) + + +def test_cost_model_requires_matching_group_topology(): + with pytest.raises(ValueError, match="node map"): + plan_thd_layout([0, 65536], world_size=8, cost_model=_cost_model()) + + +@pytest.mark.parametrize("selection", [False, True]) +def test_absolute_saving_budget_prunes_search_without_changing_relative_margin(selection): + kwargs = {"max_tokens_per_rank": 524288} if selection else {} + planner = plan_thd_selection if selection else plan_thd_layout + reusable = planner([0, 1 << 20], world_size=4, cost_model=_cost_model(), **kwargs) + assert reusable.policy != "contiguous" + # Above the entire baseline cost, no possible redistribution can repay + # the caller's preparation budget. No candidate search is necessary. + one_use = planner([0, 1 << 20], world_size=4, cost_model=_cost_model(min_gain_ms=10000), **kwargs) + assert one_use.policy == "contiguous" + assert len(one_use.evaluated_scores) == 1 + savings = dict(reusable.evaluated_scores)["contiguous"] - reusable.metrics.score + rejected = planner([0, 1 << 20], world_size=4, cost_model=_cost_model(min_gain_ms=savings + 1), **kwargs) + assert rejected.policy == "contiguous" + + +def test_rank_matching_equals_exhaustive_optimum(): + generator = random.Random(20261113) + for size in range(1, 7): + for _ in range(8): + costs = [[generator.random() * 100 for _ in range(size)] for _ in range(size)] + assignment = _minimum_assignment(costs) + actual = sum(costs[i][j] for i, j in enumerate(assignment)) + expected = min(sum(costs[i][j] for i, j in enumerate(p)) for p in permutations(range(size))) + assert sorted(assignment) == list(range(size)) + assert actual == pytest.approx(expected) + assert _minimum_assignment([[0.0] * 4 for _ in range(4)]) == (0, 1, 2, 3) + + +@pytest.mark.parametrize("total", [65536, 65540]) +def test_localized_bins_preserve_capacity_and_do_not_worsen_chosen_score(total): + plan = plan_thd_layout([0, total], world_size=4, cost_model=_cost_model(min_gain_fraction=0.0)) + _assert_exact_cover(plan, [0, total]) + assert max(plan.metrics.token_loads) - min(plan.metrics.token_loads) <= 4 + scores = dict(plan.evaluated_scores) + assert plan.metrics.score <= min(value for name, value in scores.items() if not name.endswith("-local")) + original = plan_thd_layout([0, total], world_size=4) + localized = _localize_spans([s for r in original.ranks for s in r.spans], 4, 4, _cost_model()) + loads = [sum(s.tokens for s in localized if s.owner == r) for r in range(4)] + assert tuple(loads) == original.metrics.token_loads + + +@pytest.mark.parametrize("capacity_factor", [1, 1.25, 2]) +def test_selection_planner_preserves_cover_and_explicit_capacity(capacity_factor): + rng = random.Random(62813) + for _ in range(25): + lengths = [4 * rng.randint(1, 12000) for _ in range(rng.randint(1, 40))] + cu = _boundaries(lengths) + capacity = 4 * int(((cu[-1] // 4 + 3) // 4) * capacity_factor) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=capacity, cost_model=_cost_model()) + # Empty target ranks are legal. Exact coverage is checked independently + # of scheduling order and must survive inverse redistribution. + spans = sorted((s.token_start, s.token_end) for r in plan.ranks for s in r.spans) + assert spans[0][0] == 0 and spans[-1][1] == cu[-1] + assert all(a[1] == b[0] for a, b in zip(spans, spans[1:])) + for r in plan.ranks: + assert r.local_tokens <= capacity + for s in r.spans: + assert s.owner == r.rank + assert (s.sequence_start, s.sequence_end) == tuple(cu[s.sequence : s.sequence + 2]) + assert s.sequence_start <= s.token_start < s.token_end <= s.sequence_end + assert plan.metrics.score <= dict(plan.evaluated_scores)["contiguous"] + assert plan == plan_thd_selection(cu, world_size=4, max_tokens_per_rank=capacity, cost_model=_cost_model()) + + +def test_selection_keeps_short_documents_in_source_ownership(): + cu = _boundaries([64] * 100 + [524288, 517888]) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=524288, cost_model=_cost_model()) + assert plan.policy != "contiguous" + for layout in plan.ranks: + for span in layout.spans: + if span.sequence < 100: + assert span.owner == 0 + assert span.tokens == 64 + + +@pytest.mark.parametrize("capacity,error", [(True, TypeError), (0, ValueError), (15, ValueError), (12, ValueError)]) +def test_selection_rejects_invalid_row_capacity(capacity, error): + with pytest.raises(error, match="max_tokens_per_rank"): + plan_thd_selection([0, 64], world_size=4, max_tokens_per_rank=capacity, cost_model=_cost_model()) + + +@pytest.mark.parametrize("lengths", [[32], [4, 12, 16], [4] * 8]) +@pytest.mark.parametrize("bandwidth", [0.01, 1, 10000]) +def test_transfer_lower_bound_never_exceeds_exhaustive_partition_optimum(lengths, bandwidth): + # Enumerate every two-rank assignment of eight blocks, including empty + # ranks and unequal counts. Coalesce before scoring to avoid counting + # artificial fragment costs as necessary work. + cu = tuple(_boundaries(lengths)) + model = _cost_model(rank_node_ids=(0, 1), intra_node_gbps=bandwidth, inter_node_gbps=bandwidth) + source = _contiguous_items(cu, 2, 4) + ranks = tuple( + THDRankLayout( + r, + tuple( + THDSpan(i.sequence, i.token_start, i.token_end, i.sequence_start, i.sequence_end, r) + for i, owner in source + if owner == r + ), + ) + for r in range(2) + ) + total_work = sum(sum(i // 4 for i in range(1, length + 1)) for length in lengths) + mean = (model.candidate_ns * total_work / 1e6 + model.query_ns * 32 / 1e6 + model.span_us * len(lengths) / 1e3) / 2 + bound = _transfer_lower_bound(source, 2, 4, model, mean, model.estimate_ms(ranks, block_size=4)) + optimum = float("inf") + for owners in product(range(2), repeat=8): + spans = [] + for seq, (start, end) in enumerate(zip(cu, cu[1:])): + a = start + for z in range(start + 4, end + 1, 4): + if z == end or owners[z // 4] != owners[a // 4]: + spans.append(THDSpan(seq, a, z, start, end, owners[a // 4])) + a = z + candidate = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(2)) + optimum = min(optimum, model.estimate_ms(candidate, block_size=4)) + assert bound <= optimum + 1e-12 + + +def test_peer_limit_prices_aggregate_edge_rows_not_individual_fragments(): + from benchmarks.bench_thd_transport import transport_layout + + model = _cost_model( + candidate_ns=0, + query_ns=0, + span_us=0, + packing_ns_per_query=0, + exchange_us=0, + peer_gbps=10, + intra_node_gbps=1000, + inter_node_gbps=1000, + ) + permutation = transport_layout(4, 128, 2, "inter") + fragmented = tuple( + THDRankLayout( + r.rank, + tuple( + THDSpan(s.sequence, a, a + 64, s.sequence_start, s.sequence_end, s.owner) + for s in r.spans + for a in range(s.token_start, s.token_end, 64) + ), + ) + for r in permutation + ) + expected = 128 * model.bytes_per_query / (10 * 1e6) + assert model.estimate_ms(permutation, block_size=4) == pytest.approx(expected) + assert model.estimate_ms(fragmented, block_size=4) == pytest.approx(expected) + striped = transport_layout(4, 128, 2, "striped") + assert model.estimate_ms(striped, block_size=4) == pytest.approx(expected / 4) + + +@pytest.mark.parametrize("world", [2, 4, 8, 16, 32, 64]) +@pytest.mark.parametrize("peer_rate", [0.01, 12.0, 500.0]) +def test_selection_peer_model_remains_bounded_on_unseen_topologies(world, peer_rate): + rng = random.Random(73381 + world) + lengths = [4 * rng.randint(1, 256) for _ in range(127)] + [131072, 262144] + rng.shuffle(lengths) + cu = _boundaries(lengths) + capacity = 8 * ((cu[-1] // 4 + world - 1) // world) + model = _cost_model(rank_node_ids=tuple(r // 8 for r in range(world)), peer_gbps=peer_rate) + plan = plan_thd_selection(cu, world_size=world, max_tokens_per_rank=capacity, cost_model=model) + spans = sorted((s.token_start, s.token_end) for r in plan.ranks for s in r.spans) + assert spans[0][0] == 0 and spans[-1][1] == cu[-1] + assert all(a[1] == b[0] for a, b in zip(spans, spans[1:])) + assert max(plan.metrics.token_loads) <= capacity + assert plan.metrics.score <= dict(plan.evaluated_scores)["contiguous"] + assert plan.metrics.score == model.estimate_ms(plan.ranks, block_size=4) + + +def test_full_zigzag_can_win_even_when_all_documents_are_below_split_threshold(): + from benchmarks.cp_layouts import query_layouts + + cu = _boundaries([16384, 4096, 1024]) + model = _cost_model( + candidate_ns=10, + query_ns=0, + span_us=0, + packing_ns_per_query=0, + exchange_us=0, + intra_node_gbps=1e9, + inter_node_gbps=1e9, + min_gain_fraction=0, + ) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=16384, cost_model=model) + zigzag, _ = query_layouts(cu, 4, "zigzag") + assert plan.policy == "zigzag" + assert plan.ranks == zigzag + assert dict(plan.evaluated_scores)["zigzag"] == model.estimate_ms(zigzag, block_size=4) + assert len(plan.evaluated_scores) == 2 # No long-document candidates. + _assert_exact_cover(plan, cu) + + +def test_full_zigzag_obeys_capacity_even_when_short_chunks_concentrate_on_rank_zero(): + cu = _boundaries([4] * 8 + [64]) + model = _cost_model(candidate_ns=10000, min_gain_fraction=0, min_gain_ms=0) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=24, cost_model=model) + assert "zigzag" not in dict(plan.evaluated_scores) + assert max(plan.metrics.token_loads) <= 24 + + +@pytest.mark.parametrize("world", [2, 4, 8, 16, 64]) +def test_shared_full_zigzag_matches_explicit_early_late_block_definition(world): + from cula.distributed import zigzag_thd_layout + + lengths = [4, 12, 64, 260, 16384] + cu = _boundaries(lengths) + ranks = zigzag_thd_layout(cu, world_size=world) + for r in range(world): + expected = [] + for d, length in enumerate(lengths): + q, rem = divmod(length // 4, 2 * world) + for piece in (r, 2 * world - 1 - r): + begin = cu[d] + 4 * (piece * q + min(piece, rem)) + end = cu[d] + 4 * ((piece + 1) * q + min(piece + 1, rem)) + if end > begin: + expected.append((d, begin, end)) + assert [(s.sequence, s.token_start, s.token_end) for s in ranks[r].spans] == expected + + +def test_shared_node_bandwidth_and_peer_startup_are_not_per_fragment_costs(): + from benchmarks.bench_thd_transport import transport_layout + + model = _cost_model( + candidate_ns=0, + query_ns=0, + span_us=0, + packing_ns_per_query=0, + exchange_us=0, + intra_node_gbps=1e9, + inter_node_gbps=1e9, + inter_node_aggregate_gbps=10, + peer_startup_us=7, + ) + pair = transport_layout(4, 128, 2, "inter_pair") + ring = transport_layout(4, 128, 2, "inter") + dense = transport_layout(4, 128, 2, "inter_dense") + row_ms = 128 * model.bytes_per_query / (10 * 1e6) + assert model.estimate_ms(pair, block_size=4) == pytest.approx(row_ms + 0.007) + assert model.estimate_ms(ring, block_size=4) == pytest.approx(2 * row_ms + 0.007) + assert model.estimate_ms(dense, block_size=4) == pytest.approx(2 * row_ms + 0.014) + + +@pytest.mark.parametrize( + "overrides", + [ + dict(peer_startup_us=-1), + dict(peer_startup_us=True), + dict(peer_startup_us=float("nan")), + dict(inter_node_aggregate_gbps=0), + dict(inter_node_aggregate_gbps=True), + dict(inter_node_aggregate_gbps=float("inf")), + ], +) +def test_topology_model_rejects_invalid_limits(overrides): + with pytest.raises(ValueError): + _cost_model(**overrides) + + +@pytest.mark.parametrize("world", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("overlap", [False, True]) +@pytest.mark.parametrize("topology", ["contiguous", "interleaved", "irregular"]) +def test_zigzag_bound_never_overprices_concrete_layout(world, overlap, topology): + from cula.distributed.thd_planner import _zigzag_lower_bound, zigzag_thd_layout + + rng = random.Random(27113 + world) + for _ in range(50): + cu = _boundaries([4 * rng.randint(1, 8192) for _ in range(rng.randint(1, 50))]) + model = _cost_model( + rank_node_ids=tuple( + r // 3 if topology == "contiguous" else r % 3 if topology == "interleaved" else rng.randrange(world) + for r in range(world) + ), + peer_startup_us=7, + inter_node_aggregate_gbps=12, + peer_gbps=8, + overlap_intra_inter=overlap, + ) + source = _contiguous_items(cu, world, 4) + bound = _zigzag_lower_bound(cu, source, world, 4, model) + actual = model.estimate_ms(zigzag_thd_layout(cu, world_size=world), block_size=4) + assert bound <= actual + 1e-10 + + +def test_transport_prices_only_executed_permutations(): + from dataclasses import replace + + from benchmarks.bench_thd_transport import transport_layout + + layouts = transport_layout(4, 128, 2, "striped") + model = _cost_model( + candidate_ns=0, + query_ns=0, + span_us=0, + packing_ns_per_query=1000, + exchange_us=0, + intra_node_gbps=1000, + inter_node_gbps=1000, + ) + unpriced = replace(model, packing_ns_per_query=0) + assert model.estimate_ms(layouts, block_size=4) == unpriced.estimate_ms(layouts, block_size=4) + reversed_layouts = tuple(THDRankLayout(r.rank, tuple(reversed(r.spans))) for r in layouts) + assert model.estimate_ms(reversed_layouts, block_size=4) - unpriced.estimate_ms( + reversed_layouts, block_size=4 + ) == pytest.approx(0.128) + + +def test_independent_link_model_and_its_outgoing_bound(): + from dataclasses import replace + + from benchmarks.bench_thd_transport import transport_layout + + ranks = transport_layout(4, 128, 2, "striped") + serial = _cost_model( + candidate_ns=0, query_ns=0, span_us=0, packing_ns_per_query=0, exchange_us=0, intra_node_gbps=10, inter_node_gbps=10 + ) + parallel = replace(serial, overlap_intra_inter=True) + assert parallel.estimate_ms(ranks, block_size=4) == pytest.approx(2 / 3 * serial.estimate_ms(ranks, block_size=4)) + assert parallel._outgoing_capacity_gbps(0) == 20 + assert serial._outgoing_capacity_gbps(0) == 10 + with pytest.raises(TypeError, match="boolean"): + replace(serial, overlap_intra_inter=1) + + +def test_parallel_link_lower_bound_against_all_small_assignments(): + from cula.distributed.thd_planner import _coalesce, _make_item + + cu = (0, 4, 12, 24) + world = 3 + model = _cost_model( + rank_node_ids=(0, 0, 1), + candidate_ns=10000, + query_ns=10, + overlap_intra_inter=True, + peer_startup_us=2, + inter_node_aggregate_gbps=25, + ) + base = _contiguous_items(cu, world, 4) + spans = _coalesce(base) + source = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world)) + mean = ( + model.candidate_ns * sum(i.causal_work for i, _ in base) + model.query_ns * 24 + ) / 1e6 / world + model.span_us * 3 / 1e3 / world + bound = _transfer_lower_bound(base, world, 4, model, mean, model.estimate_ms(source, block_size=4)) + blocks = [_make_item(d, a, a + 4, cu[d], cu[d + 1], 4) for d in range(3) for a in range(cu[d], cu[d + 1], 4)] + for owners in product(range(world), repeat=6): + spans = _coalesce(zip(blocks, owners)) + ranks = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world)) + assert bound <= model.estimate_ms(ranks, block_size=4) + 1e-10 diff --git a/tests/test_thd_redistribute.py b/tests/test_thd_redistribute.py new file mode 100644 index 00000000..c4935e72 --- /dev/null +++ b/tests/test_thd_redistribute.py @@ -0,0 +1,130 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU protocol tests, including unequal counts and a nontrivial receive order.""" + +from dataclasses import replace + +import pytest +import torch + +from benchmarks.cp_layouts import query_layouts +from cula._metadata import balanced_shards +from cula.distributed import THDRedistributionPlan, THDRedistributor + + +@pytest.mark.parametrize("policy", ["contiguous", "zigzag", "planner"]) +@pytest.mark.parametrize("boundaries", [(0, 64), (0, 4, 12, 32, 64), (0, 32768), (0, 4, 8, 12, 16)]) +def test_exchange_and_its_transpose(monkeypatch, policy, boundaries): + world = 4 + layouts, _ = query_layouts(boundaries, world, policy) + # Reverse local span order to require a genuine receive permutation too. + if policy != "contiguous": + layouts = tuple(replace(x, spans=x.spans[::-1]) for x in layouts) + offsets = balanced_shards(boundaries[-1], world) + plans = [THDRedistributionPlan.build(layouts, offsets, rank) for rank in range(world)] + source = [torch.arange(offsets[r], offsets[r + 1], dtype=torch.float64)[:, None].repeat(1, 3) for r in range(world)] + target = [ + ( + torch.cat([torch.arange(s.token_start, s.token_end, dtype=torch.float64) for s in x.spans]) + if x.spans + else torch.empty(0, dtype=torch.float64) + )[:, None].repeat(1, 3) + for x in layouts + ] + for rank, plan in enumerate(plans): + monkeypatch.setattr("torch.distributed.get_world_size", lambda group: world) + monkeypatch.setattr("torch.distributed.get_rank", lambda group: rank) + exchange = THDRedistributor(plan, "cpu") + calls = [] + reverse, factor = False, 1 + + def collective(out, packed, output_split_sizes, input_split_sizes, group): + calls.append(reverse) + tensors = [x * factor for x in (target if reverse else source)] + parts = [] + for sender, peer_plan in enumerate(plans): + from cula.distributed.redistribute import _permutations + + ranges = peer_plan.receive_ranges if reverse else peer_plan.send_ranges + rows = peer_plan.target_rows if reverse else peer_plan.source_rows + order, _ = _permutations(ranges, rows, "cpu") + data = tensors[sender] if order is None else tensors[sender].index_select(0, order) + counts = peer_plan.receive_counts if reverse else peer_plan.send_counts + begin = sum(counts[:rank]) + parts.append(data[begin : begin + counts[rank]]) + out.copy_(torch.cat(parts)) + expected_counts = plan.receive_counts if reverse else plan.send_counts + assert tuple(input_split_sizes) == expected_counts + assert sum(output_split_sizes) == out.shape[0] + # Verify the sender's local packing as well, independently of the + # mocked receive construction. + ranges = plan.receive_ranges if reverse else plan.send_ranges + idx = torch.tensor([i for a, z in ranges for i in range(a, z)], dtype=torch.int64) + assert torch.equal(packed, tensors[rank].index_select(0, idx)) + + monkeypatch.setattr("torch.distributed.all_to_all_single", collective) + leaf = source[rank].clone().requires_grad_() + routed = exchange.distribute(leaf) + assert torch.equal(routed, target[rank]) + reverse, factor = True, 2 + # Nonuniform gradients expose a wrong inverse even when a sum-of-rows + # test would pass. Empty destinations still participate in backward. + routed.backward(target[rank] * factor) + assert torch.equal(leaf.grad, source[rank] * factor) + reverse, factor = True, 1 + leaf = target[rank].clone().requires_grad_() + restored = exchange.restore(leaf) + assert torch.equal(restored, source[rank]) + reverse, factor = False, 3 + restored.backward(source[rank] * factor) + assert torch.equal(leaf.grad, target[rank] * factor) + assert calls == ([] if plan.identity else [False, True, True, False]) + + +def test_reject_overlap(): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + broken = (layouts[0], replace(layouts[1], spans=layouts[0].spans), *layouts[2:]) + with pytest.raises(ValueError): + THDRedistributionPlan.build(broken, (0, 16, 32, 48, 64), 0) + + +@pytest.mark.parametrize("field,value", [("owner", False), ("token_start", 0.0), ("token_end", 16.0), ("sequence_end", 64.0)]) +def test_reject_noninteger_span_metadata_before_collectives(field, value): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + span = replace(layouts[0].spans[0], **{field: value}) + layouts = (replace(layouts[0], spans=(span,)), *layouts[1:]) + with pytest.raises(TypeError, match="must be integers"): + THDRedistributionPlan.build(layouts, (0, 16, 32, 48, 64), 0) + + +def test_locally_unchanged_rank_cannot_skip_a_peer_exchange(): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + layouts = tuple( + replace(layouts[source], rank=target, spans=tuple(replace(s, owner=target) for s in layouts[source].spans)) + for target, source in enumerate((0, 2, 1, 3)) + ) + plans = [THDRedistributionPlan.build(layouts, (0, 16, 32, 48, 64), r) for r in range(4)] + assert plans[0].remote_rows == plans[3].remote_rows == 0 + assert all(not plan.identity for plan in plans) + + +def test_prepare_indices_do_not_follow_the_framework_default_device(): + from cula.distributed.redistribute import _permutations + + with torch.device("meta"): + order, inverse = _permutations(((4, 8), (0, 4)), 8, "cpu") + assert order.device.type == inverse.device.type == "cpu" + assert order.tolist() == [4, 5, 6, 7, 0, 1, 2, 3] + assert inverse.tolist() == [4, 5, 6, 7, 0, 1, 2, 3] + + +def test_unindexed_cuda_device_is_resolved_at_prepare(monkeypatch): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + plan = THDRedistributionPlan.build(layouts, (0, 16, 32, 48, 64), 2) + monkeypatch.setattr("torch.distributed.get_world_size", lambda group: 4) + monkeypatch.setattr("torch.distributed.get_rank", lambda group: 2) + monkeypatch.setattr("torch.cuda.current_device", lambda: 3) + # Identity needs no device allocation, so this also runs without CUDA. + exchange = THDRedistributor(plan, "cuda") + assert exchange.device == torch.device("cuda:3") From 4766413dda9e66c06d59b5027543f73aeff598d7 Mon Sep 17 00:00:00 2001 From: Chaofan Yu <103550325+icavan@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:18:39 +0800 Subject: [PATCH 4/9] fix(thd): preserve calibration topology and reject failed reports --- benchmarks/fit_thd_transport.py | 19 +++++++++- benchmarks/summarize_csa_layout_matrix.py | 6 ++- tests/test_csa_matrix_report.py | 23 ++++++++++++ tests/test_thd_cost_fit.py | 45 +++++++++++++++++++++++ tests/test_thd_redistribute.py | 4 +- 5 files changed, 93 insertions(+), 4 deletions(-) diff --git a/benchmarks/fit_thd_transport.py b/benchmarks/fit_thd_transport.py index 0afbeeff..082bbc16 100644 --- a/benchmarks/fit_thd_transport.py +++ b/benchmarks/fit_thd_transport.py @@ -28,6 +28,17 @@ def predict(features, coefficients, overlap_links=False): def fit(raw, profile, *, overlap_links=False): from scipy.optimize import differential_evolution, least_squares + world, per_node = raw["world"], raw["ranks_per_node"] + if ( + isinstance(world, bool) + or not isinstance(world, int) + or isinstance(per_node, bool) + or not isinstance(per_node, int) + or world < 2 + or per_node < 1 + or world % per_node + ): + raise ValueError("calibration topology requires CP >= 2 and complete nodes") rows = [r for r in raw["records"] if not r["heldout"]] if len(rows) < 6 or any(not r["marker_exact"] or r["critical_median_ms"] <= 0 for r in raw["records"]): raise ValueError("need complete, positive transport timings and exact row markers") @@ -53,7 +64,13 @@ def residual(x): x[4] = profile["model"].get("packing_ns_per_query", 0) * 65536 / 1e6 # A single-node calibration cannot identify inter-node bandwidth. # Use the same unused fallback rate, not an arbitrary optimizer parameter. - model = dict(profile["model"], overlap_intra_inter=overlap_links) + # The sweep assigns contiguous group ranks to nodes. Export that same + # topology rather than retaining the input profile's deployment mapping. + model = dict( + profile["model"], + overlap_intra_inter=overlap_links, + rank_node_ids=[rank // per_node for rank in range(world)], + ) factor = 65536 * model["bytes_per_query"] / 1e6 model.update( intra_node_gbps=factor / max(x[0], 1e-10), diff --git a/benchmarks/summarize_csa_layout_matrix.py b/benchmarks/summarize_csa_layout_matrix.py index 675ad5d1..7d55f168 100644 --- a/benchmarks/summarize_csa_layout_matrix.py +++ b/benchmarks/summarize_csa_layout_matrix.py @@ -19,7 +19,11 @@ def summarize(directory): ranks = raw["ranks"] if sorted(r["rank"] for r in ranks) != list(range(raw["world_size"])): raise ValueError(f"incomplete rank results: {path}") - if any(not all(r[k] for k in ("finite", "input_match", "redistribution_bitwise")) for r in ranks): + if any( + not all(r[k] for k in ("finite", "input_match", "redistribution_bitwise")) + or (r.get("check") is not None and not r["check"]["pass_"]) + for r in ranks + ): raise ValueError(f"failed correctness check: {path}") repeats = raw["config"]["repeats"] if not repeats or any(len(r["samples"]) != repeats for r in ranks): diff --git a/tests/test_csa_matrix_report.py b/tests/test_csa_matrix_report.py index 98604098..a4b853d4 100644 --- a/tests/test_csa_matrix_report.py +++ b/tests/test_csa_matrix_report.py @@ -105,3 +105,26 @@ def test_report_rejects_incomplete_precision_evidence(tmp_path, broken): path.write_text(json.dumps(data)) with pytest.raises(ValueError): summarize(tmp_path) + + +@pytest.mark.parametrize("layout", ["contiguous", "zigzag", "planner"]) +@pytest.mark.parametrize("rank", [0, 1]) +def test_report_rejects_failed_fp32_check(tmp_path, layout, rank): + _matrix(tmp_path) + path = tmp_path / f"layout-1m-fixture-{layout}.json" + data = json.loads(path.read_text()) + data["ranks"][rank]["check"] = {"pass_": False} + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="failed correctness check"): + summarize(tmp_path) + + +@pytest.mark.parametrize("check", [None, {"pass_": True}]) +def test_report_accepts_optional_or_successful_fp32_check(tmp_path, check): + _matrix(tmp_path) + for path in tmp_path.glob("*.json"): + data = json.loads(path.read_text()) + for rank in data["ranks"]: + rank["check"] = check + path.write_text(json.dumps(data)) + assert "1m-fixture" in summarize(tmp_path) diff --git a/tests/test_thd_cost_fit.py b/tests/test_thd_cost_fit.py index 74dc833c..0f625b57 100644 --- a/tests/test_thd_cost_fit.py +++ b/tests/test_thd_cost_fit.py @@ -134,3 +134,48 @@ def test_transport_fit_prediction_uses_the_same_cost_model_as_planning(world, pe overlap_intra_inter=overlap, ) assert predict(f, coeff, overlap) == pytest.approx(m.estimate_ms(layouts, block_size=4)) + + +@pytest.fixture +def transport_fit_inputs(monkeypatch): + from types import SimpleNamespace + + import numpy as np + + optimize = pytest.importorskip("scipy.optimize") + # Keep export/metadata regressions independent of optimizer convergence. + coefficients = np.array([1.0, 3.0, 4.0, 2.0, 0.5, 0.007, 0.01]) + monkeypatch.setattr(optimize, "differential_evolution", lambda *a, **kw: SimpleNamespace(x=coefficients.copy())) + monkeypatch.setattr(optimize, "least_squares", lambda *a, **kw: SimpleNamespace(x=coefficients.copy())) + profile = dict(model=dict(bytes_per_query=20608, rank_node_ids=[0] * 8, packing_ns_per_query=8.5)) + feature = dict(rank_intra_rows=[128] * 16, rank_inter_rows=[0] * 16, peer_rows=128, node_rows=0, copies=0, peer_degree=1) + record = dict(heldout=False, marker_exact=True, critical_median_ms=1, features=feature, kind="intra", rows=128) + return dict(world=8, ranks_per_node=8, records=[record.copy() for _ in range(6)]), profile + + +@pytest.mark.parametrize("world,per_node", [(8, 4), (16, 8), (8, 8), (4, 1)]) +@pytest.mark.parametrize("old_per_node", [4, 8]) +def test_transport_fit_exports_calibration_topology(transport_fit_inputs, world, per_node, old_per_node): + from copy import deepcopy + + from benchmarks.fit_thd_transport import fit + + raw, profile = transport_fit_inputs + profile["model"]["rank_node_ids"] = [r // old_per_node for r in range(8)] + original = deepcopy(profile) + raw.update(world=world, ranks_per_node=per_node) + result = fit(raw, profile) + assert result["model"]["rank_node_ids"] == [r // per_node for r in range(world)] + assert result["transport_calibration"]["world_size"] == world + assert result["transport_calibration"]["ranks_per_node"] == per_node + assert profile == original + + +@pytest.mark.parametrize("world,per_node", [(0, 4), (1, 1), (8, 0), (8, 3), (8, 16), (8.0, 4), (8, 4.0), (True, 1), (8, True)]) +def test_transport_fit_rejects_invalid_topology(transport_fit_inputs, world, per_node): + from benchmarks.fit_thd_transport import fit + + raw, profile = transport_fit_inputs + raw.update(world=world, ranks_per_node=per_node) + with pytest.raises(ValueError, match="topology"): + fit(raw, profile) diff --git a/tests/test_thd_redistribute.py b/tests/test_thd_redistribute.py index c4935e72..bbb3a593 100644 --- a/tests/test_thd_redistribute.py +++ b/tests/test_thd_redistribute.py @@ -84,8 +84,8 @@ def collective(out, packed, output_split_sizes, input_split_sizes, group): def test_reject_overlap(): layouts, _ = query_layouts((0, 64), 4, "contiguous") - broken = (layouts[0], replace(layouts[1], spans=layouts[0].spans), *layouts[2:]) - with pytest.raises(ValueError): + broken = (layouts[0], replace(layouts[1], spans=tuple(replace(span, owner=1) for span in layouts[0].spans)), *layouts[2:]) + with pytest.raises(ValueError, match="without gaps or overlap"): THDRedistributionPlan.build(broken, (0, 16, 32, 48, 64), 0) From 6e4c37df0521512b22553e81d765a05f5cfe2dd6 Mon Sep 17 00:00:00 2001 From: Chaofan Yu <103550325+icavan@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:25:23 +0800 Subject: [PATCH 5/9] fix(thd): harden collective validation and benchmark evidence Reject invalid source alignment group-wide and preserve Hopper preflight on independent multi-rank plans. Keep numerical failure reports strict JSON, reject nonfinite calibration timings, and counterbalance KDA benchmark pairs with recorded execution order. Add CPU regressions and distinguish historical GPU evidence from the new timing protocol. Validation: 375 focused CPU tests, repository-wide Ruff lint/format, and git diff --check. No fresh CUDA/NCCL or performance rerun. --- benchmarks/bench_full_csa_cp.py | 26 ++++-- benchmarks/bench_kda_independent_shards.py | 94 ++++++++++++++-------- benchmarks/fit_thd_transport.py | 7 +- cula/distributed/kda.py | 2 + docs/THD_SELECTION_RESULTS.md | 29 +++++-- tests/test_full_csa_cp_metadata.py | 55 ++++++++++++- tests/test_kda_independent_benchmark.py | 41 ++++++++++ tests/test_kimi_cp_helpers.py | 41 +++++++++- tests/test_thd_cost_fit.py | 18 +++++ 9 files changed, 265 insertions(+), 48 deletions(-) create mode 100644 tests/test_kda_independent_benchmark.py diff --git a/benchmarks/bench_full_csa_cp.py b/benchmarks/bench_full_csa_cp.py index 885bc3b7..9ebb1489 100644 --- a/benchmarks/bench_full_csa_cp.py +++ b/benchmarks/bench_full_csa_cp.py @@ -40,6 +40,23 @@ from cula.hybrid.index_pool import _load_cudnn_index_pool_ops +def validate_source_alignment(lengths, world): + """Make the same alignment decision on every balanced source rank.""" + if sum(lengths) % (4 * world) or any(n % 4 for n in lengths): + raise ValueError("source CP shards and documents must be block aligned") + + +def json_safe(value): + """Replace nonfinite measurements only in the serialized report copy.""" + if isinstance(value, dict): + return {key: json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(item) for item in value] + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + def metrics(actual, expected): actual, expected = actual.float(), expected.float() delta = actual - expected @@ -121,12 +138,11 @@ def __init__( torch.cuda.synchronize(device) common_prepare_start = time.perf_counter() self.rank, self.world = dist.get_rank(), dist.get_world_size() + validate_source_alignment(lengths, self.world) self.cu = (0, *accumulate(lengths)) self.plan = build_kimi_cp_plan(self.cu, group=dist.group.WORLD) self.source_n, self.total = self.plan.local_tokens, self.cu[-1] self.scale, self.heads = 576**-0.5, heads - if self.source_n % 4 or any(n % 4 for n in lengths): - raise ValueError("source CP shards and documents must be block aligned") n = self.source_n pool_plan = IndexPoolPlan.from_cu_seqlens(self.plan.local_cu_seqlens, head_dim=128, ratio=4) pool_cu_tokens, pool_cu_blocks = pool_plan.materialize(device) @@ -700,7 +716,7 @@ def main(*, keep_group=False, baseline=None, save_baseline=False, comparison_nam if iteration >= args.warmups: samples.append(sample) if rank == 0: - print("ITER", iteration, json.dumps(sample), flush=True) + print("ITER", iteration, json.dumps(json_safe(sample), allow_nan=False), flush=True) if args.check and check is None: check = check_reference(csa) finite = all( @@ -797,8 +813,8 @@ def main(*, keep_group=False, baseline=None, save_baseline=False, comparison_nam report = dict(config=vars(args), world_size=world, critical_rank_median=critical, ranks=gathered) report["planner_cost_model"] = None if cost_model is None else asdict(cost_model) with open(args.output, "w") as f: - json.dump(report, f, indent=2) - print("RESULT", json.dumps(critical), flush=True) + json.dump(json_safe(report), f, indent=2, allow_nan=False) + print("RESULT", json.dumps(json_safe(critical), allow_nan=False), flush=True) okay = all( r["finite"] and r["input_match"] diff --git a/benchmarks/bench_kda_independent_shards.py b/benchmarks/bench_kda_independent_shards.py index cf9c9bb9..02768e3b 100644 --- a/benchmarks/bench_kda_independent_shards.py +++ b/benchmarks/bench_kda_independent_shards.py @@ -20,10 +20,25 @@ import torch import torch.distributed as dist -from benchmarks.bench_full_csa_cp import compare_cpu_reference +from benchmarks.bench_full_csa_cp import compare_cpu_reference, json_safe from cula.distributed import AllGatherKimiCPTransport, KimiCPPlan, cudnn_kda_cp +def paired_samples(run, *, warmups, repeats): + """Warm both routes, then counterbalance each measured pair on all ranks.""" + if warmups < 0 or repeats < 1: + raise ValueError("warmups must be nonnegative and repeats positive") + samples = {"baseline": [], "candidate": []} + for warming, count in ((True, warmups), (False, repeats)): + for pair_index in range(count): + order = ("baseline", "candidate") if pair_index % 2 == 0 else ("candidate", "baseline") + for position, label in enumerate(order): + sample = run(label) + if not warming: + samples[label].append(dict(sample, pair_index=pair_index, execution_order=list(order), position=position)) + return samples + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, required=True) @@ -63,33 +78,40 @@ def main(): beta = torch.sigmoid(torch.randn(n, 32, device=device)) inputs = [x.requires_grad_() for x in (q, k, v, g, beta)] dout = torch.randn_like(v) + plans = {"baseline": replace(plan, requires_boundary_exchange=True), "candidate": plan} + + def timed_iteration(label, inputs=inputs, cu=cu, transport=transport, dout=dout): + for x in inputs: + x.grad = None + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, forward, end = [torch.cuda.Event(enable_timing=True) for _ in range(3)] + begin.record() + out = cudnn_kda_cp(*inputs, cu, plan=plans[label], group=dist.group.WORLD, boundary_transport=transport) + forward.record() + out.backward(dout) + end.record() + end.synchronize() + sample = dict( + total_ms=begin.elapsed_time(end), + fwd_ms=begin.elapsed_time(forward), + bwd_ms=forward.elapsed_time(end), + allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + ) + del out + return sample + + samples = paired_samples(timed_iteration, warmups=args.warmups, repeats=args.repeats) + del timed_iteration + # Run the numerical comparison separately, after both routes' timings. + # CPU copies and reference comparisons cannot perturb a measured pair. reference = None - for label, active_plan in (("baseline", replace(plan, requires_boundary_exchange=True)), ("candidate", plan)): - samples = [] - for iteration in range(args.warmups + args.repeats): - for x in inputs: - x.grad = None - dist.barrier(device_ids=[local_rank]) - torch.cuda.synchronize() - torch.cuda.reset_peak_memory_stats() - begin, forward, end = [torch.cuda.Event(enable_timing=True) for _ in range(3)] - begin.record() - out = cudnn_kda_cp(*inputs, cu, plan=active_plan, group=dist.group.WORLD, boundary_transport=transport) - forward.record() - out.backward(dout) - end.record() - end.synchronize() - if iteration >= args.warmups: - samples.append( - dict( - total_ms=begin.elapsed_time(end), - fwd_ms=begin.elapsed_time(forward), - bwd_ms=forward.elapsed_time(end), - allocated_gib=torch.cuda.max_memory_allocated() / 2**30, - ) - ) - if iteration + 1 < args.warmups + args.repeats: - del out + for label, active_plan in plans.items(): + for x in inputs: + x.grad = None + out = cudnn_kda_cp(*inputs, cu, plan=active_plan, group=dist.group.WORLD, boundary_transport=transport) + out.backward(dout) tensors = dict(output=out, **{key: x.grad for key, x in zip(("dq", "dk", "dv", "dg", "dbeta"), inputs)}) errors = {} if reference is None else {key: compare_cpu_reference(x, reference[key]) for key, x in tensors.items()} if reference is None: @@ -100,7 +122,7 @@ def main(): rank=rank, requires_boundary_exchange=active_plan.requires_boundary_exchange, intrinsic_requires_boundary_exchange=plan.requires_boundary_exchange, - samples=samples, + samples=samples[label], errors=errors, ) gathered = [None] * world @@ -122,14 +144,18 @@ def main(): args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps( - dict( - world_size=world, - source_sha256=hashlib.sha256( - (Path(__file__).resolve().parents[1] / "cula/distributed/kda.py").read_bytes() - ).hexdigest(), - results=results, + json_safe( + dict( + world_size=world, + timing_protocol="paired-alternating-v1", + source_sha256=hashlib.sha256( + (Path(__file__).resolve().parents[1] / "cula/distributed/kda.py").read_bytes() + ).hexdigest(), + results=results, + ) ), indent=2, + allow_nan=False, ) + "\n" ) diff --git a/benchmarks/fit_thd_transport.py b/benchmarks/fit_thd_transport.py index 082bbc16..ce2f9ea2 100644 --- a/benchmarks/fit_thd_transport.py +++ b/benchmarks/fit_thd_transport.py @@ -40,8 +40,11 @@ def fit(raw, profile, *, overlap_links=False): ): raise ValueError("calibration topology requires CP >= 2 and complete nodes") rows = [r for r in raw["records"] if not r["heldout"]] - if len(rows) < 6 or any(not r["marker_exact"] or r["critical_median_ms"] <= 0 for r in raw["records"]): - raise ValueError("need complete, positive transport timings and exact row markers") + if len(rows) < 6 or any( + not r["marker_exact"] or not np.isfinite(r["critical_median_ms"]) or r["critical_median_ms"] <= 0 + for r in raw["records"] + ): + raise ValueError("need complete, finite positive transport timings and exact row markers") y = np.array([r["critical_median_ms"] for r in rows]) def residual(x): diff --git a/cula/distributed/kda.py b/cula/distributed/kda.py index 603cd113..3abbf43b 100644 --- a/cula/distributed/kda.py +++ b/cula/distributed/kda.py @@ -933,6 +933,8 @@ def cudnn_kda_cp( # independent native recurrence as CP1, including native autograd; # no forward or backward state can cross ranks. The plan flag is # identical on all peers, so no rank skips a required collective. + if plan.world_size > 1 and _resolve_kimi_cp_backend(backend, q.device) == "hopper-state-probe": + _validate_hopper_state_probe_options(options, a_log, dt_bias) kimi_delta_attention, _ = _load_cudnn_kda_ops() attention_outputs = kimi_delta_attention( q, diff --git a/docs/THD_SELECTION_RESULTS.md b/docs/THD_SELECTION_RESULTS.md index 7096e18e..276be400 100644 --- a/docs/THD_SELECTION_RESULTS.md +++ b/docs/THD_SELECTION_RESULTS.md @@ -117,13 +117,13 @@ and shared rendezvous; select `thd_selection_h20_cp16.json`. The runner first executes small FP32 checks, then all 31 cases, keeping contiguous first and alternating the other two policies. `--list-cases` lists shapes without CUDA. -## Final source review +## Original submission source review (2026-11-15) The full matrix above precedes the final CPU metadata cleanup. This cleanup rejects noninteger transport intervals, removes an obsolete benchmark import, fixes preparation accounting in the public summarizer and replaces a quadratic DSA window candidate sum with an integer running total. Planner decisions and -numerical kernels are unchanged. The final branch has separate regression +numerical kernels are unchanged. That submission has separate regression validation recorded below; the historical matrix is not relabeled as a new run. Local CPU preparation microbenchmark (10 measured samples after one warmup): @@ -138,9 +138,9 @@ Local CPU preparation microbenchmark (10 measured samples after one warmup): These CPU-only measurements are not GPU or whole-chain speedups. The change removes repeated scans of already-grouped documents; it does not change scratch capacity, math precision or kernel launch boundaries. -## Final-branch GPU regression +## Original submission GPU regression -On both H20 CP16 and B300 CP8, the final source passed four small FP32 fixtures +On both H20 CP16 and B300 CP8, the original submission source passed four small FP32 fixtures and three representative 1M shapes (single long, 1000 documents with one long, and 1000-document tiny mixture), each under C/Z/P: **42 complete CSA rows**. The 1M rerun used three warmups and five measurements. A separate six-case KDA @@ -148,7 +148,7 @@ A/B on each platform compared native document-local execution with the forced boundary route, including weak decay and isolated ranks with a crossing peer: **24 A/B rows**, with zero difference in output and all five gradients. -All 181 tested source/config files match the submitted branch. The 62 original +All 181 tested source/config files matched the original submission. The 62 original planner decisions still match every measured rank's policy, row count, span count and remote rows. All **2,232** DSA rank plans across the full 31 x 3 x (CP16 + CP8) matrix have field-for-field identical window metadata before and @@ -161,3 +161,22 @@ coverage and ownership, collective participation, inverse autograd, storage lifetimes, context reuse, cost-model bounds and benchmark accounting. No handwritten GPU kernel, precision mode or math approximation is changed. This is not a claim of a complete race audit of every unchanged kernel in cuLA. + + +## Post-review validation and timing protocol + +The subsequent review fixes add group-wide source-alignment rejection, finite +calibration-input validation, strict JSON failure reports, and the SM90 +independent-shard option preflight. The independent-shard KDA benchmark now +warms both routes and alternates baseline/candidate order for every measured +pair. Each raw sample records its pair index, actual execution order and +position; new reports identify this as `paired-alternating-v1`. Numerical +output/gradient comparison runs separately after both routes' timed samples. + +The historical JSON result files and tables above are unchanged. In particular, +the original KDA A/B timings ran all baseline samples before all candidate +samples; they are not measurements under the new paired protocol. The recorded +source hashes describe those historical runs, not the later review fixes. +No fresh GPU, NCCL or performance rerun accompanies these fixes. Their +validation consists of focused CPU regressions and repository-wide Ruff +lint/format checks; a new paired GPU run is needed for updated timing claims. diff --git a/tests/test_full_csa_cp_metadata.py b/tests/test_full_csa_cp_metadata.py index 26415442..3f18e02d 100644 --- a/tests/test_full_csa_cp_metadata.py +++ b/tests/test_full_csa_cp_metadata.py @@ -3,10 +3,16 @@ """CPU checks of the experimental full-CSA benchmark's addressing contract.""" +import json +import math +import sys +from types import SimpleNamespace + import pytest import torch -from benchmarks.bench_full_csa_cp import expand_blocks, query_metadata +from benchmarks import bench_full_csa_cp as bench +from benchmarks.bench_full_csa_cp import expand_blocks, json_safe, metrics, query_metadata, validate_source_alignment from cula.distributed import THDRankLayout, THDSpan @@ -57,3 +63,50 @@ def test_bulk_metadata_is_prepared_on_cpu_under_a_different_default_device(): assert origin.tolist() == [0] * 4 assert visible.tolist() == [5, 6, 7, 8] assert rows.tolist() == [4, 5, 6, 7] + + +@pytest.mark.parametrize("lengths,world", [([32], 8), ([4, 28], 8), ([8, 8], 2), ([4], 1)]) +def test_source_alignment_accepts_group_wide_aligned_shards(lengths, world): + validate_source_alignment(lengths, world) + + +@pytest.mark.parametrize("lengths,world", [([36], 8), ([12, 24], 8), ([6, 26], 8), ([6], 1)]) +def test_source_alignment_rejects_misaligned_shards_or_documents(lengths, world): + with pytest.raises(ValueError, match="source CP shards and documents must be block aligned"): + validate_source_alignment(lengths, world) + + +@pytest.mark.parametrize("rank", range(8)) +def test_invalid_alignment_stops_every_rank_before_plan_or_collective(monkeypatch, rank): + monkeypatch.setitem(sys.modules, "cudnn", SimpleNamespace(DSA=object())) + monkeypatch.setattr(bench, "_load_cudnn_index_pool_ops", lambda: (None, None)) + monkeypatch.setattr(bench.torch.cuda, "synchronize", lambda device: None) + monkeypatch.setattr(bench.dist, "get_rank", lambda: rank) + monkeypatch.setattr(bench.dist, "get_world_size", lambda: 8) + + def unexpected(*args, **kwargs): + pytest.fail("invalid global alignment reached plan construction or a collective") + + monkeypatch.setattr(bench, "build_kimi_cp_plan", unexpected) + monkeypatch.setattr(bench.dist, "broadcast", unexpected) + with pytest.raises(ValueError, match="block aligned"): + bench.FullCSA([36], heads=64, top_k=128, device="cpu", candidate_mib=0) + + +def test_failure_report_is_strict_json_without_changing_failure_evidence(): + errors = metrics(torch.tensor([float("nan")]), torch.ones(1)) + raw = { + "finite": False, + "check": {"pass_": False, "errors": errors}, + "samples": [{"time": float("inf")}], + "extra": (float("-inf"), 1.5, True, None), + } + encoded = json.dumps(json_safe(raw), allow_nan=False) + decoded = json.loads(encoded) + assert decoded["finite"] is False + assert decoded["check"]["pass_"] is False + assert decoded["check"]["errors"] == {"max_abs": None, "relative_rms": None, "finite": False} + assert decoded["samples"] == [{"time": None}] + assert decoded["extra"] == [None, 1.5, True, None] + assert math.isnan(raw["check"]["errors"]["max_abs"]) + assert math.isinf(raw["samples"][0]["time"]) diff --git a/tests/test_kda_independent_benchmark.py b/tests/test_kda_independent_benchmark.py new file mode 100644 index 00000000..10838dad --- /dev/null +++ b/tests/test_kda_independent_benchmark.py @@ -0,0 +1,41 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from benchmarks.bench_kda_independent_shards import paired_samples + + +@pytest.mark.parametrize("warmups,repeats", [(0, 1), (1, 4), (3, 3)]) +def test_paired_samples_warm_both_routes_and_record_actual_order(warmups, repeats): + calls = [] + + def run(label): + calls.append(label) + return {"total_ms": len(calls), "fwd_ms": 1.0} + + samples = paired_samples(run, warmups=warmups, repeats=repeats) + assert len(calls) == 2 * (warmups + repeats) + assert calls[: 2 * warmups].count("baseline") == warmups + assert calls[: 2 * warmups].count("candidate") == warmups + for pair in range(repeats): + expected = ["baseline", "candidate"] if pair % 2 == 0 else ["candidate", "baseline"] + offset = 2 * (warmups + pair) + assert calls[offset : offset + 2] == expected + for position, label in enumerate(expected): + sample = samples[label][pair] + assert sample["pair_index"] == pair + assert sample["execution_order"] == expected + assert sample["position"] == position + assert sample["total_ms"] == offset + position + 1 + assert sample["fwd_ms"] == 1.0 + assert all(len(rows) == repeats for rows in samples.values()) + + +@pytest.mark.parametrize("warmups,repeats", [(-1, 1), (0, 0)]) +def test_paired_samples_reject_invalid_counts_before_running(warmups, repeats): + def unexpected(label): + pytest.fail("invalid counts executed a benchmark route") + + with pytest.raises(ValueError): + paired_samples(unexpected, warmups=warmups, repeats=repeats) diff --git a/tests/test_kimi_cp_helpers.py b/tests/test_kimi_cp_helpers.py index c9e437bd..2947b9b5 100644 --- a/tests/test_kimi_cp_helpers.py +++ b/tests/test_kimi_cp_helpers.py @@ -603,7 +603,8 @@ def backward(d_o, q, k, v, g, beta, cu, scale, **kwargs): @pytest.mark.parametrize("world", [1, 4]) -def test_independent_shards_call_native_attention_export(monkeypatch, world): +@pytest.mark.parametrize("major", [9, 10]) +def test_independent_shards_call_native_attention_export(monkeypatch, world, major): plan = KimiCPPlan.from_global_cu_seqlens(list(range(0, 2 * world + 1, 2)), rank=0, world_size=world) q = torch.zeros(2, 1, 4) v = torch.zeros(2, 1, 3) @@ -616,6 +617,7 @@ def attention(*args, **kwargs): observed["kwargs"] = kwargs return torch.ones_like(v), torch.empty(0) + monkeypatch.setattr(cp.torch.cuda, "get_device_capability", lambda device: (major, 0)) monkeypatch.setattr(cp, "_validate_inputs", lambda *args, **kwargs: None) monkeypatch.setattr(cp, "_load_cudnn_kda_ops", lambda: (attention, object())) @@ -623,3 +625,40 @@ def attention(*args, **kwargs): assert torch.equal(result, torch.ones_like(v)) assert all(actual is expected for actual, expected in zip(observed["args"][:5], (q, q, v, q, beta), strict=True)) + + +@pytest.mark.parametrize("option", ["use_qk_l2norm_in_kernel", "use_beta_sigmoid_in_kernel", "safe_gate"]) +@pytest.mark.parametrize( + "world,major,backend,reject", + [ + (4, 9, "auto", True), + (4, 10, "auto", False), + (1, 9, "auto", False), + (4, 10, "hopper-state-probe", True), + (4, 9, "frost-summary", False), + ], +) +def test_independent_shards_validate_only_multirank_hopper_options(monkeypatch, option, world, major, backend, reject): + plan = KimiCPPlan.from_global_cu_seqlens(list(range(0, 2 * world + 1, 2)), rank=0, world_size=world) + q = torch.zeros(2, 1, 4) + beta = torch.zeros(2, 1) + cu = torch.tensor([0, 2], dtype=torch.int32) + observed = {} + + def attention(*args, **kwargs): + observed.update(kwargs) + return q, None + + monkeypatch.setattr(cp, "_validate_inputs", lambda *args, **kwargs: None) + monkeypatch.setattr(cp.torch.cuda, "get_device_capability", lambda device: (major, 0)) + monkeypatch.setattr(cp, "_load_cudnn_kda_ops", lambda: (attention, object())) + options = {option: True} + if option == "safe_gate": + options.update(a_log=torch.zeros(1), dt_bias=torch.zeros(1, 4)) + if reject: + with pytest.raises(ValueError, match=option): + cp.cudnn_kda_cp(q, q, q, q, beta, cu, plan=plan, backend=backend, **options) + assert not observed + else: + assert cp.cudnn_kda_cp(q, q, q, q, beta, cu, plan=plan, backend=backend, **options) is q + assert observed[option] is True diff --git a/tests/test_thd_cost_fit.py b/tests/test_thd_cost_fit.py index 0f625b57..10573e44 100644 --- a/tests/test_thd_cost_fit.py +++ b/tests/test_thd_cost_fit.py @@ -179,3 +179,21 @@ def test_transport_fit_rejects_invalid_topology(transport_fit_inputs, world, per raw.update(world=world, ranks_per_node=per_node) with pytest.raises(ValueError, match="topology"): fit(raw, profile) + + +@pytest.mark.parametrize("heldout", [False, True]) +@pytest.mark.parametrize("timing", [float("nan"), float("inf"), float("-inf"), 0, -1]) +def test_transport_fit_rejects_invalid_timings_before_optimization(transport_fit_inputs, monkeypatch, heldout, timing): + import scipy.optimize + + from benchmarks.fit_thd_transport import fit + + raw, profile = transport_fit_inputs + raw["records"].append(dict(raw["records"][0], heldout=heldout, critical_median_ms=timing)) + + def unexpected(*args, **kwargs): + pytest.fail("invalid timing reached the optimizer") + + monkeypatch.setattr(scipy.optimize, "differential_evolution", unexpected) + with pytest.raises(ValueError, match="finite positive"): + fit(raw, profile) From d2cf4fdf7b7ec8fc4e22ecd780dd4a48d0a61670 Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Mon, 16 Nov 2026 20:25:55 +0800 Subject: [PATCH 6/9] feat(hybrid): expose prepared CSA context-parallel selection Package host planning, CUDA workspaces, source-order restoration and retained results for downstream indexers. Add an explicit selection-fragment floor and reduce repeated identity and placement work without changing default plans. Build bulk integer mappings on device from compact host ranges. Validate 128K/256K complete CSA forward/backward on CP8, including subgroup and FP32-weight contracts. Report steady execution and fresh preparation separately, retaining smaller-context regressions and limits. Signed-off-by: shuyan.ycf --- README.md | 2 + benchmarks/bench_csa_cp_windows.py | 268 + benchmarks/bench_full_csa_cp.py | 133 +- benchmarks/check_csa_cp_selection.py | 98 + cula/_metadata.py | 22 +- cula/distributed/redistribute.py | 13 +- cula/distributed/thd_planner.py | 113 +- cula/hybrid/__init__.py | 5 + cula/hybrid/csa_cp.py | 283 ++ docs/CSA_CP_SELECTION.md | 144 + docs/CSA_CP_WINDOW_RESULTS.md | 139 + docs/HYBRID_CUDNN_RUNTIME.md | 3 + docs/results/csa_cp_windows_20261116.json | 5433 +++++++++++++++++++++ tests/test_csa_cp_selection.py | 238 + 14 files changed, 6842 insertions(+), 52 deletions(-) create mode 100644 benchmarks/bench_csa_cp_windows.py create mode 100644 benchmarks/check_csa_cp_selection.py create mode 100644 cula/hybrid/csa_cp.py create mode 100644 docs/CSA_CP_SELECTION.md create mode 100644 docs/CSA_CP_WINDOW_RESULTS.md create mode 100644 docs/results/csa_cp_windows_20261116.json create mode 100644 tests/test_csa_cp_selection.py diff --git a/README.md b/README.md index e1d4c545..2206afee 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,8 @@ kernel boundary and the overlap assessment against FlashInfer and cuDNN. The optional [cuDNN hybrid-attention runtime](docs/HYBRID_CUDNN_RUNTIME.md) documents packed index pooling, memory-bounded DSA selection, and CP layout ownership without precision downgrades. +The [CSA CP selection API](docs/CSA_CP_SELECTION.md) packages adaptive THD +planning, query routing and result restoration for downstream frameworks. ## Benchmarks diff --git a/benchmarks/bench_csa_cp_windows.py b/benchmarks/bench_csa_cp_windows.py new file mode 100644 index 00000000..4342338e --- /dev/null +++ b/benchmarks/bench_csa_cp_windows.py @@ -0,0 +1,268 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Public CSA selection API: complete CSA F/B across packed context lengths. + +No projections, KDA, MoE, optimizer, or checkpoint recomputation are included. +All variants use the same BF16 inputs and FP32 communication/reduction contract. +""" + +import argparse +import datetime +import gc +import hashlib +import json +import math +import os +import statistics +import time +from dataclasses import replace +from pathlib import Path + +import torch +import torch.distributed as dist + +from benchmarks.bench_full_csa_cp import FullCSA, check_redistribution, check_reference, compare_cpu_reference, json_safe +from cula.distributed import THDPlannerConfig, THDSelectionCostModel + + +def cases(tokens): + """Five block-aligned layouts, including 512/1000-document packs.""" + short = 64 + mixed = [short] * 998 + [(tokens - 998 * short) // 2] * 2 + # The two long documents have different positions in a reproducible pack. + mixed = mixed[:199] + [mixed[-1]] + mixed[199:-2] + [mixed[-2]] + tail = [64] * 800 + [256] * 199 + [tokens - 800 * 64 - 199 * 256] + return { + "single": [tokens], + "uniform-1024": [tokens // 1024] * 1024, + "near-uniform-512": [tokens // 512 - 4, tokens // 512 + 4] * 256, + "mixed-1000": mixed, + "long-tail-1000": tail, + } + + +def tensors(csa): + r, b = csa.last, csa.indexer + return dict( + output=r["source_output"], + dq=r["source_dq"], + dkv=r["local_dkv"], + d_index_q=r["source_dqi"], + d_weights=r["source_dw"], + d_pool_kv=b.local_kv.grad, + d_pool_score=b.pool_score.grad, + d_ape=b.ape.grad, + ) + + +def iteration(csa): + dist.barrier(device_ids=[torch.cuda.current_device()]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, end = (torch.cuda.Event(enable_timing=True) for _ in range(2)) + wall = time.perf_counter() + begin.record() + csa.forward() + csa.backward() + end.record() + end.synchronize() + return dict( + total_ms=begin.elapsed_time(end), + wall_ms=1000 * (time.perf_counter() - wall), + allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + reserved_gib=torch.cuda.max_memory_reserved() / 2**30, + **csa.timings(), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--tokens", type=int, nargs="+", default=[128 << 10, 256 << 10]) + parser.add_argument("--cases", nargs="+", help="Subset of the five named distributions") + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--prepare-repeats", type=int, default=8) + parser.add_argument("--min-gain-ms", type=float, help="Explicit preparation budget override, recorded in the report") + parser.add_argument( + "--min-chunk-tokens", type=int, default=1024, help="Minimum newly created planner span; independent of context length" + ) + parser.add_argument("--max-chunks-per-rank", type=int, default=6) + parser.add_argument("--check-only", action="store_true", help="Small independent FP32 oracle fixtures") + args = parser.parse_args() + if min(args.warmups, args.repeats, args.prepare_repeats) < 1 or any(t < 128 << 10 or t % 4096 for t in args.tokens): + parser.error("warmups/repeats must be positive; contexts must be >= 128K and divisible by 4096") + torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0))) + device = torch.device("cuda", torch.cuda.current_device()) + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=10)) + rank, world = dist.get_rank(), dist.get_world_size() + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False + profiles = [json.loads(args.profile.read_text())["model"] if rank == 0 else None] + dist.broadcast_object_list(profiles, src=0) + profiles[0]["rank_node_ids"] = tuple(profiles[0]["rank_node_ids"]) + model = THDSelectionCostModel(**profiles[0]) + if args.min_gain_ms is not None: + model = replace(model, min_gain_ms=args.min_gain_ms) + if len(model.rank_node_ids) != world: + raise ValueError("profile topology does not match world size") + args.output_dir.mkdir(parents=True, exist_ok=True) + from dataclasses import asdict + + manifest = dict(arguments={k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()}, model=asdict(model)) + source = Path(__file__).resolve().parents[1] + import importlib.metadata + + try: + frontend_version = importlib.metadata.version("nvidia-cudnn-frontend") + except importlib.metadata.PackageNotFoundError: + frontend_version = None # Source-only frontend checkouts have no wheel metadata. + manifest["software"] = { + "torch": torch.__version__, + "cuda_runtime": torch.version.cuda, + "cutlass_dsl": importlib.metadata.version("nvidia-cutlass-dsl"), + "cudnn_frontend": frontend_version, + } + manifest["source_sha256"] = { + str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() + for directory in (source / "cula", source / "benchmarks") + for p in sorted(directory.rglob("*.py")) + } + manifests = [None] * world + dist.all_gather_object(manifests, manifest) + if any(m != manifests[0] for m in manifests): + raise RuntimeError("source/config differs across ranks") + if rank == 0: + (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + suite = [ + (f"{t // 1024}k-{name}", lengths) + for t in args.tokens + for name, lengths in cases(t).items() + if not args.cases or name in args.cases + ] + if args.check_only: + suite = [("check-single", [4096]), ("check-many", [4] * 1024), ("check-mixed", [4] * 64 + [1920] * 2)] + if not suite: + raise ValueError("no matching cases") + for name, lengths in suite: + variants, samples, checks, preparation = {}, {}, {}, {} + policies = ("legacy-contiguous", "contiguous", "zigzag", "planner") + for policy in policies: + torch.manual_seed(20261116 + rank) + csa = FullCSA( + lengths, + 64, + 512, + device, + 0, + layout="contiguous" if policy.startswith("legacy") else policy, + planner_cost_model=model, + planner_strategy="selection", + selection_api=not policy.startswith("legacy"), + planner_config=THDPlannerConfig( + min_chunk_tokens=args.min_chunk_tokens, max_chunks_per_rank=args.max_chunks_per_rank + ), + ) + variants[policy], samples[policy] = csa, [] + if not check_redistribution(csa): + raise RuntimeError("row identity/inverse failed") + if rank == 0: + print("CASE", name, "selected", variants["planner"].selected_policy, flush=True) + orders = [] + for i in range(args.warmups + args.repeats): + order = policies[i % len(policies) :] + policies[: i % len(policies)] + if i % 2: + order = order[::-1] + orders.append(order) + for policy in order: + sample = iteration(variants[policy]) + if i >= args.warmups: + samples[policy].append(sample) + if rank == 0: + print("ITER", name, i, {p: round(samples[p][-1]["total_ms"], 3) for p in policies if samples[p]}, flush=True) + # Fresh metadata is measured after kernels/allocators are warm, in + # rotated order. Retain cold measurements separately, without silently + # substituting cache lookup time for fresh plan construction. + cold_prepare = {p: dict(common_ms=c.common_prepare_ms, layout_ms=c.layout_prepare_ms) for p, c in variants.items()} + prepare_orders = [] + for j in range(args.prepare_repeats): + order = policies[j % len(policies) :] + policies[: j % len(policies)] + if j % 2: + order = order[::-1] + prepare_orders.append(order) + for p in order: + dist.barrier(device_ids=[torch.cuda.current_device()]) + preparation.setdefault(p, []).append(variants[p].prepare_metadata()) + reference = {k: v.detach().cpu() for k, v in tensors(variants["legacy-contiguous"]).items()} + # Loss is normalized by global tokens in FullCSA. Reduce its scalar + # separately; selection routing must not change the objective. + losses = torch.stack([variants[p].last["loss"].detach() for p in policies]) + dist.all_reduce(losses) + loss_values = losses.cpu().tolist() + okay = True + for policy, csa in variants.items(): + errors = {k: compare_cpu_reference(v, reference[k]) for k, v in tensors(csa).items()} + ref = check_reference(csa) if args.check_only else None + okay &= all(e["finite"] and e["relative_rms"] < 0.01 for e in errors.values()) + loss = loss_values[policies.index(policy)] + loss_delta = abs(loss - loss_values[0]) + okay &= ref is None or ref["pass_"] + okay &= math.isfinite(loss) and loss_delta < 1e-5 + checks[policy] = dict( + errors=errors, + global_loss=loss, + loss_abs_diff=loss_delta, + fp32=ref, + selected_policy=csa.selected_policy, + cold_prepare=cold_prepare[policy], + prepare_samples=preparation[policy], + source_rows=csa.source_n, + selection_rows=csa.selection_layout.local_tokens, + selection_spans=len(csa.selection_layout.spans), + workspace_bytes=0 + if csa.indexer.select_plan is None + else csa.indexer.select_plan.candidate_workspace_floats * 4, + ) + records = [None] * world + dist.all_gather_object(records, dict(rank=rank, samples=samples, checks=checks, okay=okay)) + if rank == 0: + summary = {} + for policy in policies: + step = statistics.median( + max(r["samples"][policy][i]["total_ms"] for r in records) for i in range(args.repeats) + ) + prepare_samples = [ + max(sum(r["checks"][policy]["prepare_samples"][j].values()) for r in records) + for j in range(args.prepare_repeats) + ] + prepare = statistics.median(prepare_samples) + summary[policy] = dict( + step_ms=step, + prepare_ms=prepare, + one_use_ms=step + prepare, + prepare_min_ms=min(prepare_samples), + prepare_max_ms=max(prepare_samples), + ) + report = dict( + case=name, + lengths=lengths, + world_size=world, + orders=orders, + prepare_orders=prepare_orders, + summary=summary, + ranks=records, + ) + (args.output_dir / f"{name}.json").write_text(json.dumps(json_safe(report), indent=2, allow_nan=False) + "\n") + print("RESULT", name, json.dumps(summary), flush=True) + if not all(r["okay"] for r in records): + raise RuntimeError("numerical qualification failed; result retained") + del variants, csa, reference, records + gc.collect() + torch.cuda.empty_cache() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_full_csa_cp.py b/benchmarks/bench_full_csa_cp.py index 9ebb1489..c4f9cf68 100644 --- a/benchmarks/bench_full_csa_cp.py +++ b/benchmarks/bench_full_csa_cp.py @@ -36,7 +36,16 @@ build_kimi_cp_plan, cudnn_kda_cp, ) -from cula.hybrid import DsaSelectionPlan, DsaSelectionWorkspace, IndexPoolPlan, cudnn_dsa_select, cudnn_index_pool +from cula.hybrid import ( + CsaCPSelectionPlan, + CsaCPSelectionWorkspace, + DsaSelectionPlan, + DsaSelectionWorkspace, + IndexPoolPlan, + cudnn_csa_select_cp, + cudnn_dsa_select, + cudnn_index_pool, +) from cula.hybrid.index_pool import _load_cudnn_index_pool_ops @@ -50,7 +59,7 @@ def json_safe(value): """Replace nonfinite measurements only in the serialized report copy.""" if isinstance(value, dict): return {key: json_safe(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): + if isinstance(value, list | tuple): return [json_safe(item) for item in value] if isinstance(value, float) and not math.isfinite(value): return None @@ -127,6 +136,8 @@ def __init__( redistribution_scope="selection", planner_cost_model=None, planner_strategy="balanced", + selection_api=False, + planner_config=None, ): import cudnn @@ -134,6 +145,11 @@ def __init__( self.top_k = top_k self.pool_backward_order = pool_backward_order self.redistribution_scope = redistribution_scope + self.selection_api = selection_api + if selection_api and ( + redistribution_scope != "selection" or (layout == "planner" and planner_strategy != "selection") + ): + raise ValueError("the public API requires selection-only routing and the selection planner") _, self.pool_backward = _load_cudnn_index_pool_ops() torch.cuda.synchronize(device) common_prepare_start = time.perf_counter() @@ -177,23 +193,86 @@ def __init__( self.global_kv = torch.empty(self.total, 576, device=device, dtype=torch.bfloat16) self.sink = torch.full((heads,), -torch.inf, device=device, dtype=torch.float32) self.unit_grad = torch.ones((), device=device, dtype=torch.float32) + self._layout_options = dict( + device=device, + candidate_mib=candidate_mib, + layout=layout, + redistribution_scope=redistribution_scope, + planner_cost_model=planner_cost_model, + planner_strategy=planner_strategy, + selection_api=selection_api, + planner_config=planner_config, + ) + self._prepare_layout(**self._layout_options) + self.events = [] + self.last = {} + + def prepare_metadata(self): + """Rebuild fresh metadata after warmup, without regenerating inputs. + + Synchronization includes H2D metadata copies and workspace allocation; + compilation and RNG initialization are outside this preparation metric. + """ + device = self.source_q.device + torch.cuda.synchronize(device) + start = time.perf_counter() + self.plan = build_kimi_cp_plan(self.cu, group=dist.group.WORLD) + b = self.indexer + b.pool_plan = IndexPoolPlan.from_cu_seqlens(self.plan.local_cu_seqlens, head_dim=128, ratio=4) + b.pool_cu_tokens, b.pool_cu_blocks = b.pool_plan.materialize(device) + torch.cuda.synchronize(device) + self.common_prepare_ms = (time.perf_counter() - start) * 1000 + self._prepare_layout(**self._layout_options) + return dict(common_ms=self.common_prepare_ms, layout_ms=self.layout_prepare_ms) + + def _prepare_layout( + self, + *, + device, + candidate_mib, + layout, + redistribution_scope, + planner_cost_model, + planner_strategy, + selection_api, + planner_config, + ): + b, top_k = self.indexer, self.top_k torch.cuda.synchronize(device) prepare_start = time.perf_counter() if planner_cost_model is not None and redistribution_scope != "selection": raise ValueError("the selection cost model is valid only for selection-only redistribution") if planner_strategy == "selection" and redistribution_scope != "selection": raise ValueError("selection planning requires selection-only redistribution") - ranks, self.selected_policy = query_layouts( - self.cu, self.world, layout, cost_model=planner_cost_model, planner_strategy=planner_strategy - ) + if selection_api: + cp_selection_plan = CsaCPSelectionPlan.from_cu_seqlens( + self.cu, + rank=self.rank, + world_size=self.world, + top_k=top_k, + policy=layout, + cost_model=planner_cost_model, + config=planner_config, + max_tokens_per_rank=self.total if layout == "zigzag" else None, + candidate_budget_bytes=None if candidate_mib == 0 else candidate_mib << 20, + ) + self.cp_selection = CsaCPSelectionWorkspace.from_plan(cp_selection_plan, device, group=dist.group.WORLD) + ranks, self.selected_policy = cp_selection_plan.ranks, cp_selection_plan.policy + else: + ranks, self.selected_policy = query_layouts( + self.cu, self.world, layout, cost_model=planner_cost_model, planner_strategy=planner_strategy + ) if redistribution_scope == "queries" and any(r.local_tokens == 0 for r in ranks): raise ValueError( "query-wide redistribution requires nonempty attention ranks; use selection scope for empty ranks" ) self.selection_layout = ranks[self.rank] source_offsets = balanced_shards(self.total, self.world) - exchange_plan = THDRedistributionPlan.build(ranks, source_offsets, self.rank) - self.selection_exchange = THDRedistributor(exchange_plan, device, group=dist.group.WORLD) + if selection_api: + self.selection_exchange = self.cp_selection.exchange + else: + exchange_plan = THDRedistributionPlan.build(ranks, source_offsets, self.rank) + self.selection_exchange = THDRedistributor(exchange_plan, device, group=dist.group.WORLD) if redistribution_scope == "selection": attention_ranks, _ = query_layouts(self.cu, self.world, "contiguous") self.layout = attention_ranks[self.rank] @@ -211,7 +290,9 @@ def __init__( key_ranges.append((span.sequence_start // 4, span.sequence_end // 4)) b.select_plan = None b.select_workspace = None - if self.selection_layout.local_tokens: + if selection_api: + b.select_plan, b.select_workspace = cp_selection_plan.selection, self.cp_selection.selection + elif self.selection_layout.local_tokens: b.select_plan = DsaSelectionPlan.from_cu_seqlens( self.selection_layout.local_cu_seqlens, key_prefix, @@ -222,7 +303,9 @@ def __init__( ) b.select_workspace = DsaSelectionWorkspace.from_plan(b.select_plan, device) self.selection_key_ids = None - if not key_ranges: + if selection_api: + self.selection_key_slice, self.selection_key_ids = self.cp_selection.key_slice, self.cp_selection.key_indices + elif not key_ranges: self.selection_key_slice = slice(0, 0) elif all(left[1] == right[0] for left, right in zip(key_ranges, key_ranges[1:])): self.selection_key_slice = slice(key_ranges[0][0], key_ranges[-1][1]) @@ -230,8 +313,6 @@ def __init__( self.selection_key_ids = torch.cat([torch.arange(a, z, device="cpu") for a, z in key_ranges]).to(device) torch.cuda.synchronize(device) self.layout_prepare_ms = (time.perf_counter() - prepare_start) * 1000 - self.events = [] - self.last = {} def mark(self, name): event = torch.cuda.Event(enable_timing=True) @@ -249,8 +330,10 @@ def forward(self): self.last = {} with torch.cuda.nvtx.range("csa.query_redistribute"): self.q = self.exchange.distribute(self.source_q) - selection_qi = self.selection_exchange.distribute(self.source_qi) - selection_weights = self.selection_exchange.distribute(self.source_weights) + selection_qi = self.source_qi if self.selection_api else self.selection_exchange.distribute(self.source_qi) + selection_weights = ( + self.source_weights if self.selection_api else self.selection_exchange.distribute(self.source_weights) + ) b.qi = self.source_qi if self.redistribution_scope == "selection" else selection_qi b.weights = self.source_weights if self.redistribution_scope == "selection" else selection_weights self.mark("query_redistribute") @@ -266,13 +349,25 @@ def forward(self): self.mark("pool_gather") with torch.cuda.nvtx.range("csa.selection_key_layout"): select_keys = ( - b.global_kbar[self.selection_key_slice] - if self.selection_key_ids is None - else b.global_kbar.index_select(0, self.selection_key_ids) + None + if self.selection_api + else ( + b.global_kbar[self.selection_key_slice] + if self.selection_key_ids is None + else b.global_kbar.index_select(0, self.selection_key_ids) + ) ) self.mark("selection_key_layout") with torch.cuda.nvtx.range("csa.selection"): - if b.select_plan is None: + if self.selection_api: + selected = cudnn_csa_select_cp( + self.source_qi, + b.global_kbar[:, None], + self.source_weights, + workspace=self.cp_selection, + deterministic=False, + ) + elif b.select_plan is None: selected = SimpleNamespace( block_ids=torch.empty((0, self.top_k), device=selection_qi.device, dtype=torch.int32), scores=torch.empty((0, self.top_k), device=selection_qi.device, dtype=torch.float32), @@ -288,7 +383,7 @@ def forward(self): ) self.mark("selection") with torch.cuda.nvtx.range("csa.selection_restore"): - if self.redistribution_scope == "selection": + if self.redistribution_scope == "selection" and not self.selection_api: selected_ids = self.selection_exchange.restore(selected.block_ids) selected_scores = self.selection_exchange.restore(selected.scores) else: @@ -611,6 +706,7 @@ def main(*, keep_group=False, baseline=None, save_baseline=False, comparison_nam parser.add_argument("--redistribution-scope", choices=("selection", "queries"), default="selection") parser.add_argument("--planner-cost-model", help="JSON calibrated THDSelectionCostModel coefficients") parser.add_argument("--planner-strategy", choices=("balanced", "selection"), default="balanced") + parser.add_argument("--selection-api", action="store_true", help="Use the public CSA CP selection API") parser.add_argument("--memory-stages", action="store_true", help="Diagnostic allocator counters at each phase boundary") parser.add_argument("--pool-backward-order", choices=("per-loss", "sum-first"), default="per-loss") parser.add_argument("--output", required=True) @@ -649,6 +745,7 @@ def main(*, keep_group=False, baseline=None, save_baseline=False, comparison_nam args.redistribution_scope, cost_model, args.planner_strategy, + args.selection_api, ) csa.record_memory = args.memory_stages redistribution_ok = check_redistribution(csa) diff --git a/benchmarks/check_csa_cp_selection.py b/benchmarks/check_csa_cp_selection.py new file mode 100644 index 00000000..7cb386fb --- /dev/null +++ b/benchmarks/check_csa_cp_selection.py @@ -0,0 +1,98 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Native distributed checks of source order, subgroups and retained results.""" + +import argparse +import datetime +import json +import math +import os +from itertools import accumulate +from pathlib import Path + +import torch +import torch.distributed as dist + +from cula.hybrid import CsaCPSelectionPlan, CsaCPSelectionWorkspace, cudnn_csa_select_cp + + +def canonical(result): + ids, order = result.block_ids.sort(-1) + scores = result.scores.gather(1, order).masked_fill(ids < 0, 0) + return ids, scores + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--weights-dtype", choices=("bf16", "fp32"), default="bf16") + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + device = torch.device("cuda", torch.cuda.current_device()) + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=5)) + world = dist.get_world_size() + if world < 4 or world % 2: + raise ValueError("launch an even number of >= 4 ranks") + groups = [dist.new_group(list(range(parity, world, 2))) for parity in (0, 1)] + group = groups[dist.get_rank() % 2] + rank, size = dist.get_rank(group), dist.get_world_size(group) + records = [] + # Aligned but unequal source lengths, document-crossing shards, and empty + # zigzag destinations. All policies must restore the same native result. + for lengths in ([4] * (2 * size), [4, 4 * (size + 1)], [4096], [4] * 20 + [984] * 2): + cu = (0, *accumulate(lengths)) + plan = CsaCPSelectionPlan.from_cu_seqlens(cu, rank=rank, world_size=size, top_k=512, policy="contiguous") + torch.manual_seed(20261116 + dist.get_rank()) + n = plan.exchange.source_rows + qi = torch.randn(n, 64, 128, dtype=torch.bfloat16, device=device) + dtype = torch.bfloat16 if args.weights_dtype == "bf16" else torch.float32 + weights = torch.randn(n, 64, dtype=dtype, device=device) / 64 + # Distinct replicated key sets per subgroup catch use of WORLD. + torch.manual_seed(713 + dist.get_rank() % 2) + keys = torch.randn(cu[-1] // 4, 1, 128, dtype=torch.bfloat16, device=device) + baseline = None + for policy in ("contiguous", "zigzag"): + plan = CsaCPSelectionPlan.from_cu_seqlens( + cu, rank=rank, world_size=size, top_k=512, policy=policy, max_tokens_per_rank=cu[-1] + ) + workspace = CsaCPSelectionWorkspace.from_plan(plan, device, group=group) + result = cudnn_csa_select_cp(qi, keys, weights, workspace=workspace) + ids, scores = canonical(result) + saved_ids, saved_scores = result.block_ids.clone(), result.scores.clone() + cudnn_csa_select_cp(qi * 2, keys, weights, workspace=workspace) + retained = torch.equal(saved_ids, result.block_ids) and torch.equal(saved_scores, result.scores) + if baseline is None: + baseline = ids.clone(), scores.clone() + ids_equal = torch.equal(ids, baseline[0]) + delta = (scores - baseline[1]).float() + relative = (delta.norm() / baseline[1].norm().clamp_min(1e-12)).item() + okay = retained and ids_equal and bool(torch.isfinite(scores).all()) and relative < 1e-5 + records.append( + dict( + lengths=lengths, + policy=policy, + weights_dtype=args.weights_dtype, + source_rows=n, + target_rows=plan.exchange.target_rows, + retained=retained, + ids_equal=ids_equal, + score_relative_rms=relative if math.isfinite(relative) else None, + okay=okay, + ) + ) + all_records = [None] * world + dist.all_gather_object(all_records, records) + if dist.get_rank() == 0: + out = Path(os.environ.get("CSA_CHECK_OUTPUT", "csa-cp-checks.json")) + out.write_text(json.dumps(all_records, indent=2, allow_nan=False) + "\n") + print("CHECKS", sum(len(r) for r in all_records), "PASS", all(x["okay"] for r in all_records for x in r), flush=True) + if not all(x["okay"] for r in all_records for x in r): + raise RuntimeError("CSA CP selection qualification failed") + for g in groups: + if g != dist.GroupMember.NON_GROUP_MEMBER: + dist.destroy_process_group(g) + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/cula/_metadata.py b/cula/_metadata.py index 850058fc..e6a02500 100644 --- a/cula/_metadata.py +++ b/cula/_metadata.py @@ -1,7 +1,7 @@ # Copyright 2025-2026 Ant Group Co., Ltd. # SPDX-License-Identifier: Apache-2.0 -"""CPU-only packed-sequence and contiguous-shard metadata helpers.""" +"""Host sequence/shard planning and explicit device index materialization.""" from __future__ import annotations @@ -56,4 +56,24 @@ def validate_shards(shard_offsets: Sequence[int], *, total_tokens: int, world_si return offsets +def _range_indices(ranges: Sequence[tuple[int, int]], device: torch.device) -> torch.Tensor: + """Materialize validated half-open ranges without a large CPU index array. + + CUDA receives only O(ranges) host metadata. The known output size avoids + repeat_interleave's device-to-host size synchronization. This helper does + not synchronize streams; callers retain normal PyTorch stream ordering. + """ + sizes, shifts, total = [], [], 0 + for start, end in ranges: + sizes.append(end - start) + shifts.append(start - total) + total += end - start + if not total: + return torch.empty(0, dtype=torch.int64, device=device) + metadata = torch.tensor((sizes, shifts), dtype=torch.int64, device=device) + result = torch.arange(total, dtype=torch.int64, device=device) + result += torch.repeat_interleave(metadata[1], metadata[0], output_size=total) + return result + + __all__ = ["as_boundaries", "balanced_shards", "validate_shards"] diff --git a/cula/distributed/redistribute.py b/cula/distributed/redistribute.py index 4672ca7c..61c81aa5 100644 --- a/cula/distributed/redistribute.py +++ b/cula/distributed/redistribute.py @@ -5,7 +5,7 @@ Only row ownership/order changes. Floating point values are neither reduced nor cast. The backward is the exact inverse exchange. Preparation is CPU -metadata work plus device index allocation; execution is eager PyTorch/NCCL +metadata work plus device index construction; execution is eager PyTorch/NCCL and allocates outputs. No CUDA Graph contract is implied. """ @@ -15,6 +15,7 @@ import torch.distributed as dist from torch.autograd.function import once_differentiable +from cula._metadata import _range_indices from cula.distributed.thd_planner import THDRankLayout @@ -117,16 +118,12 @@ def _permutations(ranges, rows, device): cursor = end if ordered and cursor == rows: return None, None - order = ( - torch.cat([torch.arange(a, z, dtype=torch.int64, device="cpu") for a, z in ranges]) - if rows - else torch.empty(0, dtype=torch.int64, device="cpu") - ) + order = _range_indices(ranges, device) if order.numel() != rows: raise ValueError("incomplete permutation") inverse = torch.empty_like(order) - inverse[order] = torch.arange(rows, device="cpu") - return order.to(device), inverse.to(device) + inverse[order] = torch.arange(rows, device=device) + return order, inverse class THDRedistributor: diff --git a/cula/distributed/thd_planner.py b/cula/distributed/thd_planner.py index 009e7c5b..1d3247f2 100644 --- a/cula/distributed/thd_planner.py +++ b/cula/distributed/thd_planner.py @@ -110,6 +110,12 @@ class THDPlannerConfig: token_weight: float = 1.0 causal_weight: float = 1.0 boundary_weight: float = 0.02 + min_chunk_tokens: int = 0 + """Minimum newly created selection span; zero preserves the legacy search. + + Existing source fragments and complete short documents remain legal. + This does not change the explicitly requested full-zigzag baseline. + """ @dataclass(frozen=True, slots=True) @@ -346,6 +352,13 @@ def _validate_inputs( raise ValueError(f"{name} must be a positive integer") if config.min_split_tokens % config.block_size: raise ValueError("min_split_tokens must be block aligned") + if ( + isinstance(config.min_chunk_tokens, bool) + or not isinstance(config.min_chunk_tokens, int) + or config.min_chunk_tokens < 0 + or config.min_chunk_tokens % config.block_size + ): + raise ValueError("min_chunk_tokens must be nonnegative and block aligned") if any((right - left) % config.block_size for left, right in zip(boundaries, boundaries[1:])): raise ValueError("every sequence length must be block aligned") if boundaries[-1] // config.block_size < world_size: @@ -415,6 +428,8 @@ def _adaptive_items( # prevents short THD samples from paying a max-CP fragmentation tax. if length_blocks < 2 * min_split_blocks: pieces = 1 + if config.min_chunk_tokens: + pieces = min(pieces, max(1, length_blocks // (config.min_chunk_tokens // block_size))) offsets = balanced_shards(length_blocks, pieces) for left, right in zip(offsets, offsets[1:]): start = sequence_start + left * block_size @@ -988,6 +1003,7 @@ def work(item): intersections = tuple(_source_intersections(placed, offsets)) cost = work(placed) max_compute, max_incoming, max_outgoing = max(compute), max(incoming), max(outgoing) + max_copies = max(offsets[r + 1] - offsets[r] + n for r, n in enumerate(loads)) if model.overlap_intra_inter: max_incoming, max_outgoing = max(*intra_in, *inter_in), max(*intra_out, *inter_out) best = None @@ -1036,23 +1052,22 @@ def work(item): receive_peers[rank] + len(new_edges), max((send_peers[s] + 1 for s in new_edges), default=0), ) - node_additions = defaultdict(int) - for s, n in intersections: - if model.rank_node_ids[s] != model.rank_node_ids[rank]: - node_additions[model.rank_node_ids[s]] += n - candidate_node = max( - node_peak, - node_in[model.rank_node_ids[rank]] + sum(node_additions.values()), - max((node_out[n] + rows for n, rows in node_additions.items()), default=0), - ) + candidate_node = node_peak if model.inter_node_aggregate_gbps is not None: + node_additions = defaultdict(int) + for s, n in intersections: + if model.rank_node_ids[s] != model.rank_node_ids[rank]: + node_additions[model.rank_node_ids[s]] += n + candidate_node = max( + node_peak, + node_in[model.rank_node_ids[rank]] + sum(node_additions.values()), + max((node_out[n] + rows for n, rows in node_additions.items()), default=0), + ) transfer = max(transfer, candidate_node * model.bytes_per_query / (model.inter_node_aggregate_gbps * 1e6)) transfer += candidate_degree * model.peer_startup_us / 1e3 # Partial placement uses an upper bound on copies. Final # scoring checks the actual source/destination permutations. - copies = max( - offsets[r + 1] - offsets[r] + n + (placed.tokens if r == rank else 0) for r, n in enumerate(loads) - ) + copies = max(max_copies, offsets[rank + 1] - offsets[rank] + loads[rank] + placed.tokens) critical = max(max_compute, compute[rank] + cost) + transfer if candidate_peer_peak: critical += copies * model.packing_ns_per_query / 1e6 + model.exchange_us / 1e3 @@ -1205,6 +1220,38 @@ def overlap(left, right): return compute + transfer +def _source_selection_candidate(source, world, model): + """Price known identity ownership without reconstructing its transport. + + Source items already have exact causal work and global token order. Reuse + both for baseline metrics instead of recomputing candidate prefixes and + source intersections several times on the common no-migration path. + """ + groups = [[] for _ in range(world)] + tokens, causal, boundaries, compute = ([0] * world for _ in range(4)) + previous = None + for item, rank in source: + groups[rank].append( + THDSpan(item.sequence, item.token_start, item.token_end, item.sequence_start, item.sequence_end, rank) + ) + tokens[rank] += item.tokens + causal[rank] += item.causal_work + compute[rank] += model.candidate_ns * item.causal_work / 1e6 + model.query_ns * item.tokens / 1e6 + model.span_us / 1e3 + if previous is not None and previous[0].sequence == item.sequence and previous[1] != rank: + boundaries[previous[1]] += 1 + boundaries[rank] += 1 + previous = item, rank + metrics = THDLayoutMetrics( + tuple(tokens), + tuple(causal), + tuple(boundaries), + max(tokens) / (sum(tokens) / world), + max(causal) / (sum(causal) / world) if sum(causal) else 1.0, + max(compute), + ) + return tuple(THDRankLayout(r, tuple(g)) for r, g in enumerate(groups)), metrics + + def plan_thd_selection( global_cu_seqlens: torch.Tensor | Sequence[int], *, @@ -1241,18 +1288,27 @@ def plan_thd_selection( if max_tokens_per_rank % config.block_size or max_tokens_per_rank < max(b - a for a, b in zip(offsets, offsets[1:])): raise ValueError("max_tokens_per_rank must be block aligned and fit every source shard") base = _contiguous_items(boundaries, world_size, config.block_size) + source_fragments = ( + {(item.sequence, item.token_start, item.token_end) for item, _ in base} if config.min_chunk_tokens else set() + ) movable = {i for i, (a, z) in enumerate(zip(boundaries, boundaries[1:])) if z - a >= 2 * config.min_split_tokens} pinned = [(item, rank) for item, rank in base if item.sequence not in movable] pinned_spans = () - candidates = [] - - def add(assignments, policy, *, include_pinned=True): - spans = _coalesce(assignments) - if include_pinned: - # Pinned and movable documents are disjoint. Reusing their spans - # avoids rebuilding hundreds of identical short-document objects - # for every planning grid. - spans = sorted((*pinned_spans, *spans), key=lambda s: (s.sequence, s.token_start)) + source_ranks, source_metrics = _source_selection_candidate(base, world_size, cost_model) + candidates = [("contiguous", source_ranks, source_metrics.score)] + + def add(assignments, policy): + # Pinned and movable documents are disjoint. Reusing pinned spans + # avoids rebuilding hundreds of short-document objects per grid. + spans = sorted((*pinned_spans, *_coalesce(assignments)), key=lambda s: (s.sequence, s.token_start)) + # Capacity repair can split an otherwise useful item into tiny pieces. + # Reject the whole candidate instead of silently weakening the floor. + # Existing short source fragments are unavoidable and remain legal. + if policy != "contiguous" and any( + span.tokens < config.min_chunk_tokens and (span.sequence, span.token_start, span.token_end) not in source_fragments + for span in spans + ): + return grouped = [[] for _ in range(world_size)] for span in spans: grouped[span.owner].append(span) @@ -1262,7 +1318,6 @@ def add(assignments, policy, *, include_pinned=True): score = cost_model.estimate_ms(ranks, block_size=config.block_size) candidates.append((policy, ranks, score)) - add(base, "contiguous", include_pinned=False) baseline_ms = candidates[0][2] lower_bound = baseline_ms # This bound allows arbitrary row movement, including short documents. @@ -1308,14 +1363,22 @@ def add(assignments, policy, *, include_pinned=True): and _zigzag_lower_bound(boundaries, base, world_size, config.block_size, cost_model) < threshold ): zigzag = zigzag_thd_layout(boundaries, world_size=world_size, block_size=config.block_size) - if all(rank.local_tokens <= max_tokens_per_rank for rank in zigzag): + if all(rank.local_tokens <= max_tokens_per_rank for rank in zigzag) and all( + span.tokens >= config.min_chunk_tokens or (span.sequence, span.token_start, span.token_end) in source_fragments + for rank in zigzag + for span in rank.spans + ): candidates.append(("zigzag", zigzag, cost_model.estimate_ms(zigzag, block_size=config.block_size))) selected = min(candidates, key=lambda c: (c[2], c[0] != "contiguous", c[0])) if selected[2] >= cost_model._migration_threshold_ms(baseline_ms): selected = candidates[0] policy, ranks, score = selected - metrics = _metrics( - [span for rank in ranks for span in rank.spans], world_size=world_size, block_size=config.block_size, config=config + metrics = ( + source_metrics + if policy == "contiguous" + else _metrics( + [span for rank in ranks for span in rank.spans], world_size=world_size, block_size=config.block_size, config=config + ) ) metrics = replace(metrics, score=score) return THDLayoutPlan( diff --git a/cula/hybrid/__init__.py b/cula/hybrid/__init__.py index 708d6636..998458c9 100644 --- a/cula/hybrid/__init__.py +++ b/cula/hybrid/__init__.py @@ -3,6 +3,7 @@ """cuDNN-backed hybrid-attention integration primitives.""" +from cula.hybrid.csa_cp import CsaCPSelectionPlan, CsaCPSelectionResult, CsaCPSelectionWorkspace, cudnn_csa_select_cp from cula.hybrid.distributed_pool import cudnn_index_pool_cp from cula.hybrid.dsa import ( DEFAULT_CANDIDATE_BUDGET_BYTES, @@ -18,6 +19,9 @@ from cula.hybrid.plan import IndexPoolPlan, IndexPoolShardPlan __all__ = [ + "CsaCPSelectionPlan", + "CsaCPSelectionResult", + "CsaCPSelectionWorkspace", "DEFAULT_CANDIDATE_BUDGET_BYTES", "MAX_BALANCED_CANDIDATE_BUDGET_BYTES", "MEMORY_EFFICIENT_CANDIDATE_BUDGET_BYTES", @@ -28,6 +32,7 @@ "IndexPoolPlan", "IndexPoolShardPlan", "cudnn_dsa_select", + "cudnn_csa_select_cp", "cudnn_index_pool", "cudnn_index_pool_cp", ] diff --git a/cula/hybrid/csa_cp.py b/cula/hybrid/csa_cp.py new file mode 100644 index 00000000..99df789e --- /dev/null +++ b/cula/hybrid/csa_cp.py @@ -0,0 +1,283 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepared CSA indexer selection with adaptive CP routing and inverse mapping.""" + +from __future__ import annotations + +import math +from bisect import bisect_left, bisect_right +from collections.abc import Sequence +from dataclasses import dataclass + +import torch +import torch.distributed as dist + +from cula._metadata import _range_indices, as_boundaries, balanced_shards +from cula.distributed.redistribute import THDRedistributionPlan, THDRedistributor +from cula.distributed.thd_planner import ( + THDPlannerConfig, + THDRankLayout, + THDSelectionCostModel, + THDSpan, + plan_thd_selection, + zigzag_thd_layout, +) +from cula.hybrid.dsa import DsaSelectionPlan, DsaSelectionWorkspace, cudnn_dsa_select + + +@dataclass(frozen=True, slots=True) +class CsaCPSelectionPlan: + """Host-only selection plan for one rank of a packed THD batch. + + Source tokens are contiguous, balanced in complete pooling blocks. All + ranks must use identical global boundaries, policy, capacity and costs. + Documents must be block-aligned; selection destinations may be empty. + No CUDA metadata is read, hardware probed, or process group accessed here. + """ + + global_cu_seqlens: tuple[int, ...] + source_offsets: tuple[int, ...] + ranks: tuple[THDRankLayout, ...] + policy: str + evaluated_scores: tuple[tuple[str, float], ...] + top_k: int + block_size: int + exchange: THDRedistributionPlan + selection: DsaSelectionPlan | None + key_ranges: tuple[tuple[int, int], ...] + cost_model: THDSelectionCostModel | None + + @classmethod + def from_cu_seqlens( + cls, + global_cu_seqlens: torch.Tensor | Sequence[int], + *, + rank: int, + world_size: int, + top_k: int, + policy: str = "planner", + cost_model: THDSelectionCostModel | None = None, + max_tokens_per_rank: int | None = None, + config: THDPlannerConfig | None = None, + candidate_budget_bytes: int | None = None, + ) -> CsaCPSelectionPlan: + """Choose a layout, then prepare native selection-window metadata. + + ``planner`` requires explicit calibrated costs. Fixed ``contiguous`` + and complete per-document ``zigzag`` policies need no cost model. + The default destination row cap is twice the largest source shard; + it is separate from the native compact-logit scratch budget. + """ + for name, value, minimum in (("world_size", world_size, 1), ("rank", rank, 0), ("top_k", top_k, 1)): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{name} must be an integer >= {minimum}") + if rank >= world_size or top_k > 2048: + raise ValueError("rank must be below world_size and top_k must be <= 2048") + config = THDPlannerConfig() if config is None else config + if not isinstance(config, THDPlannerConfig): + raise TypeError("config must be a THDPlannerConfig") + block = config.block_size + if isinstance(block, bool) or not isinstance(block, int) or block < 1: + raise ValueError("config.block_size must be a positive integer") + if policy not in ("contiguous", "zigzag", "planner"): + raise ValueError("policy must be contiguous, zigzag, or planner") + if cost_model is not None and not isinstance(cost_model, THDSelectionCostModel): + raise TypeError("cost_model must be a THDSelectionCostModel") + if cost_model is not None and len(cost_model.rank_node_ids) != world_size: + raise ValueError("cost-model node map must match the process-group size") + if policy == "planner" and cost_model is None: + raise ValueError("planner requires an explicit calibrated cost_model") + if candidate_budget_bytes is not None and ( + isinstance(candidate_budget_bytes, bool) + or not isinstance(candidate_budget_bytes, int) + or candidate_budget_bytes < 4 + ): + raise ValueError("candidate_budget_bytes must hold at least one FP32 element") + cu = as_boundaries(global_cu_seqlens, name="global_cu_seqlens") + if cu[-1] >= 2**31 or any(x % block for x in cu): + raise ValueError("document boundaries must be block-aligned and fit int32") + offsets = tuple(x * block for x in balanced_shards(cu[-1] // block, world_size)) + source_cap = max(z - a for a, z in zip(offsets, offsets[1:])) + cap = 2 * source_cap if max_tokens_per_rank is None else max_tokens_per_rank + if isinstance(cap, bool) or not isinstance(cap, int) or cap < source_cap or cap % block: + raise ValueError("max_tokens_per_rank must be block-aligned and fit every source shard") + scores = () + if policy == "planner": + planned = plan_thd_selection( + cu, world_size=world_size, max_tokens_per_rank=cap, cost_model=cost_model, config=config + ) + ranks, policy, scores = planned.ranks, planned.policy, planned.evaluated_scores + elif policy == "zigzag": + ranks = zigzag_thd_layout(cu, world_size=world_size, block_size=block) + else: + ranks = tuple( + THDRankLayout( + owner, + tuple( + THDSpan(seq, max(left, a), min(right, z), a, z, owner) + for seq in range(bisect_right(cu, left) - 1, bisect_left(cu, right)) + for a, z in (cu[seq : seq + 2],) + ), + ) + for owner, (left, right) in enumerate(zip(offsets, offsets[1:])) + ) + if any(r.local_tokens > cap for r in ranks): + raise ValueError("selected layout exceeds max_tokens_per_rank") + local = ranks[rank] + ranges = tuple((s.sequence_start // block, s.sequence_end // block) for s in local.spans) + prefix = [0] + for a, z in ranges: + prefix.append(prefix[-1] + z - a) + native = None + if local.local_tokens: + native = DsaSelectionPlan.from_cu_seqlens( + local.local_cu_seqlens, + prefix, + top_k=top_k, + block_size=block, + q_causal_offsets=local.causal_offsets, + candidate_budget_bytes=candidate_budget_bytes, + ) + return cls( + cu, + offsets, + ranks, + policy, + scores, + top_k, + block, + THDRedistributionPlan.build(ranks, offsets, rank), + native, + ranges, + cost_model, + ) + + +@dataclass(frozen=True, slots=True) +class CsaCPSelectionWorkspace: + """Device metadata, transport and reusable selection scratch. + + Prepare/use on the same CUDA stream, or explicitly synchronize between + streams. Use a separate workspace for overlapping invocations; the native + scratch is mutable. Keep the process group alive through every invocation. + This eager, allocating transport has no CUDA Graph capture contract. + """ + + plan: CsaCPSelectionPlan + exchange: THDRedistributor + selection: DsaSelectionWorkspace | None + key_slice: slice + key_indices: torch.Tensor | None + + @classmethod + def from_plan(cls, plan: CsaCPSelectionPlan, device: torch.device | str, *, group=None) -> CsaCPSelectionWorkspace: + if not isinstance(plan, CsaCPSelectionPlan): + raise TypeError("plan must be a CsaCPSelectionPlan") + device = torch.device(device) + if device.type != "cuda": + raise ValueError("CSA CP selection requires a CUDA device") + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + exchange = THDRedistributor(plan.exchange, device, group=group) + native = None if plan.selection is None else DsaSelectionWorkspace.from_plan(plan.selection, device) + ranges, indices = plan.key_ranges, None + key_slice = slice(0, 0) + if ranges: + if all(a[1] == b[0] for a, b in zip(ranges, ranges[1:])): + key_slice = slice(ranges[0][0], ranges[-1][1]) + else: + indices = _range_indices(ranges, device) + return cls(plan, exchange, native, key_slice, indices) + + +@dataclass(frozen=True, slots=True) +class CsaCPSelectionResult: + """Source-ordered document-local int32 block IDs and FP32 raw scores. + + Outputs own their storage and survive workspace reuse. Invalid slots are + -1 IDs; slot order is unspecified. Selection is discrete and has no autograd + connection: downstream indexer training uses its explicit backward/loss. + """ + + block_ids: torch.Tensor + scores: torch.Tensor + + +@torch.no_grad() +def cudnn_csa_select_cp( + qi: torch.Tensor, + global_kbar: torch.Tensor, + weights: torch.Tensor, + *, + workspace: CsaCPSelectionWorkspace, + sm_scale: float = 1.0, + deterministic: bool = False, +) -> CsaCPSelectionResult: + """Route local queries, select pooled blocks, and restore original rows. + + ``qi`` is contiguous BF16 [source_rows, H, D], ``weights`` is BF16/FP32 + [source_rows, H], and ``global_kbar`` is contiguous BF16 [total_blocks, 1, D] + in original packed-document order, already gathered by the caller. Pooling, + attention, projections and their gradients stay independently composable. + No precision casts, head padding, or device-to-host metadata copies occur. + + All ranks must enter in the same order with matching tensor trailing + dimensions/dtypes and scalar options, including empty destination ranks. + Tensor validation is local, like the underlying PyTorch collectives. + """ + if not isinstance(workspace, CsaCPSelectionWorkspace): + raise TypeError("workspace must be a CsaCPSelectionWorkspace") + if not isinstance(deterministic, bool): + raise TypeError("deterministic must be a bool") + if isinstance(sm_scale, bool) or not isinstance(sm_scale, int | float) or not math.isfinite(sm_scale): + raise ValueError("sm_scale must be a finite host scalar") + plan, exchange = workspace.plan, workspace.exchange + if dist.get_rank(exchange.group) != plan.exchange.rank or dist.get_world_size(exchange.group) != plan.exchange.world_size: + raise ValueError("workspace process group no longer matches the plan") + if qi.ndim != 3 or qi.shape[0] != plan.exchange.source_rows or min(qi.shape[1:]) < 1: + raise ValueError("qi must have shape [source_rows, H, D] with positive H/D") + if weights.shape != qi.shape[:2] or global_kbar.shape != (plan.global_cu_seqlens[-1] // plan.block_size, 1, qi.shape[2]): + raise ValueError("weights/global_kbar shapes do not match the source query contract") + if ( + qi.dtype != torch.bfloat16 + or global_kbar.dtype != torch.bfloat16 + or weights.dtype not in (torch.bfloat16, torch.float32) + ): + raise TypeError("qi/global_kbar must be BF16 and weights BF16 or FP32; no implicit quantization") + for tensor in (qi, global_kbar, weights): + if tensor.device != exchange.device or not tensor.is_contiguous(): + raise ValueError("inputs must be contiguous and on the workspace device") + if plan.cost_model is not None: + payload = qi.shape[1] * qi.shape[2] * qi.element_size() + qi.shape[1] * weights.element_size() + plan.top_k * 8 + if payload != plan.cost_model.bytes_per_query: + raise ValueError("cost_model.bytes_per_query must match Q/weights plus returning int32 IDs/FP32 scores") + routed_q, routed_weights = exchange.distribute(qi), exchange.distribute(weights) + if plan.selection is None: + ids = torch.empty((0, plan.top_k), dtype=torch.int32, device=exchange.device) + scores = torch.empty((0, plan.top_k), dtype=torch.float32, device=exchange.device) + else: + keys = ( + global_kbar[workspace.key_slice] + if workspace.key_indices is None + else global_kbar.index_select(0, workspace.key_indices) + ) + selected = cudnn_dsa_select( + routed_q, + keys, + routed_weights, + plan=plan.selection, + workspace=workspace.selection, + sm_scale=sm_scale, + deterministic=deterministic, + ) + ids, scores = selected.block_ids, selected.scores + ids, scores = exchange.restore(ids), exchange.restore(scores) + # The identity transport returns aliases, whereas a real exchange owns its + # outputs. Provide the same retained-result lifetime for every policy. + if plan.exchange.identity: + ids, scores = ids.clone(), scores.clone() + return CsaCPSelectionResult(ids, scores) + + +__all__ = ["CsaCPSelectionPlan", "CsaCPSelectionWorkspace", "CsaCPSelectionResult", "cudnn_csa_select_cp"] diff --git a/docs/CSA_CP_SELECTION.md b/docs/CSA_CP_SELECTION.md new file mode 100644 index 00000000..695341f7 --- /dev/null +++ b/docs/CSA_CP_SELECTION.md @@ -0,0 +1,144 @@ +# CSA context-parallel selection API + +`cudnn_csa_select_cp` packages THD planning, query/weight redistribution, +cuDNN block selection, and the inverse exchange. It returns document-local +block IDs and FP32 scores in the caller's original token order. Downstream +attention implementations can use the planner without copying benchmark code. + +The interface stops at selection. Projection weights, pooling transforms, +sparse attention, and the indexer loss remain independently composable. +`cudnn_index_pool_cp` can pool before gathering; the caller supplies global +pooled keys after its normalization/position transform. Attention and KDA +keep source ownership. + +## Integration + +```python +import torch.distributed as dist +from cula.distributed import THDPlannerConfig, THDSelectionCostModel +from cula.hybrid import ( + CsaCPSelectionPlan, + CsaCPSelectionWorkspace, + cudnn_csa_select_cp, +) + +# Identical host-visible boundaries on all ranks, prepared outside forward. +plan = CsaCPSelectionPlan.from_cu_seqlens( + global_cu_seqlens_cpu, + rank=dist.get_rank(cp_group), + world_size=dist.get_world_size(cp_group), + top_k=512, + policy="planner", + cost_model=THDSelectionCostModel(**calibrated_coefficients), + config=THDPlannerConfig(min_chunk_tokens=1024), + candidate_budget_bytes=512 << 20, +) +workspace = CsaCPSelectionWorkspace.from_plan(plan, qi.device, group=cp_group) + +# Reuse for the same packed layout. Nondeterministic selection is the default. +selected = cudnn_csa_select_cp( + qi, # BF16 [source_rows, index_heads, index_dim] + global_kbar[:, None], # BF16 [global_blocks, 1, index_dim] + index_weights, # BF16 or FP32 [source_rows, index_heads] + workspace=workspace, +) +# selected.block_ids: int32 [source_rows, 512], document-local, -1 padding +# selected.scores: FP32 [source_rows, 512], paired raw scores +``` + +`plan.policy` and `plan.evaluated_scores` expose the chosen candidate and modeled +costs. Fixed `contiguous` and complete per-document `zigzag` policies need no +cost model. There is no implicit hardware detection, calibration, cache, or +device-to-host metadata readback. Planning remains on the host; workspace +materialization uploads compact interval metadata and builds the large integer +index/permutation arrays on the target device. This avoids constructing and +copying token-sized CPU arrays. Device operations use explicit output sizes. + +## Layout, lifetime and numerical contract + +- Documents are block-aligned, with pooling block size four by default. Source + ownership divides the packed stream into block-balanced contiguous shards. + `plan.source_offsets` gives the exact cuts, which can have unequal lengths. + There must be at least one block per source rank. +- All ranks use identical global metadata, costs and constraints. Rank IDs and + topology maps are relative to the supplied group; subgroups are supported. + Every rank enters the same exchanges, including empty selection destinations. +- Inputs are contiguous on the workspace device. Trailing dimensions, dtypes, + scale and deterministic mode agree across peers. No casts or head padding + are implicit; native backend head/dimension restrictions still apply. +- `cost_model.bytes_per_query` must match outgoing Q/weights and returning + int32 IDs/FP32 scores: `H*D*2 + H*weights.element_size() + top_k*8`. + This checks payload shape, not applicability of fitted kernel/bandwidth costs. +- IDs stay local to the original document. Rows are restored to source order; + output slot order is unspecified. Keep IDs and scores paired when sorting. + Add document block offsets only for a backend requiring global IDs. +- Results own their storage and survive workspace reuse. Identity routing + copies native outputs to provide the same lifetime as a real exchange. +- Selection is discrete and runs without autograd. Indexer training uses its + explicit backward/loss interface, as in the complete CSA benchmark. +- Scratch is mutable. Prepare/use on one stream, or establish an explicit + stream dependency. Overlapping invocations require separate workspaces. + Keep the process group alive. The eager transport allocates and does not + promise CUDA Graph capture/replay support. + +## Granularity and smaller contexts + +One planner handles 128K, 256K and 1M contexts. For a single long document, +causal selection work falls roughly quadratically with context length, while +moved bytes fall roughly linearly. Startup, span launches and preparation can +dominate shorter contexts. + +`min_split_tokens` controls document eligibility, not resulting piece size. +`min_chunk_tokens` separately constrains newly created selection spans: + +1. Adaptive grids cap their piece count to meet the floor. +2. Candidates violating it after placement/capacity repair are rejected. +3. Existing source fragments and complete short documents remain legal. +4. Contiguous remains feasible even if the requested grain is too large. +5. The planner applies the constraint to its full-zigzag candidate; explicitly + forcing `policy="zigzag"` preserves the baseline layout. + +The floor defaults to zero for backward compatibility. The example explicitly +uses 1024 tokens, a tunable constraint rather than a universally optimal value. +For 128K at CP64, six pieces per rank average about 341 tokens; a 1024-token +floor caps the adaptive partition at 128 pieces. This is a CPU planning +property, not a measured CP64 performance result. + +Destination capacity defaults to twice the largest source shard. It limits +query rows, not total memory. Candidate scratch has a separate budget; global +keys, routed payloads, retained results and key packing consume additional +memory. Explicit full zigzag can exceed the default cap for tiny documents; +raise `max_tokens_per_rank` deliberately to benchmark that layout. + +Calibrate for the actual shape, topology and message sizes. Historical 1M +profiles' 20/40 ms `min_gain_ms` values are one-use preparation budgets, not +universal defaults. Reusing them at shorter contexts can reject useful layouts. +Measure preparation, transport and native selection separately, then validate +their sum on held-out distributions. Use explicit calibrated parameters rather +than case-name or context-length-specific layout rules. + +## Validation + +- `tests/test_csa_cp_selection.py`: host metadata parity, invalid inputs, + capacities, result lifetime and CP64 grain constraints. +- `benchmarks/check_csa_cp_selection.py`: native selection in two disjoint + subgroups, unequal source sizes, empty destinations, exact source mapping, + and workspace reuse. +- `benchmarks/bench_csa_cp_windows.py`: legacy contiguous versus the public + contiguous/zigzag/planner paths, with rotating measurement order and full + output/seven-gradient comparisons. Small fixtures add independent FP32 + attention/indexer/pooling checks. + +The performance scope is complete CSA operator F/B, including pooling, +selection, sparse attention, teacher/KL backward, layouts and communication. +It excludes projections, KDA, MoE, optimizer and checkpoint recomputation. +Comparison fixtures coexist during timing: diagnostic allocator peaks include +other variants and are not standalone per-variant memory measurements. +Workspace bytes are reported separately. First-use compilation is outside +warmed timing and is not interpreted as a planner speedup. Fresh preparation +is rebuilt after warmup, with rotated and reversed policy order. Its full +wall time includes workspace allocation, H2D metadata and a completion sync; +there is no implicit cache or plan-reuse credit. Host-time outliers are retained. + +Measured B300 CP8 results and the preparation limitations are documented in +[the 128K/256K study](CSA_CP_WINDOW_RESULTS.md). diff --git a/docs/CSA_CP_WINDOW_RESULTS.md b/docs/CSA_CP_WINDOW_RESULTS.md new file mode 100644 index 00000000..b6d27a7b --- /dev/null +++ b/docs/CSA_CP_WINDOW_RESULTS.md @@ -0,0 +1,139 @@ +# CSA CP selection at 128K and 256K + +This tests the public `cudnn_csa_select_cp` API on eight B300 GPUs. The workload +is **one complete CSA operator forward/backward**, including ratio-4 pooling, +pooled-key and attention-KV gathers, selection routing/restoration, sparse +attention, teacher/KL, and all activation/parameter gradients. It excludes +KDA, projections, MoE, optimizer and checkpoint recomputation. These numbers +must not be substituted for a connected seven-KDA/one-CSA training block. + +## Protocol + +- CSA H64/D576/V512; indexer H64/D128; Top-K512; BF16 Q/K/weights; FP32 scores, + nonlinearities and communication reductions. TF32 and BF16 reduced-precision + GEMM intermediate reductions are disabled. Native nondeterministic mode. +- PyTorch 2.12.1+cu130, CUDA runtime 13.0, CuTeDSL 4.8.1. cuDNN Frontend source + `855ccd77ba9efed22441fd454f7cc0206a068e94`: all 3,285 checked Python/CUDA/C++ + source files match that commit. The installed frontend distribution version + in the JSON is wheel metadata, not the identity of the imported source tree. +- Five distributions per context: one document, 1,024 equal documents, 512 + slightly unequal documents, 998 short plus two long documents, and a + 1,000-document short/medium/long-tail mixture. `bench_csa_cp_windows.cases` + specifies exact lengths without a tokenizer or private dataset. +- Two warmups, seven execution samples and eight fresh preparation samples. + Policy order is rotated and reversed. Tables use the median of each sample's + maximum-rank duration. Preparation includes CPU planning, workspace/index + construction, H2D copies and a completion sync. It is rebuilt, not cached. + RNG generation, JIT compilation and correctness oracles are outside timing. +- Throughput experiment: existing compute/transport coefficients, explicit + `min_gain_ms=0`, `max_chunks_per_rank=6`, and `min_chunk_tokens=1024`. + This tests reuse-oriented planning; **zero is not a recommendation for a + fresh layout used once**. It does not replace the shipped 1M profile. + +## Complete CSA results + +C = public contiguous; Z = public complete per-document zigzag; P = public +planner. All times are milliseconds. The last column compares **preparation +plus one step** against the faster C/Z under that same accounting. + +| Case | C step | Z step | P step | C preparation | P preparation | P one-use gap | +|---|---:|---:|---:|---:|---:|---:| +| 128k-long-tail-1000 | 32.37 | 37.90 | 32.35 | 7.58 | 12.13 | +11.34% | +| 128k-mixed-1000 | 33.07 | 38.75 | 33.07 | 7.64 | 28.31 | +50.77% | +| 128k-near-uniform-512 | 10.01 | 11.21 | 10.03 | 3.74 | 4.96 | +8.92% | +| 128k-single | 47.60 | 46.23 | 45.82 | 0.73 | 7.55 | +13.21% | +| 128k-uniform-1024 | 9.29 | 11.10 | 9.30 | 6.78 | 8.97 | +13.70% | +| 256k-long-tail-1000 | 99.36 | 115.25 | 92.20 | 9.70 | 31.81 | +13.72% | +| 256k-mixed-1000 | 83.64 | 96.05 | 82.15 | 9.59 | 32.67 | +23.16% | +| 256k-near-uniform-512 | 21.79 | 24.21 | 21.76 | 3.94 | 5.02 | +4.10% | +| 256k-single | 115.29 | 105.58 | 104.15 | 1.12 | 7.79 | +4.89% | +| 256k-uniform-1024 | 18.96 | 22.56 | 18.94 | 6.91 | 8.97 | +7.89% | + +The worst measured steady-state gap against faster C/Z is **+0.142%**. Single +long documents improve over contiguous by about **3.7% at 128K** and **9.7% at +256K**. The 256K long-tail case improves about **7.2%**. These are observed +candidate comparisons, not a proof of optimality over arbitrary THD inputs. + +**Fresh preparation is still a limitation.** Searching can cost more than the +execution it saves, particularly when a mixed layout ultimately returns to +contiguous. The complete preparation-plus-one-step result is not uniformly +near the best fixed policy, and this standalone CSA study does not establish +an average <3% preparation bound. Host-time spikes remain in the raw critical +samples; no samples or garbage-collection time were removed. For example, the +256K long-tail planner preparation has a 31.81 ms median and a 132.93 ms maximum. +Do not infer a stable preparation budget from a single sample. + +## Sensitivity and next cost-model work + +Two additional experiments are retained in the machine-readable results: + +- Keeping the historical **20 ms** absolute headroom selected contiguous in + all ten cases, missing useful single-long-document routing. +- Reducing search to **two chunks/rank with 2 ms headroom** retained the single + long-document gains, but the 256K two-long/998-short mixture selected a route + **2.84% slower** than contiguous. Full search found a faster route. Preparation + for that exploratory run has only three samples and is not directly comparable + to the final eight-sample protocol. + +These are sensitivity experiments, **not independent short-context coefficient +calibration**. Neither a smaller fixed headroom nor fewer search grids is a +universal fix. A deployment should budget expected reuse and measure its message +sizes and kernel granularity. Further improvement should price the preparation +of each candidate and the value of additional search, rather than adding a +separate algorithm or case-name lookup for each context length. The interface +keeps costs, capacity, search granularity and minimum fragment size explicit. + +The fragment floor prevents increasingly fine splitting at large CP. At +128K/CP64, a 1,024-token floor caps an adaptive partition at 128 pieces rather +than the requested 384. Original short source fragments remain legal; a +candidate violating the floor after capacity repair is rejected. **CP64 here +is CPU metadata validation only**, not a 64-GPU performance measurement. + +## Correctness and review + +- **435 focused CPU tests pass**, including API validation, transport inverses, + empty targets, capacities, metadata parity at 128K/256K/1M and CP64 floors. +- **124 historical plans are exactly unchanged** at the default zero floor: + 31 distributions × CP8/CP16 × zero/original headroom, compared field-for-field + against the previous PR head. This is host-plan parity, not a new H20 run. +- **128 native subgroup checks pass**, covering BF16 and FP32 weights, unequal + source lengths, empty destinations, exact IDs/scores versus unrouted native + selection, and retained results across workspace reuse. Maximum score + difference is zero in these fixtures. +- All ten final context/distribution cases pass full-tensor comparison of the + output and seven gradients against the legacy contiguous implementation. + Largest relative RMS is **0.214%**; maximum auxiliary loss absolute difference + is **4.66e-10**. The native nondeterministic computation is not bitwise stable; + this does not change its dtype or numerical approximation contract. +- Three small fixtures under all four paths pass independent FP32 attention, + teacher/KL and pooling-gradient checks. These references use the selected key + set; they do not independently prove native discrete ranking. +- The 125 source files in the final timing manifest match the reviewed checkout. + Review checked group participation, unique inverse-permutation writes, source + ownership, empty destinations, retained outputs, explicit stream dependencies, + dtype validation and absence of D2H metadata reads. No handwritten GPU kernel + or native precision mode is changed. + +Allocator peaks in the raw harness include four coexisting comparison fixtures; +they are **not standalone per-policy memory claims**. The JSON separately +records candidate-workspace bytes for each rank. + +[Machine-readable results, error envelopes, samples and source hashes](results/csa_cp_windows_20261116.json) + +## Reproduction + +```bash +python -m torch.distributed.run --standalone --nproc-per-node=8 \ + -m benchmarks.bench_csa_cp_windows \ + --profile benchmarks/profiles/thd_selection_b300_cp8.json \ + --output-dir results/csa-windows --tokens 131072 262144 \ + --min-gain-ms 0 --min-chunk-tokens 1024 --max-chunks-per-rank 6 \ + --warmups 2 --repeats 7 --prepare-repeats 8 + +python -m torch.distributed.run --standalone --nproc-per-node=8 \ + -m benchmarks.check_csa_cp_selection --weights-dtype fp32 +``` + +See the [integration contract](CSA_CP_SELECTION.md) for plan/workspace lifetimes, +source ownership and explicit preparation. Calibrate for the actual deployment; +this experiment does not install a universal small-context profile. diff --git a/docs/HYBRID_CUDNN_RUNTIME.md b/docs/HYBRID_CUDNN_RUNTIME.md index 1c479461..1f7bbf56 100644 --- a/docs/HYBRID_CUDNN_RUNTIME.md +++ b/docs/HYBRID_CUDNN_RUNTIME.md @@ -1,5 +1,8 @@ # cuDNN hybrid-attention runtime +For downstream context-parallel indexers, the [CSA CP selection API](CSA_CP_SELECTION.md) +combines adaptive planning, distributed selection and exact source-order restoration. + cuLA's hybrid layer is a layout-aware integration boundary for composing cuDNN Frontend attention kernels. It is intentionally not another graph API: cuDNN owns the kernels and their numerical contracts; cuLA owns static packed diff --git a/docs/results/csa_cp_windows_20261116.json b/docs/results/csa_cp_windows_20261116.json new file mode 100644 index 00000000..6dc9905c --- /dev/null +++ b/docs/results/csa_cp_windows_20261116.json @@ -0,0 +1,5433 @@ +{ + "schema_version": 1, + "date": "2026-11-16", + "platform": "B300 CP8", + "scope": "Complete CSA operator forward/backward; no KDA, projections, MoE or optimizer", + "native_source_verification": { + "commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "checked_files": 3285, + "mismatches": [] + }, + "cpu_tests": 435, + "historical_exact_plan_comparisons": 124, + "runs": { + "full_search": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "cases": { + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "1ce20ae25f95e468a8ceb74f0607b96cd4b3c96afba3f565edacffb45121c460", + "policies": { + "legacy-contiguous": { + "step_ms": 32.311038970947266, + "prepare_ms": 8.923984481953084, + "one_use_ms": 41.23502345290035, + "prepare_min_ms": 8.652233984321356, + "prepare_max_ms": 9.180870023556054, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 32.3218879699707, + 32.34201431274414, + 32.311038970947266, + 32.27436828613281, + 32.30780792236328, + 32.3006706237793, + 32.32668685913086 + ], + "critical_prepare_samples_ms": [ + 9.180870023556054, + 8.702786988578737, + 8.885186980478466, + 9.141317917965353, + 8.652233984321356, + 8.755585993640125, + 8.962781983427703, + 8.985426160506904 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "contiguous": { + "step_ms": 32.37452697753906, + "prepare_ms": 7.578457531053573, + "one_use_ms": 39.952984508592635, + "prepare_min_ms": 7.474332000128925, + "prepare_max_ms": 12.586126918904483, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015154178483681854, + "dq": 0.0004590856367981372, + "dkv": 0.00046865559368039917, + "d_index_q": 0.0003544140670990448, + "d_weights": 0.0003176278618649191, + "d_pool_kv": 0.0005387196908820256, + "d_pool_score": 0.0005430168708674836, + "d_ape": 4.7473252849530434e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.37452697753906, + 32.392513275146484, + 32.36988830566406, + 32.39535903930664, + 32.35891342163086, + 32.35808181762695, + 32.382911682128906 + ], + "critical_prepare_samples_ms": [ + 7.727506919763982, + 7.595576927997172, + 7.546272943727672, + 12.586126918904483, + 7.49861600343138, + 7.561338134109974, + 7.474332000128925, + 7.61300593148917 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "zigzag": { + "step_ms": 37.90224075317383, + "prepare_ms": 45.84232851630077, + "one_use_ms": 83.7445692694746, + "prepare_min_ms": 44.70354691147804, + "prepare_max_ms": 155.66171507816762, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016252246640432245, + "dq": 0.0004916736119971625, + "dkv": 0.0004861918573380407, + "d_index_q": 0.0003569257876233377, + "d_weights": 0.0003177650459406723, + "d_pool_kv": 0.0005486278732828732, + "d_pool_score": 0.0005445220484194565, + "d_ape": 3.860767227143574e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 37.866241455078125, + 37.90224075317383, + 37.91574478149414, + 37.9218864440918, + 37.847808837890625, + 37.96307373046875, + 37.85862350463867 + ], + "critical_prepare_samples_ms": [ + 47.00984014198184, + 45.00916204415262, + 44.70354691147804, + 153.49186910316348, + 46.21615505311638, + 45.468501979485154, + 44.9322130298242, + 155.66171507816762 + ], + "candidate_workspace_bytes_per_rank": [ + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344 + ] + }, + "planner": { + "step_ms": 32.353633880615234, + "prepare_ms": 12.12866249261424, + "one_use_ms": 44.482296373229474, + "prepare_min_ms": 11.894998839125037, + "prepare_max_ms": 123.44329594634473, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015138876329661513, + "dq": 0.00045860825477340696, + "dkv": 0.00046849135874668684, + "d_index_q": 0.0003562482112602212, + "d_weights": 0.00032118335948173643, + "d_pool_kv": 0.0005438274579043697, + "d_pool_score": 0.0005519648229923422, + "d_ape": 4.258189850384637e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.353633880615234, + 32.412353515625, + 32.39923095703125, + 32.37456130981445, + 32.34396743774414, + 32.31990432739258, + 32.341121673583984 + ], + "critical_prepare_samples_ms": [ + 12.077874038368464, + 12.179450946860015, + 123.44329594634473, + 12.385182897560298, + 11.991281062364578, + 12.406638008542359, + 11.894998839125037, + 11.912805959582329 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "a42d7db2614d9e21c3bd24a8de34199977030c3a75e1b28856d51a604529dfdb", + "policies": { + "legacy-contiguous": { + "step_ms": 33.05331039428711, + "prepare_ms": 8.72880796669051, + "one_use_ms": 41.78211836097762, + "prepare_min_ms": 8.669499889947474, + "prepare_max_ms": 9.25529608502984, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 33.07484817504883, + 33.05331039428711, + 33.061248779296875, + 33.03206253051758, + 33.019039154052734, + 33.1328010559082, + 33.0307502746582 + ], + "critical_prepare_samples_ms": [ + 9.25529608502984, + 8.874272927641869, + 8.747634943574667, + 8.709980989806354, + 8.782866061665118, + 8.703469997271895, + 8.687923080287874, + 8.669499889947474 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "contiguous": { + "step_ms": 33.07494354248047, + "prepare_ms": 7.641692471224815, + "one_use_ms": 40.71663601370528, + "prepare_min_ms": 7.504745037294924, + "prepare_max_ms": 110.33575492911041, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002124358368844286, + "dq": 0.0006708973533251172, + "dkv": 0.0006748384557586351, + "d_index_q": 0.000419026902945799, + "d_weights": 0.0003304557285110155, + "d_pool_kv": 0.0005772261277278353, + "d_pool_score": 0.0005797688319552085, + "d_ape": 7.489935465690975e-06 + }, + "max_loss_abs_diff": 3.4924596548080444e-10, + "critical_step_samples_ms": [ + 33.100799560546875, + 33.073055267333984, + 33.057918548583984, + 33.08787155151367, + 33.07494354248047, + 33.04643249511719, + 33.11958312988281 + ], + "critical_prepare_samples_ms": [ + 8.83635098580271, + 7.582117919810116, + 110.33575492911041, + 7.812974974513054, + 7.556077907793224, + 7.6242099748924375, + 7.504745037294924, + 7.659174967557192 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "zigzag": { + "step_ms": 38.74604797363281, + "prepare_ms": 45.72551843011752, + "one_use_ms": 84.47156640375033, + "prepare_min_ms": 44.49985595420003, + "prepare_max_ms": 152.91648893617094, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.002136063325688249, + "dq": 0.0006448208867270936, + "dkv": 0.0005397046574182863, + "d_index_q": 0.0003537235166520844, + "d_weights": 0.0003170419424130787, + "d_pool_kv": 0.0005661726253782145, + "d_pool_score": 0.0005473705393115235, + "d_ape": 7.236730944585608e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.72604751586914, + 38.74604797363281, + 38.78224182128906, + 38.78335952758789, + 38.72016143798828, + 38.75788879394531, + 38.730464935302734 + ], + "critical_prepare_samples_ms": [ + 47.61352890636772, + 45.467405929230154, + 44.49985595420003, + 152.91648893617094, + 45.98363093100488, + 45.106612145900726, + 44.917382998391986, + 145.95098292920738 + ], + "candidate_workspace_bytes_per_rank": [ + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104 + ] + }, + "planner": { + "step_ms": 33.0742073059082, + "prepare_ms": 28.312876995187253, + "one_use_ms": 61.387084301095456, + "prepare_min_ms": 28.01844000350684, + "prepare_max_ms": 123.6400690395385, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0020962956426669433, + "dq": 0.0006347467076768539, + "dkv": 0.0005276886282440079, + "d_index_q": 0.00035682941768751356, + "d_weights": 0.00032945679607176856, + "d_pool_kv": 0.0005648985794352423, + "d_pool_score": 0.0005625030405139739, + "d_ape": 6.634011369785548e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 33.05827331542969, + 33.0742073059082, + 33.59049606323242, + 33.06959915161133, + 33.093441009521484, + 33.08927917480469, + 33.03887939453125 + ], + "critical_prepare_samples_ms": [ + 28.298633988015354, + 28.01844000350684, + 123.6400690395385, + 28.612435911782086, + 28.067155042663217, + 28.327120002359152, + 28.597721946425736, + 28.228612965904176 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "be33650d531537cf859041aa184f7846ca91ee7d6f07866e13f5eff3df7ff982", + "policies": { + "legacy-contiguous": { + "step_ms": 9.969632148742676, + "prepare_ms": 4.448324500117451, + "one_use_ms": 14.417956648860127, + "prepare_min_ms": 4.3677909998223186, + "prepare_max_ms": 5.127329030074179, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.9651517868042, + 9.94489574432373, + 9.969632148742676, + 9.992608070373535, + 10.025152206420898, + 9.945216178894043, + 10.024991989135742 + ], + "critical_prepare_samples_ms": [ + 4.6669780276715755, + 5.127329030074179, + 4.3677909998223186, + 4.452915978617966, + 4.42988402210176, + 4.433039925061166, + 4.443733021616936, + 4.50379098765552 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "contiguous": { + "step_ms": 10.01318359375, + "prepare_ms": 3.743164998013526, + "one_use_ms": 13.756348591763526, + "prepare_min_ms": 3.6540150176733732, + "prepare_max_ms": 3.838455071672797, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 4.31067005302863e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.485686835600299e-05, + "d_pool_score": 3.734317152918191e-05, + "d_ape": 1.5884632522867948e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.00160026550293, + 10.01318359375, + 9.96771240234375, + 10.067392349243164, + 9.997440338134766, + 10.017472267150879, + 10.042176246643066 + ], + "critical_prepare_samples_ms": [ + 3.793359035626054, + 3.752059070393443, + 3.740546992048621, + 3.739589941687882, + 3.700686036609113, + 3.7457830039784312, + 3.6540150176733732, + 3.838455071672797 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "zigzag": { + "step_ms": 11.205792427062988, + "prepare_ms": 24.085226003080606, + "one_use_ms": 35.291018430143595, + "prepare_min_ms": 23.83457194082439, + "prepare_max_ms": 139.23344609793276, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7925220371458967e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 7.136588948385061e-05, + "d_pool_score": 4.065888036840021e-05, + "d_ape": 1.7333242400394557e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.202655792236328, + 11.212991714477539, + 11.187968254089355, + 11.230527877807617, + 11.205792427062988, + 11.16857624053955, + 11.311712265014648 + ], + "critical_prepare_samples_ms": [ + 27.768890955485404, + 25.140583980828524, + 24.11484904587269, + 23.88202305883169, + 24.055602960288525, + 24.04588391073048, + 23.83457194082439, + 139.23344609793276 + ], + "candidate_workspace_bytes_per_rank": [ + 1855488, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536 + ] + }, + "planner": { + "step_ms": 10.027359962463379, + "prepare_ms": 4.955698561388999, + "one_use_ms": 14.983058523852378, + "prepare_min_ms": 4.824013100005686, + "prepare_max_ms": 5.206209025345743, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3273367675160776e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.542720423634009e-05, + "d_pool_score": 5.10650784981466e-05, + "d_ape": 2.1403371983524285e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.017024040222168, + 10.369183540344238, + 10.027359962463379, + 10.05395221710205, + 10.025664329528809, + 10.148415565490723, + 10.026752471923828 + ], + "critical_prepare_samples_ms": [ + 5.206209025345743, + 5.033118999563158, + 4.875925951637328, + 4.824013100005686, + 4.920054925605655, + 4.9812550423666835, + 4.930142080411315, + 5.035332054831088 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + } + } + }, + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "4ffc0c811647416d205412593d727352a535ab6cd11c26ad666b219d87fde48f", + "policies": { + "legacy-contiguous": { + "step_ms": 47.70070266723633, + "prepare_ms": 0.7209100876934826, + "one_use_ms": 48.42161275492981, + "prepare_min_ms": 0.6982759805396199, + "prepare_max_ms": 0.9826208697631955, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 47.30112075805664, + 47.723167419433594, + 47.41270446777344, + 47.268062591552734, + 47.727745056152344, + 47.810176849365234, + 47.70070266723633 + ], + "critical_prepare_samples_ms": [ + 0.9826208697631955, + 0.7432779530063272, + 0.7265520980581641, + 0.6982759805396199, + 0.7152680773288012, + 0.714245019480586, + 0.7317300187423825, + 0.7042690413072705 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "contiguous": { + "step_ms": 47.59836959838867, + "prepare_ms": 0.7295145187526941, + "one_use_ms": 48.327884117141366, + "prepare_min_ms": 0.7136678323149681, + "prepare_max_ms": 0.8075819350779057, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015765946646741124, + "dq": 0.0005790440951112175, + "dkv": 0.0006679192583025653, + "d_index_q": 0.0007036192249884126, + "d_weights": 0.0006587839768356234, + "d_pool_kv": 0.0008462754770025072, + "d_pool_score": 0.000887909900268915, + "d_ape": 2.647273742900797e-05 + }, + "max_loss_abs_diff": 1.4551915228366852e-10, + "critical_step_samples_ms": [ + 47.504417419433594, + 47.59836959838867, + 47.79350280761719, + 47.29996871948242, + 47.922752380371094, + 47.66716766357422, + 47.468990325927734 + ], + "critical_prepare_samples_ms": [ + 0.8075819350779057, + 0.7368980441242456, + 0.7332341047003865, + 0.7257949328050017, + 0.7136678323149681, + 0.7397379959002137, + 0.7191249169409275, + 0.7216879166662693 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "zigzag": { + "step_ms": 46.22537612915039, + "prepare_ms": 0.9122525225393474, + "one_use_ms": 47.13762865168974, + "prepare_min_ms": 0.8884459966793656, + "prepare_max_ms": 2.3740038741379976, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015766157105562468, + "dq": 0.0005791250183306168, + "dkv": 0.0005826837325579235, + "d_index_q": 0.0006550745397822342, + "d_weights": 0.0005112390665302352, + "d_pool_kv": 0.0007098261716766501, + "d_pool_score": 0.0007224017331145301, + "d_ape": 2.402751970718015e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 46.06342315673828, + 46.19878387451172, + 46.51465606689453, + 46.22537612915039, + 46.21033477783203, + 46.31951904296875, + 46.25299072265625 + ], + "critical_prepare_samples_ms": [ + 2.3740038741379976, + 0.9390920167788863, + 0.9824440348893404, + 0.9092459222301841, + 0.9014920797199011, + 0.9148720419034362, + 0.8884459966793656, + 0.9096330031752586 + ], + "candidate_workspace_bytes_per_rank": [ + 536861644, + 536828104, + 536838528, + 536812296, + 536778684, + 536816940, + 536795936, + 536828336 + ] + }, + "planner": { + "step_ms": 45.816734313964844, + "prepare_ms": 7.54898099694401, + "one_use_ms": 53.365715310908854, + "prepare_min_ms": 7.393982145003974, + "prepare_max_ms": 7.698302972130477, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0015837265043694255, + "dq": 0.0005816460263006044, + "dkv": 0.0007768235696223359, + "d_index_q": 0.0007047741714831017, + "d_weights": 0.000687690452839031, + "d_pool_kv": 0.0008500666348145983, + "d_pool_score": 0.0009004768884284642, + "d_ape": 2.473088932251163e-05 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 45.82166290283203, + 45.84272003173828, + 45.73017501831055, + 46.25267028808594, + 45.816734313964844, + 45.81379318237305, + 45.737918853759766 + ], + "critical_prepare_samples_ms": [ + 7.6499179704114795, + 7.698302972130477, + 7.51110992860049, + 7.575499941594899, + 7.636827998794615, + 7.517814985476434, + 7.522462052293122, + 7.393982145003974 + ], + "candidate_workspace_bytes_per_rank": [ + 536828104, + 536812296, + 536816940, + 536864252, + 536795936, + 536778684, + 536838528, + 536861644 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "38e8fdad2096841ce9b57196cdb05dfe4cd17b7c2311cf5bede714c90c5d3a91", + "policies": { + "legacy-contiguous": { + "step_ms": 9.238431930541992, + "prepare_ms": 8.05632706033066, + "one_use_ms": 17.29475899087265, + "prepare_min_ms": 7.888036081567407, + "prepare_max_ms": 8.295490988530219, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.228096008300781, + 9.246272087097168, + 9.238431930541992, + 9.236512184143066, + 10.079296112060547, + 9.257311820983887, + 9.214655876159668 + ], + "critical_prepare_samples_ms": [ + 8.295490988530219, + 7.888036081567407, + 8.236170979216695, + 8.010777062736452, + 8.126633008942008, + 7.968081976287067, + 8.101877057924867, + 7.93901807628572 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "contiguous": { + "step_ms": 9.287199974060059, + "prepare_ms": 6.7794660571962595, + "one_use_ms": 16.066666031256318, + "prepare_min_ms": 6.7019599955528975, + "prepare_max_ms": 7.129473960958421, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7034465202561103e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.80841349081689e-05, + "d_pool_score": 4.969745146211584e-05, + "d_ape": 2.463488004291754e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.327584266662598, + 9.295040130615234, + 9.287199974060059, + 9.278176307678223, + 9.278143882751465, + 9.258208274841309, + 9.473567962646484 + ], + "critical_prepare_samples_ms": [ + 6.837799912318587, + 6.7019599955528975, + 6.714918999932706, + 6.821699091233313, + 7.129473960958421, + 6.737233023159206, + 6.707626045681536, + 6.954675889573991 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "zigzag": { + "step_ms": 11.102463722229004, + "prepare_ms": 47.22392506664619, + "one_use_ms": 58.32638878887519, + "prepare_min_ms": 45.078030903823674, + "prepare_max_ms": 156.1478169169277, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.723088398789181e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.7408244692422145e-05, + "d_pool_score": 6.371221934010526e-05, + "d_ape": 3.3825512629530646e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.100447654724121, + 11.142175674438477, + 11.123104095458984, + 11.102463722229004, + 11.112383842468262, + 11.097536087036133, + 11.057215690612793 + ], + "critical_prepare_samples_ms": [ + 48.150649992749095, + 46.42646899446845, + 45.078030903823674, + 151.0155029827729, + 47.763709095306695, + 46.68414103798568, + 45.34025501925498, + 156.1478169169277 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "planner": { + "step_ms": 9.295295715332031, + "prepare_ms": 8.97316396003589, + "one_use_ms": 18.26845967536792, + "prepare_min_ms": 8.708094945177436, + "prepare_max_ms": 123.31375910434872, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.9905925067018862e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.216577144844658e-05, + "d_pool_score": 6.720340578861908e-05, + "d_ape": 3.3511195947336086e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.287327766418457, + 9.30742359161377, + 9.270848274230957, + 9.30361557006836, + 9.306976318359375, + 9.284768104553223, + 9.295295715332031 + ], + "critical_prepare_samples_ms": [ + 8.869540877640247, + 8.825646946206689, + 123.31375910434872, + 9.473679936490953, + 8.962570922449231, + 8.708094945177436, + 9.538239915855229, + 8.98375699762255 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "f6f2fb90fa19746799d7f723f65a57a1d55b4e768a2fbaff20ba8977a77dfb5e", + "policies": { + "legacy-contiguous": { + "step_ms": 98.89091491699219, + "prepare_ms": 10.643926972988993, + "one_use_ms": 109.53484188998118, + "prepare_min_ms": 10.527928941883147, + "prepare_max_ms": 11.221883120015264, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 98.17501068115234, + 99.16777801513672, + 99.69891357421875, + 98.36495971679688, + 98.89091491699219, + 100.19251251220703, + 98.40354919433594 + ], + "critical_prepare_samples_ms": [ + 11.221883120015264, + 10.699034086428583, + 10.865055839531124, + 10.588819859549403, + 10.527928941883147, + 10.56276191957295, + 11.06762106064707, + 10.588489938527346 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "contiguous": { + "step_ms": 99.36064147949219, + "prepare_ms": 9.696168475784361, + "one_use_ms": 109.05680995527655, + "prepare_min_ms": 9.520240011624992, + "prepare_max_ms": 10.234966059215367, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010563700585577058, + "dq": 0.00035831076606808053, + "dkv": 0.0004733622916756713, + "d_index_q": 0.0005939123996360771, + "d_weights": 0.0005024215185567405, + "d_pool_kv": 0.000752053852842638, + "d_pool_score": 0.0007688502326254845, + "d_ape": 6.434388548257363e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 98.40025329589844, + 99.20460510253906, + 100.11779022216797, + 99.36064147949219, + 99.45503997802734, + 98.65779113769531, + 99.67919921875 + ], + "critical_prepare_samples_ms": [ + 9.97902499511838, + 9.690101025626063, + 9.577112039551139, + 9.55722399521619, + 9.520240011624992, + 9.813543991185725, + 10.234966059215367, + 9.70223592594266 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "zigzag": { + "step_ms": 115.25299072265625, + "prepare_ms": 147.0409840112552, + "one_use_ms": 262.29397473391145, + "prepare_min_ms": 47.70176007878035, + "prepare_max_ms": 167.99743298906833, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001096030426138749, + "dq": 0.00036871769863560327, + "dkv": 0.0005522801000187844, + "d_index_q": 0.000646529994069314, + "d_weights": 0.0005200898687794669, + "d_pool_kv": 0.0007782778462285144, + "d_pool_score": 0.0007902708796079504, + "d_ape": 6.243475107842311e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 115.171875, + 115.26223754882812, + 115.35302734375, + 115.21993255615234, + 115.44630432128906, + 115.25299072265625, + 115.22217559814453 + ], + "critical_prepare_samples_ms": [ + 137.1572030475363, + 154.45262903813273, + 167.99743298906833, + 48.344484996050596, + 47.70176007878035, + 143.54554004967213, + 155.44524206779897, + 150.53642797283828 + ], + "candidate_workspace_bytes_per_rank": [ + 1073708016, + 1073692980, + 1073648060, + 536804832, + 536760312, + 536814420, + 536809884, + 536864760 + ] + }, + "planner": { + "step_ms": 92.2047348022461, + "prepare_ms": 31.8146159988828, + "one_use_ms": 124.0193508011289, + "prepare_min_ms": 31.22708504088223, + "prepare_max_ms": 132.93269090354443, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.001048036358338339, + "dq": 0.00036006689300403495, + "dkv": 0.0004952787980932682, + "d_index_q": 0.0005006808620709585, + "d_weights": 0.00047204477185006934, + "d_pool_kv": 0.0006647742697551269, + "d_pool_score": 0.0006600247570603834, + "d_ape": 6.129309354351373e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 91.95887756347656, + 92.06761932373047, + 91.93360137939453, + 92.53968048095703, + 92.28575897216797, + 92.2804183959961, + 92.2047348022461 + ], + "critical_prepare_samples_ms": [ + 31.714363023638725, + 126.031952095218, + 31.22708504088223, + 31.914868974126875, + 31.32477297913283, + 31.713226926513016, + 132.93269090354443, + 32.04708802513778 + ], + "candidate_workspace_bytes_per_rank": [ + 412638216, + 526081780, + 536819724, + 696705780, + 782017780, + 754142216, + 725136004, + 1073725588 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "299ffffd41e8b2e4c2be973c6bdb05ae54dc4a991bc3051df8ee4e0a6127615f", + "policies": { + "legacy-contiguous": { + "step_ms": 83.7985610961914, + "prepare_ms": 10.569794510956854, + "one_use_ms": 94.36835560714826, + "prepare_min_ms": 10.402328800410032, + "prepare_max_ms": 11.000228114426136, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 84.08464050292969, + 83.7985610961914, + 83.04704284667969, + 84.08707427978516, + 83.55884552001953, + 84.34912109375, + 83.70649719238281 + ], + "critical_prepare_samples_ms": [ + 11.000228114426136, + 10.834337095730007, + 10.54192113224417, + 10.52551600150764, + 10.856117936782539, + 10.443822015076876, + 10.597667889669538, + 10.402328800410032 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "contiguous": { + "step_ms": 83.6385269165039, + "prepare_ms": 9.585648542270064, + "one_use_ms": 93.22417545877397, + "prepare_min_ms": 9.470761171542108, + "prepare_max_ms": 109.36465603299439, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017248119636322726, + "dq": 0.0005231477545538955, + "dkv": 0.0008859578942512667, + "d_index_q": 0.0003679247450818875, + "d_weights": 0.0003583038720109353, + "d_pool_kv": 0.0013756446763457486, + "d_pool_score": 0.0013569902061103135, + "d_ape": 9.213348057789348e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 83.79923248291016, + 83.6385269165039, + 83.57939147949219, + 83.14595031738281, + 84.14755249023438, + 83.68710327148438, + 83.60105895996094 + ], + "critical_prepare_samples_ms": [ + 11.086768005043268, + 9.873543865978718, + 9.49363096151501, + 9.626674116589129, + 9.470761171542108, + 9.488550946116447, + 109.36465603299439, + 9.544622967951 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "zigzag": { + "step_ms": 96.04755401611328, + "prepare_ms": 92.28663396788761, + "one_use_ms": 188.3341879840009, + "prepare_min_ms": 46.753630973398685, + "prepare_max_ms": 166.36592405848205, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017395277068004997, + "dq": 0.0005277805775999957, + "dkv": 0.0006782649359844867, + "d_index_q": 0.0003691170318294211, + "d_weights": 0.0003624685324509999, + "d_pool_kv": 0.0007577977882457726, + "d_pool_score": 0.0007313251119719116, + "d_ape": 8.559385750338412e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 96.06777954101562, + 96.25202941894531, + 96.04755401611328, + 95.81129455566406, + 95.63484954833984, + 96.13632202148438, + 95.58025360107422 + ], + "critical_prepare_samples_ms": [ + 48.80867002066225, + 166.36592405848205, + 149.53875401988626, + 48.5578749794513, + 46.753630973398685, + 135.76459791511297, + 152.24266005679965, + 47.11186105851084 + ], + "candidate_workspace_bytes_per_rank": [ + 536828196, + 536788788, + 518462572, + 480072156, + 441681740, + 403291324, + 364900908, + 326510492 + ] + }, + "planner": { + "step_ms": 82.14995574951172, + "prepare_ms": 32.668465457390994, + "one_use_ms": 114.81842120690271, + "prepare_min_ms": 32.24971203599125, + "prepare_max_ms": 33.218687982298434, + "selected_policy": "selection-local-5x", + "max_relative_rms": { + "output": 0.0017299188701990226, + "dq": 0.0005249688471103487, + "dkv": 0.0006932649020557219, + "d_index_q": 0.0003879256230062098, + "d_weights": 0.0003670382435938008, + "d_pool_kv": 0.0013198355292136361, + "d_pool_score": 0.0013151740439955597, + "d_ape": 8.808128280757483e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 81.99295806884766, + 82.14995574951172, + 82.31292724609375, + 82.3354263305664, + 81.54524993896484, + 82.6478042602539, + 81.62854766845703 + ], + "critical_prepare_samples_ms": [ + 32.491752062924206, + 32.684653997421265, + 32.89240214508027, + 32.540695974603295, + 32.65227691736072, + 33.04584801662713, + 32.24971203599125, + 33.218687982298434 + ], + "candidate_workspace_bytes_per_rank": [ + 536858700, + 536788788, + 536788788, + 536828196, + 173740692, + 518264420, + 479874004, + 536828196 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "3abdfc48993cc3747d2f08fcfbd801e8c6fa1abaa8a607d585da37c318d9b529", + "policies": { + "legacy-contiguous": { + "step_ms": 21.674495697021484, + "prepare_ms": 4.580917942803353, + "one_use_ms": 26.255413639824837, + "prepare_min_ms": 4.535833024419844, + "prepare_max_ms": 5.053048022091389, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.720863342285156, + 21.699359893798828, + 21.674495697021484, + 21.658496856689453, + 21.68822479248047, + 21.657312393188477, + 21.64521598815918 + ], + "critical_prepare_samples_ms": [ + 5.053048022091389, + 4.535833024419844, + 4.551360965706408, + 4.582296940498054, + 4.611545125953853, + 4.579538945108652, + 4.551580990664661, + 4.621528903953731 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "contiguous": { + "step_ms": 21.785919189453125, + "prepare_ms": 3.9414139464497566, + "one_use_ms": 25.72733313590288, + "prepare_min_ms": 3.8159850519150496, + "prepare_max_ms": 4.042547079734504, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.297053926215977e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.412638872669938e-05, + "d_pool_score": 4.495461915173074e-05, + "d_ape": 1.4301480341692518e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.769792556762695, + 21.76304054260254, + 22.114463806152344, + 21.785919189453125, + 21.940128326416016, + 21.767871856689453, + 21.940895080566406 + ], + "critical_prepare_samples_ms": [ + 4.042547079734504, + 3.9293068693950772, + 3.8159850519150496, + 3.9682361530140042, + 3.8398168981075287, + 3.9214659482240677, + 3.953521023504436, + 3.995029954239726 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "zigzag": { + "step_ms": 24.209856033325195, + "prepare_ms": 24.67734646052122, + "one_use_ms": 48.88720249384642, + "prepare_min_ms": 24.372468004003167, + "prepare_max_ms": 142.0474809128791, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2708240134177204e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.3500598722775094e-05, + "d_pool_score": 5.347177795761659e-05, + "d_ape": 1.327084474519006e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 24.196128845214844, + 24.229408264160156, + 24.214399337768555, + 24.209856033325195, + 24.24527931213379, + 24.169696807861328, + 24.205184936523438 + ], + "critical_prepare_samples_ms": [ + 26.387990918010473, + 24.77585698943585, + 24.372468004003167, + 24.47995904367417, + 24.4268769165501, + 24.750579963438213, + 24.60411295760423, + 142.0474809128791 + ], + "candidate_workspace_bytes_per_rank": [ + 7901184, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376 + ] + }, + "planner": { + "step_ms": 21.7589111328125, + "prepare_ms": 5.023348960094154, + "one_use_ms": 26.782260092906654, + "prepare_min_ms": 4.9473929684609175, + "prepare_max_ms": 5.072213942185044, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.554213974193077e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.1905810341815174e-05, + "d_pool_score": 4.350734098547027e-05, + "d_ape": 1.4490775323448985e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.780927658081055, + 21.7589111328125, + 21.778976440429688, + 21.859455108642578, + 21.743295669555664, + 21.751680374145508, + 21.750688552856445 + ], + "critical_prepare_samples_ms": [ + 5.0161018734797835, + 5.0161059480160475, + 5.0472968723624945, + 5.041101947426796, + 4.989685956388712, + 5.03059197217226, + 5.072213942185044, + 4.9473929684609175 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "27e077b4ff55aa00c71a035377fafcb97d9db8bfb198c213673c877e5ded916b", + "policies": { + "legacy-contiguous": { + "step_ms": 115.51570892333984, + "prepare_ms": 1.11213797936216, + "one_use_ms": 116.627846902702, + "prepare_min_ms": 1.0933621088042855, + "prepare_max_ms": 1.5027470653876662, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 116.38162994384766, + 115.88809967041016, + 115.89097595214844, + 115.51570892333984, + 114.80028533935547, + 115.36595153808594, + 115.4543685913086 + ], + "critical_prepare_samples_ms": [ + 1.5027470653876662, + 1.1409649159759283, + 1.1095319641754031, + 1.0933621088042855, + 1.114262966439128, + 1.1321530910208821, + 1.110012992285192, + 1.104076043702662 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "contiguous": { + "step_ms": 115.2938232421875, + "prepare_ms": 1.1161149595864117, + "one_use_ms": 116.40993820177391, + "prepare_min_ms": 1.087293028831482, + "prepare_max_ms": 1.304103061556816, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014842079275755727, + "dq": 0.0005035188023387811, + "dkv": 0.0010278174526612903, + "d_index_q": 0.0006256382167037119, + "d_weights": 0.000563762569940494, + "d_pool_kv": 0.0013303364084540658, + "d_pool_score": 0.001377639655655214, + "d_ape": 2.0452408219956506e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 114.4939193725586, + 115.84454345703125, + 115.2938232421875, + 114.8194580078125, + 116.123779296875, + 114.58399963378906, + 115.50758361816406 + ], + "critical_prepare_samples_ms": [ + 1.304103061556816, + 1.1089229956269264, + 1.0957310441881418, + 1.1007870780304074, + 1.1440730886533856, + 1.123306923545897, + 1.087293028831482, + 1.187960966490209 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "zigzag": { + "step_ms": 105.58483123779297, + "prepare_ms": 1.1357140028849244, + "one_use_ms": 106.7205452406779, + "prepare_min_ms": 1.1315711308270693, + "prepare_max_ms": 2.692296984605491, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014841542205465054, + "dq": 0.00050380319214946, + "dkv": 0.0010287937173898806, + "d_index_q": 0.0006482205309808245, + "d_weights": 0.000583151648179517, + "d_pool_kv": 0.0013331964396419756, + "d_pool_score": 0.0013779578045436441, + "d_ape": 1.9963974118421826e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 106.04412841796875, + 105.58483123779297, + 105.72246551513672, + 105.1151351928711, + 105.20396423339844, + 106.02706909179688, + 105.4378890991211 + ], + "critical_prepare_samples_ms": [ + 2.692296984605491, + 1.1655869893729687, + 1.1315711308270693, + 1.1326848762109876, + 1.1354410089552402, + 1.15663290489465, + 1.1359869968146086, + 1.1334370356053114 + ], + "candidate_workspace_bytes_per_rank": [ + 1073712188, + 1073660864, + 1073659492, + 1073631792, + 1073736736, + 1073737680, + 1073713936, + 1073713120 + ] + }, + "planner": { + "step_ms": 104.14985656738281, + "prepare_ms": 7.791060954332352, + "one_use_ms": 111.94091752171516, + "prepare_min_ms": 7.6777610229328275, + "prepare_max_ms": 8.784707984887064, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014818148634560368, + "dq": 0.0005030012855905718, + "dkv": 0.0010310600226582155, + "d_index_q": 0.0007705202387904052, + "d_weights": 0.0006243137936169513, + "d_pool_kv": 0.0013332458370744855, + "d_pool_score": 0.0013751792972335336, + "d_ape": 2.096916046719033e-05 + }, + "max_loss_abs_diff": 2.0372681319713593e-10, + "critical_step_samples_ms": [ + 103.95587158203125, + 104.33695983886719, + 103.87484741210938, + 104.14985656738281, + 104.16598510742188, + 104.65894317626953, + 104.02249908447266 + ], + "critical_prepare_samples_ms": [ + 7.850671885535121, + 8.708197972737253, + 7.70237494725734, + 7.6777610229328275, + 8.784707984887064, + 7.718592998571694, + 7.731450023129582, + 8.389059104956686 + ], + "candidate_workspace_bytes_per_rank": [ + 1073660864, + 1073631792, + 1073737680, + 1073736096, + 1073713936, + 1073736736, + 1073659492, + 1073712188 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "a3e04775cd4674a149c44452e86572771d80fd867f99c915f4d8703ea7cf55c1", + "policies": { + "legacy-contiguous": { + "step_ms": 18.887359619140625, + "prepare_ms": 8.322144509293139, + "one_use_ms": 27.209504128433764, + "prepare_min_ms": 7.978446898050606, + "prepare_max_ms": 10.85648185107857, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.922527313232422, + 18.899232864379883, + 18.88604736328125, + 18.904191970825195, + 18.860639572143555, + 18.850208282470703, + 18.887359619140625 + ], + "critical_prepare_samples_ms": [ + 8.449674933217466, + 7.978446898050606, + 8.195264963433146, + 8.395488141104579, + 8.383898064494133, + 8.15566093660891, + 8.260390954092145, + 10.85648185107857 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "contiguous": { + "step_ms": 18.96121597290039, + "prepare_ms": 6.907838920596987, + "one_use_ms": 25.869054893497378, + "prepare_min_ms": 6.844045943580568, + "prepare_max_ms": 7.187637034803629, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.724041124260225e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.511801279303253e-05, + "d_pool_score": 5.232610864426897e-05, + "d_ape": 1.4967124181438183e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 19.019712448120117, + 18.97760009765625, + 18.96121597290039, + 18.9564151763916, + 18.95644760131836, + 18.9434871673584, + 19.144479751586914 + ], + "critical_prepare_samples_ms": [ + 7.187637034803629, + 6.97691401001066, + 6.844045943580568, + 6.993726012296975, + 6.876001018099487, + 6.887514842674136, + 6.885394104756415, + 6.928162998519838 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "zigzag": { + "step_ms": 22.557504653930664, + "prepare_ms": 46.86611460056156, + "one_use_ms": 69.42361925449222, + "prepare_min_ms": 46.30116908811033, + "prepare_max_ms": 167.44421899784356, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8637480027866625e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.271880161331187e-05, + "d_pool_score": 5.610531234471807e-05, + "d_ape": 1.5175327931420397e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 22.557504653930664, + 22.60767936706543, + 22.548383712768555, + 22.567264556884766, + 22.551103591918945, + 22.51372718811035, + 22.592575073242188 + ], + "critical_prepare_samples_ms": [ + 47.66656889114529, + 46.35808803141117, + 46.458124998025596, + 167.44421899784356, + 47.36804007552564, + 47.025946085341275, + 46.30116908811033, + 46.706283115781844 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "planner": { + "step_ms": 18.94291114807129, + "prepare_ms": 8.967155939899385, + "one_use_ms": 27.910067087970674, + "prepare_min_ms": 8.858141023665667, + "prepare_max_ms": 14.381637098267674, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.590593616004352e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.534328689387801e-05, + "d_pool_score": 5.3623912772759155e-05, + "d_ape": 1.7238388124362549e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.938079833984375, + 18.94291114807129, + 18.93289566040039, + 18.952415466308594, + 18.924928665161133, + 18.946847915649414, + 19.089344024658203 + ], + "critical_prepare_samples_ms": [ + 9.043016005307436, + 8.858141023665667, + 8.99953895714134, + 8.93477292265743, + 9.00569697842002, + 8.906646049581468, + 8.866342948749661, + 14.381637098267674 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + } + } + } + } + }, + "reduced_search_sensitivity": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "min_gain_ms": 2.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 2, + "check_only": false + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 2.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "cases": { + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "206ac833d00cc5bf1c2437b126f72c4d797ec86a89c522f164b5ef808f1fd8d6", + "policies": { + "legacy-contiguous": { + "step_ms": 32.31577682495117, + "prepare_ms": 9.774404112249613, + "one_use_ms": 42.090180937200785, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 32.31577682495117, + 32.28287887573242, + 32.33244705200195, + 32.309471130371094, + 32.30284881591797, + 32.32038497924805, + 32.39052963256836 + ], + "critical_prepare_samples_ms": [ + 9.07967402599752, + 9.774404112249613, + 9.776547900401056 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "contiguous": { + "step_ms": 32.3752326965332, + "prepare_ms": 8.127258974127471, + "one_use_ms": 40.502491670660675, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015114969438123802, + "dq": 0.0004578816300774803, + "dkv": 0.0004676909672103567, + "d_index_q": 0.00035461648640452504, + "d_weights": 0.0003343071935462977, + "d_pool_kv": 0.0005292565852777516, + "d_pool_score": 0.000537489115750937, + "d_ape": 4.008685712418047e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.35612869262695, + 32.37507247924805, + 32.387840270996094, + 32.435489654541016, + 32.35036849975586, + 32.3752326965332, + 32.56003189086914 + ], + "critical_prepare_samples_ms": [ + 7.735361927188933, + 8.529279031790793, + 8.127258974127471 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "zigzag": { + "step_ms": 37.885311126708984, + "prepare_ms": 45.99909298121929, + "one_use_ms": 83.88440410792828, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016234906071296098, + "dq": 0.0004906073228918477, + "dkv": 0.000485240208978095, + "d_index_q": 0.0003558193972514481, + "d_weights": 0.00032124146055954113, + "d_pool_kv": 0.0005457322304231309, + "d_pool_score": 0.0005399599227922468, + "d_ape": 4.763308121602455e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 37.87958526611328, + 37.90604782104492, + 37.908992767333984, + 38.01612854003906, + 37.85958480834961, + 37.88185501098633, + 37.885311126708984 + ], + "critical_prepare_samples_ms": [ + 47.39365400746465, + 45.99909298121929, + 45.87791208177805 + ], + "candidate_workspace_bytes_per_rank": [ + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344 + ] + }, + "planner": { + "step_ms": 32.36540985107422, + "prepare_ms": 10.196399874985218, + "one_use_ms": 42.56180972605944, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00150859254036341, + "dq": 0.0004572904703051933, + "dkv": 0.0004729963619223918, + "d_index_q": 0.0003518774483353445, + "d_weights": 0.0003240170008018262, + "d_pool_kv": 0.0005420697840284305, + "d_pool_score": 0.0005310764478034278, + "d_ape": 4.198265664849056e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.37472152709961, + 32.34428787231445, + 32.33184051513672, + 32.42252731323242, + 32.364959716796875, + 32.383583068847656, + 32.36540985107422 + ], + "critical_prepare_samples_ms": [ + 10.196399874985218, + 10.216351016424596, + 10.10116096585989 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "14d573aed02d0fbe56347afe73070308fa4b52a1f50a5f39099069f9a822f622", + "policies": { + "legacy-contiguous": { + "step_ms": 33.04886245727539, + "prepare_ms": 8.66100809071213, + "one_use_ms": 41.70987054798752, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 33.05459213256836, + 33.069950103759766, + 33.01395034790039, + 33.04579162597656, + 33.04886245727539, + 33.069664001464844, + 33.03974533081055 + ], + "critical_prepare_samples_ms": [ + 9.070100029930472, + 8.66100809071213, + 8.639303967356682 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "contiguous": { + "step_ms": 33.116798400878906, + "prepare_ms": 7.43433297611773, + "one_use_ms": 40.551131376996636, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002093747276231706, + "dq": 0.0006334083022316513, + "dkv": 0.0005305985785479698, + "d_index_q": 0.00035164505841025513, + "d_weights": 0.0003191535424605087, + "d_pool_kv": 0.000567973707910079, + "d_pool_score": 0.0005569264865225348, + "d_ape": 6.866102141759954e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 33.125152587890625, + 33.091617584228516, + 33.11123275756836, + 33.104000091552734, + 33.144351959228516, + 33.116798400878906, + 33.14662551879883 + ], + "critical_prepare_samples_ms": [ + 7.8012129524722695, + 7.354254019446671, + 7.43433297611773 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "zigzag": { + "step_ms": 38.7558708190918, + "prepare_ms": 45.50877504516393, + "one_use_ms": 84.26464586425573, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.002181374361486575, + "dq": 0.0006867861882885665, + "dkv": 0.0006765172784291122, + "d_index_q": 0.00041965392520146793, + "d_weights": 0.0003215599066397513, + "d_pool_kv": 0.0005907853387548043, + "d_pool_score": 0.0005889070011877098, + "d_ape": 7.251414744984451e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.72137451171875, + 38.7558708190918, + 38.76057434082031, + 38.8023681640625, + 38.688480377197266, + 38.73535919189453, + 38.8474235534668 + ], + "critical_prepare_samples_ms": [ + 47.14965296443552, + 45.50877504516393, + 44.8957160115242 + ], + "candidate_workspace_bytes_per_rank": [ + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104 + ] + }, + "planner": { + "step_ms": 33.0863037109375, + "prepare_ms": 9.439464891329408, + "one_use_ms": 42.52576860226691, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0020996398701580354, + "dq": 0.0006346109709168223, + "dkv": 0.0005287856528683287, + "d_index_q": 0.0003540673085342941, + "d_weights": 0.00031974887292759063, + "d_pool_kv": 0.0005708471487096938, + "d_pool_score": 0.0005611318284191702, + "d_ape": 7.035264280944537e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 33.08342361450195, + 33.0863037109375, + 33.07936096191406, + 33.14601516723633, + 33.088958740234375, + 33.13372802734375, + 33.07321548461914 + ], + "critical_prepare_samples_ms": [ + 9.505677036941051, + 9.329304099082947, + 9.439464891329408 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "c37fdc7491d81ef27e39eeb993950c31376deee5b81033de6c80951402a75014", + "policies": { + "legacy-contiguous": { + "step_ms": 9.961440086364746, + "prepare_ms": 4.771576030179858, + "one_use_ms": 14.733016116544604, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.95577621459961, + 9.969663619995117, + 9.967071533203125, + 10.025728225708008, + 9.94979190826416, + 9.959168434143066, + 9.961440086364746 + ], + "critical_prepare_samples_ms": [ + 4.771576030179858, + 7.079613162204623, + 4.413567017763853 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "contiguous": { + "step_ms": 10.006815910339355, + "prepare_ms": 3.7964109797030687, + "one_use_ms": 13.803226890042424, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.9978755880494213e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.444174851955467e-05, + "d_pool_score": 2.8797816112323566e-05, + "d_ape": 1.2955959260855573e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.015935897827148, + 10.006815910339355, + 9.975359916687012, + 10.021504402160645, + 9.999263763427734, + 10.000639915466309, + 10.02940845489502 + ], + "critical_prepare_samples_ms": [ + 3.7964109797030687, + 3.9685630472376943, + 3.6812990438193083 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "zigzag": { + "step_ms": 11.198399543762207, + "prepare_ms": 25.439652032218874, + "one_use_ms": 36.63805157598108, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.1161980846562704e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.0865890337618216e-05, + "d_pool_score": 4.153862716484465e-05, + "d_ape": 1.7307645934332891e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.415776252746582, + 11.247648239135742, + 11.255423545837402, + 11.198399543762207, + 11.182592391967773, + 11.181695938110352, + 11.188159942626953 + ], + "critical_prepare_samples_ms": [ + 25.439652032218874, + 26.81133185978979, + 23.72458192985505 + ], + "candidate_workspace_bytes_per_rank": [ + 1855488, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536 + ] + }, + "planner": { + "step_ms": 10.006272315979004, + "prepare_ms": 4.958153003826737, + "one_use_ms": 14.964425319805741, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.4528841297910174e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 3.920732113107061e-05, + "d_pool_score": 3.440938115604093e-05, + "d_ape": 1.4369673349993467e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.022047996520996, + 9.988991737365723, + 10.006272315979004, + 10.018207550048828, + 9.986016273498535, + 10.019583702087402, + 9.992416381835938 + ], + "critical_prepare_samples_ms": [ + 4.973354982212186, + 4.958153003826737, + 4.738058079965413 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + } + } + }, + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "9d588616976075235c01154d101322f7df108c26a45382507bffa41dc275f3c4", + "policies": { + "legacy-contiguous": { + "step_ms": 47.360191345214844, + "prepare_ms": 0.7180591346696019, + "one_use_ms": 48.078250479884446, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 47.293216705322266, + 47.327392578125, + 47.29286575317383, + 47.53891372680664, + 47.360191345214844, + 47.48175811767578, + 47.39324951171875 + ], + "critical_prepare_samples_ms": [ + 0.9503101464360952, + 0.7180591346696019, + 0.7097340421751142 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "contiguous": { + "step_ms": 47.65436935424805, + "prepare_ms": 0.7628588937222958, + "one_use_ms": 48.41722824797034, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015751207193441088, + "dq": 0.0005783199131152748, + "dkv": 0.0006685131626782165, + "d_index_q": 0.0007961003786003505, + "d_weights": 0.0006623062461361172, + "d_pool_kv": 0.0008504045561905792, + "d_pool_score": 0.0009088681729959377, + "d_ape": 2.734829661652657e-05 + }, + "max_loss_abs_diff": 6.111804395914078e-10, + "critical_step_samples_ms": [ + 47.42502212524414, + 47.697471618652344, + 47.871456146240234, + 47.40972900390625, + 47.312225341796875, + 47.71052932739258, + 47.65436935424805 + ], + "critical_prepare_samples_ms": [ + 0.8085910230875015, + 0.7628588937222958, + 0.7170919561758637 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "zigzag": { + "step_ms": 46.215328216552734, + "prepare_ms": 0.9315180359408259, + "one_use_ms": 47.14684625249356, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015815559567693414, + "dq": 0.0005808441534881571, + "dkv": 0.0005049512279862388, + "d_index_q": 0.0006542155084291897, + "d_weights": 0.0005079701559067017, + "d_pool_kv": 0.0007064207861540901, + "d_pool_score": 0.0007066278713246955, + "d_ape": 2.418222008041728e-05 + }, + "max_loss_abs_diff": 2.6193447411060333e-10, + "critical_step_samples_ms": [ + 45.99225616455078, + 46.215328216552734, + 46.77628707885742, + 46.08924865722656, + 46.17289733886719, + 46.40963363647461, + 46.41756820678711 + ], + "critical_prepare_samples_ms": [ + 2.246724907308817, + 0.9315180359408259, + 0.899365870282054 + ], + "candidate_workspace_bytes_per_rank": [ + 536861644, + 536828104, + 536838528, + 536812296, + 536778684, + 536816940, + 536795936, + 536828336 + ] + }, + "planner": { + "step_ms": 45.837215423583984, + "prepare_ms": 2.398140146397054, + "one_use_ms": 48.23535556998104, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0015835534868685192, + "dq": 0.0005814588792502117, + "dkv": 0.0006013543899328077, + "d_index_q": 0.0005534685427820214, + "d_weights": 0.000646131642927837, + "d_pool_kv": 0.0007677387392274655, + "d_pool_score": 0.0007507812475082407, + "d_ape": 2.2943076205792098e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 45.837215423583984, + 45.95561599731445, + 45.775390625, + 45.85027313232422, + 45.8144645690918, + 46.047359466552734, + 45.776161193847656 + ], + "critical_prepare_samples_ms": [ + 2.542854053899646, + 2.398140146397054, + 2.3720700992271304 + ], + "candidate_workspace_bytes_per_rank": [ + 536828104, + 536812296, + 536816940, + 536864252, + 536795936, + 536778684, + 536838528, + 536861644 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "7ac651892601ff79612fe7b78ef4aab5b87db6c5202d8a79ea3ebc2ca232cb13", + "policies": { + "legacy-contiguous": { + "step_ms": 9.289216041564941, + "prepare_ms": 7.89993314538151, + "one_use_ms": 17.18914918694645, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.295328140258789, + 9.296895980834961, + 9.288703918457031, + 9.29372787475586, + 9.246944427490234, + 9.253151893615723, + 9.289216041564941 + ], + "critical_prepare_samples_ms": [ + 8.253396954387426, + 7.89993314538151, + 7.871040143072605 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "contiguous": { + "step_ms": 9.327391624450684, + "prepare_ms": 6.7193860886618495, + "one_use_ms": 16.046777713112533, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.093166359762604e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.9217821848352475e-05, + "d_pool_score": 4.03182609837131e-05, + "d_ape": 2.242167264657663e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.32156753540039, + 9.48083209991455, + 9.290656089782715, + 9.440832138061523, + 9.508383750915527, + 9.291359901428223, + 9.327391624450684 + ], + "critical_prepare_samples_ms": [ + 6.848165998235345, + 6.7193860886618495, + 6.64277991745621 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "zigzag": { + "step_ms": 11.107904434204102, + "prepare_ms": 50.2719139913097, + "one_use_ms": 61.379818425513804, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.234117372908357e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.218470753097149e-05, + "d_pool_score": 3.817734976583123e-05, + "d_ape": 1.9858886890377225e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.10422420501709, + 11.310079574584961, + 11.14742374420166, + 11.099967956542969, + 11.13548755645752, + 11.08351993560791, + 11.107904434204102 + ], + "critical_prepare_samples_ms": [ + 50.2719139913097, + 45.780996908433735, + 164.21099100261927 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "planner": { + "step_ms": 9.316512107849121, + "prepare_ms": 8.645661058835685, + "one_use_ms": 17.962173166684806, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8816705153238948e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.762204604666035e-05, + "d_pool_score": 4.186287177697969e-05, + "d_ape": 2.1236779901697946e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.291647911071777, + 9.311360359191895, + 9.318400382995605, + 9.316512107849121, + 9.325599670410156, + 10.244704246520996, + 9.314784049987793 + ], + "critical_prepare_samples_ms": [ + 8.862095070071518, + 8.567675948143005, + 8.645661058835685 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "306c17e1b10c9055bca24d23542d6e0ae72ba86f18743e7c1db0e21333ee9e34", + "policies": { + "legacy-contiguous": { + "step_ms": 98.6659164428711, + "prepare_ms": 11.852037045173347, + "one_use_ms": 110.51795348804444, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 98.6659164428711, + 98.62493133544922, + 98.76188659667969, + 98.96463775634766, + 98.2577896118164, + 98.4927978515625, + 98.73375701904297 + ], + "critical_prepare_samples_ms": [ + 11.852037045173347, + 11.283355066552758, + 12.048232019878924 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "contiguous": { + "step_ms": 99.30899047851562, + "prepare_ms": 10.390229988843203, + "one_use_ms": 109.69922046735883, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010573826208271084, + "dq": 0.0003592223006292566, + "dkv": 0.0005484814447866894, + "d_index_q": 0.0006455438914783363, + "d_weights": 0.0005120783559164215, + "d_pool_kv": 0.0007752975553353049, + "d_pool_score": 0.0007896526582265112, + "d_ape": 6.200486588157099e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 99.23235321044922, + 98.54048156738281, + 99.35852813720703, + 99.44525146484375, + 99.30899047851562, + 98.63680267333984, + 100.43740844726562 + ], + "critical_prepare_samples_ms": [ + 10.390229988843203, + 9.821086074225605, + 10.521577904000878 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "zigzag": { + "step_ms": 115.28396606445312, + "prepare_ms": 144.87404597457498, + "one_use_ms": 260.1580120390281, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0010834614507619967, + "dq": 0.0003684552651301375, + "dkv": 0.00039418431972343653, + "d_index_q": 0.0003783078523965674, + "d_weights": 0.0003770943893548319, + "d_pool_kv": 0.0006627945441974228, + "d_pool_score": 0.0006772935978763263, + "d_ape": 5.986287096458122e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 115.21177673339844, + 115.45823669433594, + 115.331298828125, + 115.17340850830078, + 115.28396606445312, + 115.195068359375, + 115.5271987915039 + ], + "critical_prepare_samples_ms": [ + 50.712813972495496, + 144.87404597457498, + 168.62917703110725 + ], + "candidate_workspace_bytes_per_rank": [ + 1073708016, + 1073692980, + 1073648060, + 536804832, + 536760312, + 536814420, + 536809884, + 536864760 + ] + }, + "planner": { + "step_ms": 92.93939208984375, + "prepare_ms": 106.1690280912444, + "one_use_ms": 199.10842018108815, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0011127535868433215, + "dq": 0.0003930554637779379, + "dkv": 0.0005549246019563618, + "d_index_q": 0.000646853644347867, + "d_weights": 0.0005176427284762521, + "d_pool_kv": 0.0007843419483566719, + "d_pool_score": 0.0007981760582909821, + "d_ape": 6.011172940416037e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 92.96339416503906, + 92.93939208984375, + 93.06483459472656, + 92.72089385986328, + 92.65577697753906, + 93.29020690917969, + 92.58598327636719 + ], + "critical_prepare_samples_ms": [ + 106.1690280912444, + 122.90593807119876, + 22.352106869220734 + ], + "candidate_workspace_bytes_per_rank": [ + 536852000, + 536840844, + 536869000, + 536794800, + 1073680320, + 536834316, + 536856816, + 1073601720 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "c6cfc83abb6f9da3890128a6674d1a2d33a3dd4f89b4d7f6d15f4fa70bd3d5b8", + "policies": { + "legacy-contiguous": { + "step_ms": 83.52304077148438, + "prepare_ms": 10.926121030934155, + "one_use_ms": 94.44916180241853, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 83.31452941894531, + 84.28518676757812, + 83.41043090820312, + 83.52304077148438, + 83.24291229248047, + 83.67024230957031, + 83.7089614868164 + ], + "critical_prepare_samples_ms": [ + 11.107583995908499, + 10.926121030934155, + 10.472901980392635 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "contiguous": { + "step_ms": 83.8061752319336, + "prepare_ms": 10.013072984293103, + "one_use_ms": 93.8192482162267, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017285115614443013, + "dq": 0.0005245019570177233, + "dkv": 0.0008850784611107852, + "d_index_q": 0.0003679564403986905, + "d_weights": 0.00036400805474966116, + "d_pool_kv": 0.001373117872850444, + "d_pool_score": 0.001360731586663582, + "d_ape": 9.068534898484564e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 84.08528137207031, + 83.67862701416016, + 83.81353759765625, + 83.15155029296875, + 84.11788940429688, + 83.8061752319336, + 83.74156951904297 + ], + "critical_prepare_samples_ms": [ + 11.039006989449263, + 10.013072984293103, + 9.5121389022097 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "zigzag": { + "step_ms": 96.0337905883789, + "prepare_ms": 154.22721300274134, + "one_use_ms": 250.26100359112024, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017382082680359764, + "dq": 0.0005269916395286206, + "dkv": 0.00045928158371192967, + "d_index_q": 0.0003679860460189847, + "d_weights": 0.0003632725669426103, + "d_pool_kv": 0.0006520830116070919, + "d_pool_score": 0.0006456380441297271, + "d_ape": 7.835311140252914e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-10, + "critical_step_samples_ms": [ + 96.42201232910156, + 95.8373794555664, + 96.0337905883789, + 95.5379867553711, + 95.66000366210938, + 96.18585968017578, + 96.0810546875 + ], + "critical_prepare_samples_ms": [ + 52.832534071058035, + 155.64053610432893, + 154.22721300274134 + ], + "candidate_workspace_bytes_per_rank": [ + 536828196, + 536788788, + 518462572, + 480072156, + 441681740, + 403291324, + 364900908, + 326510492 + ] + }, + "planner": { + "step_ms": 86.18553924560547, + "prepare_ms": 21.83594205416739, + "one_use_ms": 108.02148129977286, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.001745158118362825, + "dq": 0.0005293636644694899, + "dkv": 0.0005248456195568502, + "d_index_q": 0.0003697755327810351, + "d_weights": 0.00036312147739179905, + "d_pool_kv": 0.000647131001315703, + "d_pool_score": 0.0006558826907249498, + "d_ape": 7.933114227903847e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 86.18553924560547, + 86.66947174072266, + 86.13078308105469, + 86.79126739501953, + 85.99791717529297, + 86.30726623535156, + 86.09295654296875 + ], + "critical_prepare_samples_ms": [ + 21.673545124940574, + 130.55709702894092, + 21.83594205416739 + ], + "candidate_workspace_bytes_per_rank": [ + 536820936, + 536822596, + 536837028, + 536865908, + 101228656, + 536822596, + 536865908, + 536837028 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "466b346789a4d59098aad2ca2b04aca877953815c3163fc8f2c0f749a341ceae", + "policies": { + "legacy-contiguous": { + "step_ms": 21.694368362426758, + "prepare_ms": 5.549413966946304, + "one_use_ms": 27.24378232937306, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.685184478759766, + 21.670495986938477, + 21.69536018371582, + 21.73049545288086, + 21.663488388061523, + 21.694368362426758, + 21.732704162597656 + ], + "critical_prepare_samples_ms": [ + 4.9613380106166005, + 5.684494040906429, + 5.549413966946304 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "contiguous": { + "step_ms": 21.787263870239258, + "prepare_ms": 4.69140603672713, + "one_use_ms": 26.47866990696639, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2664848913808245e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.527269782608548e-05, + "d_pool_score": 5.23650389771624e-05, + "d_ape": 1.326061817639286e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.81724739074707, + 21.77782440185547, + 21.77788734436035, + 21.807296752929688, + 21.818464279174805, + 21.787263870239258, + 21.775104522705078 + ], + "critical_prepare_samples_ms": [ + 4.69140603672713, + 4.833335056900978, + 4.65017999522388 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "zigzag": { + "step_ms": 24.22969627380371, + "prepare_ms": 25.856652995571494, + "one_use_ms": 50.086349269375205, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.4942269741698106e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.163642701493704e-05, + "d_pool_score": 6.043722187737683e-05, + "d_ape": 1.4394808166454493e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 24.23721694946289, + 24.284608840942383, + 24.22969627380371, + 24.18796730041504, + 24.21548843383789, + 24.231679916381836, + 24.21513557434082 + ], + "critical_prepare_samples_ms": [ + 26.876677060499787, + 25.856652995571494, + 24.792716838419437 + ], + "candidate_workspace_bytes_per_rank": [ + 7901184, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376 + ] + }, + "planner": { + "step_ms": 21.749343872070312, + "prepare_ms": 5.762123968452215, + "one_use_ms": 27.511467840522528, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.8881779744528944e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.6376933606401454e-05, + "d_pool_score": 6.304859853384069e-05, + "d_ape": 1.3838066052628374e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.749055862426758, + 21.768640518188477, + 21.900991439819336, + 21.749343872070312, + 21.737119674682617, + 21.747936248779297, + 21.77097511291504 + ], + "critical_prepare_samples_ms": [ + 5.77679299749434, + 5.70077495649457, + 5.762123968452215 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "f85bfbf7322809f6c47450cfbe646a38e29419a8c8cc8a0e1f14662b5242ecc1", + "policies": { + "legacy-contiguous": { + "step_ms": 115.33245086669922, + "prepare_ms": 1.1144890449941158, + "one_use_ms": 116.44693991169333, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 115.0580825805664, + 115.42076873779297, + 115.65145874023438, + 115.51197052001953, + 114.57266998291016, + 114.61859130859375, + 115.33245086669922 + ], + "critical_prepare_samples_ms": [ + 1.432059914804995, + 1.1144890449941158, + 1.092861988581717 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "contiguous": { + "step_ms": 115.45356750488281, + "prepare_ms": 1.1493739439174533, + "one_use_ms": 116.60294144880027, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014814819659666792, + "dq": 0.0005025477042892497, + "dkv": 0.0010280580317239322, + "d_index_q": 0.0007709961298283565, + "d_weights": 0.0006240027811370497, + "d_pool_kv": 0.0013297219628949863, + "d_pool_score": 0.0013706802273474096, + "d_ape": 2.047579904414999e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 115.91276550292969, + 114.74950408935547, + 115.78205108642578, + 115.53488159179688, + 115.12838745117188, + 115.45356750488281, + 114.86653137207031 + ], + "critical_prepare_samples_ms": [ + 1.2612229911610484, + 1.1493739439174533, + 1.085231895558536 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "zigzag": { + "step_ms": 105.22351837158203, + "prepare_ms": 1.1590360663831234, + "one_use_ms": 106.38255443796515, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001487600849902013, + "dq": 0.000504489249271614, + "dkv": 0.0006238415299610192, + "d_index_q": 0.0007130704206747019, + "d_weights": 0.0005857291818919476, + "d_pool_kv": 0.0009670844224943009, + "d_pool_score": 0.00099420840848902, + "d_ape": 2.209921814997075e-05 + }, + "max_loss_abs_diff": 3.2014213502407074e-10, + "critical_step_samples_ms": [ + 105.09027099609375, + 105.3821792602539, + 104.91484832763672, + 105.22351837158203, + 105.06269073486328, + 105.24441528320312, + 105.65392303466797 + ], + "critical_prepare_samples_ms": [ + 4.508284968324006, + 1.1590360663831234, + 1.105463015846908 + ], + "candidate_workspace_bytes_per_rank": [ + 1073712188, + 1073660864, + 1073659492, + 1073631792, + 1073736736, + 1073737680, + 1073713936, + 1073713120 + ] + }, + "planner": { + "step_ms": 104.40592193603516, + "prepare_ms": 2.6892030145972967, + "one_use_ms": 107.09512495063245, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014828593003667822, + "dq": 0.000503123061268814, + "dkv": 0.0010299556200644897, + "d_index_q": 0.0007715033409916601, + "d_weights": 0.0006256374689194227, + "d_pool_kv": 0.001331098022685323, + "d_pool_score": 0.001368611082483086, + "d_ape": 2.171553401170655e-05 + }, + "max_loss_abs_diff": 3.2014213502407074e-10, + "critical_step_samples_ms": [ + 105.03376007080078, + 104.3993911743164, + 104.48409271240234, + 104.16764831542969, + 103.87769317626953, + 104.97811126708984, + 104.40592193603516 + ], + "critical_prepare_samples_ms": [ + 2.7697039768099785, + 2.6892030145972967, + 2.602526918053627 + ], + "candidate_workspace_bytes_per_rank": [ + 1073660864, + 1073631792, + 1073737680, + 1073736096, + 1073713936, + 1073736736, + 1073659492, + 1073712188 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "85c6bf57bb0995c8feb5c0df0d3db82adea7e3a606ccee9a2dfa44315561c946", + "policies": { + "legacy-contiguous": { + "step_ms": 18.90492820739746, + "prepare_ms": 9.129596059210598, + "one_use_ms": 28.03452426660806, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.90729522705078, + 18.882495880126953, + 19.03046417236328, + 18.907712936401367, + 18.893888473510742, + 18.89472007751465, + 18.90492820739746 + ], + "critical_prepare_samples_ms": [ + 8.733341936022043, + 9.129596059210598, + 100.35485692787915 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "contiguous": { + "step_ms": 18.984800338745117, + "prepare_ms": 7.653723005205393, + "one_use_ms": 26.63852334395051, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.249396326647605e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.6476196629912486e-05, + "d_pool_score": 6.0312485136719826e-05, + "d_ape": 1.9333346650755677e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.984800338745117, + 18.974912643432617, + 18.95270347595215, + 18.997695922851562, + 19.007232666015625, + 18.9531192779541, + 19.015039443969727 + ], + "critical_prepare_samples_ms": [ + 7.653723005205393, + 7.782059954479337, + 6.843541981652379 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "zigzag": { + "step_ms": 22.576736450195312, + "prepare_ms": 48.61433804035187, + "one_use_ms": 71.19107449054718, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3626305216121236e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.584356380441e-05, + "d_pool_score": 6.326438372102144e-05, + "d_ape": 1.884969194270893e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 22.52841567993164, + 22.75379180908203, + 22.576736450195312, + 22.577600479125977, + 22.57868766784668, + 22.550016403198242, + 22.55062484741211 + ], + "critical_prepare_samples_ms": [ + 48.61433804035187, + 47.93463693931699, + 167.71281801629812 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "planner": { + "step_ms": 18.959936141967773, + "prepare_ms": 9.468792006373405, + "one_use_ms": 28.42872814834118, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.9131160333384314e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.272192949548379e-05, + "d_pool_score": 6.528122941093681e-05, + "d_ape": 1.9596341111569527e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.956384658813477, + 18.92902374267578, + 18.959936141967773, + 18.937664031982422, + 18.97270393371582, + 18.96233558654785, + 18.96553611755371 + ], + "critical_prepare_samples_ms": [ + 9.437776985578239, + 9.566316031850874, + 9.468792006373405 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + } + } + } + } + }, + "legacy_margin_sensitivity": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 5, + "min_gain_ms": null, + "min_chunk_tokens": 1024, + "check_only": false + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 20, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "cases": { + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "d5dfbe3522b83384879cbf63c9340cc656199e93e699dd17ec6c755058085782", + "policies": { + "legacy-contiguous": { + "step_ms": 32.31264114379883, + "prepare_ms": 11.450840043835342, + "one_use_ms": 43.76348118763417, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 32.31264114379883, + 32.298912048339844, + 32.30374526977539, + 32.318016052246094, + 32.354881286621094 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "contiguous": { + "step_ms": 32.34806442260742, + "prepare_ms": 9.241741965524852, + "one_use_ms": 41.589806388132274, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015055117580929998, + "dq": 0.0004562001787174165, + "dkv": 0.00046682488364094504, + "d_index_q": 0.0003528964133666067, + "d_weights": 0.0003143728165677277, + "d_pool_kv": 0.0005616682841830009, + "d_pool_score": 0.0005502770556613789, + "d_ape": 4.118661662693684e-06 + }, + "critical_step_samples_ms": [ + 32.39360046386719, + 32.34806442260742, + 32.347774505615234, + 32.392127990722656, + 32.3389778137207 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "zigzag": { + "step_ms": 37.92006301879883, + "prepare_ms": 57.76502494700253, + "one_use_ms": 95.68508796580136, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001615737541975861, + "dq": 0.0004889801391640974, + "dkv": 0.00048574739383770417, + "d_index_q": 0.00035745246808257353, + "d_weights": 0.0003359146161968018, + "d_pool_kv": 0.0005580949946886438, + "d_pool_score": 0.0005679364170603011, + "d_ape": 4.1200106000536755e-06 + }, + "critical_step_samples_ms": [ + 38.000545501708984, + 37.92006301879883, + 37.89564895629883, + 37.883583068847656, + 37.9365119934082 + ], + "candidate_workspace_bytes_per_rank": [ + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344 + ] + }, + "planner": { + "step_ms": 32.34601593017578, + "prepare_ms": 11.20570907369256, + "one_use_ms": 43.55172500386834, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015106313582280778, + "dq": 0.0004573933437522137, + "dkv": 0.00046884446588082903, + "d_index_q": 0.00035414067780251005, + "d_weights": 0.00031735270241838205, + "d_pool_kv": 0.0005302273051469876, + "d_pool_score": 0.0005465640751035636, + "d_ape": 4.408512538131001e-06 + }, + "critical_step_samples_ms": [ + 32.34092712402344, + 32.34601593017578, + 32.35830307006836, + 32.395103454589844, + 32.34467315673828 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "2975aafa9ca831aa160ee84b82aefad9c2bf3c9db7670afd63a68195a68cc635", + "policies": { + "legacy-contiguous": { + "step_ms": 33.041534423828125, + "prepare_ms": 11.169639183208346, + "one_use_ms": 44.21117360703647, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 33.05654525756836, + 33.0382080078125, + 33.05062484741211, + 33.041534423828125, + 33.029502868652344 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "contiguous": { + "step_ms": 33.08902359008789, + "prepare_ms": 9.206014103256166, + "one_use_ms": 42.29503769334406, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002097984063477685, + "dq": 0.0006349000123178144, + "dkv": 0.0005296393426551206, + "d_index_q": 0.00035507980865031745, + "d_weights": 0.000322679720692784, + "d_pool_kv": 0.0005795026037481414, + "d_pool_score": 0.0005673212325304447, + "d_ape": 6.963864564318172e-06 + }, + "critical_step_samples_ms": [ + 33.08902359008789, + 33.07833480834961, + 33.089473724365234, + 33.090240478515625, + 33.07475280761719 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "zigzag": { + "step_ms": 38.74492645263672, + "prepare_ms": 57.125494931824505, + "one_use_ms": 95.87042138446122, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0021380216992278336, + "dq": 0.0006444019165939065, + "dkv": 0.0005376332039364028, + "d_index_q": 0.000355143576767433, + "d_weights": 0.00032763168537997136, + "d_pool_kv": 0.0005620423395904252, + "d_pool_score": 0.0005485233599230848, + "d_ape": 6.923240736087287e-06 + }, + "critical_step_samples_ms": [ + 38.74143981933594, + 38.77017593383789, + 38.7586555480957, + 38.720672607421875, + 38.74492645263672 + ], + "candidate_workspace_bytes_per_rank": [ + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104 + ] + }, + "planner": { + "step_ms": 33.06643295288086, + "prepare_ms": 11.127481004223228, + "one_use_ms": 44.19391395710409, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002079553939769878, + "dq": 0.0006294535496018204, + "dkv": 0.0005275027195349258, + "d_index_q": 0.0003534628562349375, + "d_weights": 0.00032433872571991425, + "d_pool_kv": 0.0005567261666655464, + "d_pool_score": 0.0005407815979470878, + "d_ape": 7.193694112788869e-06 + }, + "critical_step_samples_ms": [ + 33.06643295288086, + 33.075233459472656, + 33.07798385620117, + 33.06131362915039, + 33.062496185302734 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "ffabaa10208a2f42e4baa2ad2d0f1cd618ef64f9ee8bcb4f9af41228a94be166", + "policies": { + "legacy-contiguous": { + "step_ms": 9.972160339355469, + "prepare_ms": 7.681053131818771, + "one_use_ms": 17.65321347117424, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 9.942336082458496, + 9.972352027893066, + 9.972895622253418, + 9.938719749450684, + 9.972160339355469 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "contiguous": { + "step_ms": 10.000096321105957, + "prepare_ms": 4.725359962321818, + "one_use_ms": 14.725456283427775, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.593801996048873e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.7593107187593284e-05, + "d_pool_score": 5.0368489828505864e-05, + "d_ape": 1.729257008310043e-06 + }, + "critical_step_samples_ms": [ + 9.989888191223145, + 10.002559661865234, + 9.998368263244629, + 10.021471977233887, + 10.000096321105957 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "zigzag": { + "step_ms": 11.208576202392578, + "prepare_ms": 28.68070499971509, + "one_use_ms": 39.88928120210767, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.527012691451957e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.41627835231595e-05, + "d_pool_score": 6.0764600335971727e-05, + "d_ape": 1.969570217598954e-06 + }, + "critical_step_samples_ms": [ + 11.20035171508789, + 11.220447540283203, + 11.208576202392578, + 11.198304176330566, + 11.610400199890137 + ], + "candidate_workspace_bytes_per_rank": [ + 1855488, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536 + ] + }, + "planner": { + "step_ms": 9.991231918334961, + "prepare_ms": 5.699237925000489, + "one_use_ms": 15.69046984333545, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.603275222450197e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.436856634563869e-05, + "d_pool_score": 5.375680093457066e-05, + "d_ape": 1.992918705712155e-06 + }, + "critical_step_samples_ms": [ + 9.991231918334961, + 10.003168106079102, + 10.058303833007812, + 9.991135597229004, + 9.991168022155762 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + } + } + }, + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "d154370b3567b56ba5abef41ff1d72e4b4dbc22ab2453867c16a07754cba7353", + "policies": { + "legacy-contiguous": { + "step_ms": 47.74051284790039, + "prepare_ms": 6.36032095644623, + "one_use_ms": 54.10083380434662, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 47.74051284790039, + 47.38595199584961, + 47.4453125, + 47.911231994628906, + 47.81068801879883 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "contiguous": { + "step_ms": 47.775840759277344, + "prepare_ms": 2.173455082811415, + "one_use_ms": 49.94929584208876, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015778303977359808, + "dq": 0.000579295779912375, + "dkv": 0.0006681889896006872, + "d_index_q": 0.0007045097134893591, + "d_weights": 0.0006651717024489214, + "d_pool_kv": 0.0008470738686227776, + "d_pool_score": 0.0008863334597355397, + "d_ape": 2.6518480232396688e-05 + }, + "critical_step_samples_ms": [ + 47.775840759277344, + 47.55440139770508, + 48.050174713134766, + 47.42118453979492, + 48.06953430175781 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "zigzag": { + "step_ms": 46.505985260009766, + "prepare_ms": 2.9064940754324198, + "one_use_ms": 49.412479335442185, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015801558044298372, + "dq": 0.0005802450594053097, + "dkv": 0.0006717745484660001, + "d_index_q": 0.0007960406948691224, + "d_weights": 0.0006640151282144832, + "d_pool_kv": 0.0008450890931264659, + "d_pool_score": 0.0008844659218940599, + "d_ape": 2.4965860553199313e-05 + }, + "critical_step_samples_ms": [ + 46.105472564697266, + 46.51551818847656, + 46.505985260009766, + 46.52262496948242, + 46.18937683105469 + ], + "candidate_workspace_bytes_per_rank": [ + 536861644, + 536828104, + 536838528, + 536812296, + 536778684, + 536816940, + 536795936, + 536828336 + ] + }, + "planner": { + "step_ms": 47.641632080078125, + "prepare_ms": 6.449845037423074, + "one_use_ms": 54.0914771175012, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015794720823890324, + "dq": 0.000579758894437607, + "dkv": 0.0006678818260938846, + "d_index_q": 0.0007048063939166791, + "d_weights": 0.0006638073325243364, + "d_pool_kv": 0.0008637417489227197, + "d_pool_score": 0.0008861272240136442, + "d_ape": 2.7367223437852473e-05 + }, + "critical_step_samples_ms": [ + 47.46495819091797, + 47.86592102050781, + 47.32950210571289, + 47.87481689453125, + 47.641632080078125 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "796f64c340be56e9f3f4ab48e99daa85af564b2a885b5d7b0efdcf9ee0b57af0", + "policies": { + "legacy-contiguous": { + "step_ms": 9.263520240783691, + "prepare_ms": 11.319936951622367, + "one_use_ms": 20.58345719240606, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 9.272000312805176, + 9.263520240783691, + 9.285951614379883, + 9.262751579284668, + 9.24227237701416 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "contiguous": { + "step_ms": 9.281855583190918, + "prepare_ms": 8.454012917354703, + "one_use_ms": 17.73586850054562, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.268005405014033e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.299814918031266e-05, + "d_pool_score": 3.7219175723565274e-05, + "d_ape": 1.958796016299265e-06 + }, + "critical_step_samples_ms": [ + 9.310367584228516, + 9.270112037658691, + 9.281855583190918, + 9.308287620544434, + 9.251744270324707 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "zigzag": { + "step_ms": 11.09340763092041, + "prepare_ms": 53.628099034540355, + "one_use_ms": 64.72150666546077, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7903255632299044e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.328703291192055e-05, + "d_pool_score": 3.499598071138621e-05, + "d_ape": 1.9553001191787064e-06 + }, + "critical_step_samples_ms": [ + 11.090751647949219, + 11.105728149414062, + 11.078559875488281, + 11.09340763092041, + 11.114175796508789 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "planner": { + "step_ms": 9.327072143554688, + "prepare_ms": 10.402858024463058, + "one_use_ms": 19.729930168017745, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.053952717562547e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.891765909667368e-05, + "d_pool_score": 3.83438621871523e-05, + "d_ape": 1.860085527233789e-06 + }, + "critical_step_samples_ms": [ + 9.677375793457031, + 9.265312194824219, + 9.342687606811523, + 9.327072143554688, + 9.311936378479004 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "2edb473a627497dbeee590e633990317905ccda3234a5ed584397d300307f688", + "policies": { + "legacy-contiguous": { + "step_ms": 99.59091186523438, + "prepare_ms": 15.09766501840204, + "one_use_ms": 114.68857688363642, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 98.39814758300781, + 99.7735366821289, + 100.0940170288086, + 99.59091186523438, + 98.48457336425781 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "contiguous": { + "step_ms": 99.29817962646484, + "prepare_ms": 15.911947004497051, + "one_use_ms": 115.2101266309619, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.001110904233409444, + "dq": 0.00039648636058058897, + "dkv": 0.0003898854111343413, + "d_index_q": 0.0005600748026567583, + "d_weights": 0.000507263174267189, + "d_pool_kv": 0.00066199529419305, + "d_pool_score": 0.000678040642214578, + "d_ape": 5.9787640002907704e-06 + }, + "critical_step_samples_ms": [ + 98.9152603149414, + 99.38972473144531, + 99.86335754394531, + 98.69843292236328, + 99.29817962646484 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "zigzag": { + "step_ms": 115.35395050048828, + "prepare_ms": 69.53235797118396, + "one_use_ms": 184.88630847167224, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0010856430143221802, + "dq": 0.00036992129334733367, + "dkv": 0.0004972554645909258, + "d_index_q": 0.0005008559419775389, + "d_weights": 0.0004722618115882083, + "d_pool_kv": 0.000655208868851891, + "d_pool_score": 0.000654726690762559, + "d_ape": 5.867056633821389e-06 + }, + "critical_step_samples_ms": [ + 115.24447631835938, + 115.35395050048828, + 115.61705780029297, + 115.43267059326172, + 115.2938232421875 + ], + "candidate_workspace_bytes_per_rank": [ + 1073708016, + 1073692980, + 1073648060, + 536804832, + 536760312, + 536814420, + 536809884, + 536864760 + ] + }, + "planner": { + "step_ms": 99.38566589355469, + "prepare_ms": 12.992098927497864, + "one_use_ms": 112.37776482105255, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010568060715839816, + "dq": 0.00035934651144455286, + "dkv": 0.0004699011755092007, + "d_index_q": 0.0005465734905216791, + "d_weights": 0.0004997240432028573, + "d_pool_kv": 0.0006642917490044416, + "d_pool_score": 0.0006663724937641111, + "d_ape": 6.077476352327514e-06 + }, + "critical_step_samples_ms": [ + 98.25363159179688, + 99.62761688232422, + 99.38566589355469, + 99.48448181152344, + 98.54268646240234 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "49a55e79bd6c68bf131589a3e5cf5e565916facd5606d7dce7ce57c963f5b941", + "policies": { + "legacy-contiguous": { + "step_ms": 83.68646240234375, + "prepare_ms": 21.228261990472674, + "one_use_ms": 104.91472439281642, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 83.78521728515625, + 83.57373046875, + 83.68646240234375, + 83.2590103149414, + 84.24208068847656 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "contiguous": { + "step_ms": 83.48681640625, + "prepare_ms": 11.41117897350341, + "one_use_ms": 94.89799537975341, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017271400594987304, + "dq": 0.0005238692042413498, + "dkv": 0.0006757752642502718, + "d_index_q": 0.0003682615647156005, + "d_weights": 0.00036747700384791933, + "d_pool_kv": 0.0007684068273359155, + "d_pool_score": 0.0007428614103826032, + "d_ape": 8.689178552655459e-06 + }, + "critical_step_samples_ms": [ + 83.48681640625, + 83.37471771240234, + 84.20655822753906, + 83.2452163696289, + 84.60848236083984 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "zigzag": { + "step_ms": 95.8419189453125, + "prepare_ms": 70.37775486242026, + "one_use_ms": 166.21967380773276, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001741600433126006, + "dq": 0.0005280536977853182, + "dkv": 0.000694620863478123, + "d_index_q": 0.00039115890285579556, + "d_weights": 0.0003701005915049805, + "d_pool_kv": 0.0013274674931419044, + "d_pool_score": 0.0013120376158842905, + "d_ape": 9.092468895337202e-06 + }, + "critical_step_samples_ms": [ + 96.7213134765625, + 95.74854278564453, + 96.22220611572266, + 95.78582763671875, + 95.8419189453125 + ], + "candidate_workspace_bytes_per_rank": [ + 536828196, + 536788788, + 518462572, + 480072156, + 441681740, + 403291324, + 364900908, + 326510492 + ] + }, + "planner": { + "step_ms": 83.3333740234375, + "prepare_ms": 12.85100297536701, + "one_use_ms": 96.18437699880451, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017312793940356549, + "dq": 0.0005248991559417814, + "dkv": 0.0006219782668235454, + "d_index_q": 0.0003919900749740805, + "d_weights": 0.0003715013408516693, + "d_pool_kv": 0.0006638570005121069, + "d_pool_score": 0.0006602028790867173, + "d_ape": 8.978271066130069e-06 + }, + "critical_step_samples_ms": [ + 83.3333740234375, + 83.44710540771484, + 83.32115173339844, + 83.80867004394531, + 82.96355438232422 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "b47a41744461e8029c8fdc96e07b896746a31b8fd492a812fb4fd6051df3b277", + "policies": { + "legacy-contiguous": { + "step_ms": 21.743040084838867, + "prepare_ms": 7.12892203591764, + "one_use_ms": 28.871962120756507, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 21.842655181884766, + 21.743040084838867, + 21.711936950683594, + 21.78006362915039, + 21.708768844604492 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "contiguous": { + "step_ms": 21.762304306030273, + "prepare_ms": 4.859174019657075, + "one_use_ms": 26.62147832568735, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.1016467473129694e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.8276387051590115e-05, + "d_pool_score": 5.540698248431048e-05, + "d_ape": 1.4415619316682367e-06 + }, + "critical_step_samples_ms": [ + 21.784767150878906, + 21.762304306030273, + 21.736831665039062, + 21.763744354248047, + 21.739456176757812 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "zigzag": { + "step_ms": 24.220352172851562, + "prepare_ms": 30.287406989373267, + "one_use_ms": 54.50775916222483, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.408343901936022e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.233960704879021e-05, + "d_pool_score": 4.683946617373621e-05, + "d_ape": 1.4767297408883958e-06 + }, + "critical_step_samples_ms": [ + 24.222047805786133, + 24.204736709594727, + 24.24095916748047, + 24.193023681640625, + 24.220352172851562 + ], + "candidate_workspace_bytes_per_rank": [ + 7901184, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376 + ] + }, + "planner": { + "step_ms": 21.748031616210938, + "prepare_ms": 6.001263973303139, + "one_use_ms": 27.749295589514077, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.143517200502267e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.689493212930183e-05, + "d_pool_score": 5.4293748387891673e-05, + "d_ape": 1.290989423691988e-06 + }, + "critical_step_samples_ms": [ + 21.748031616210938, + 21.722944259643555, + 21.850048065185547, + 21.749183654785156, + 21.720064163208008 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "a0b6f467329398c2d88275ae6c33a1ac285cf76a5fdc45d076f2749ff36ae59e", + "policies": { + "legacy-contiguous": { + "step_ms": 115.62249755859375, + "prepare_ms": 6.42365706153214, + "one_use_ms": 122.04615462012589, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 116.22777557373047, + 115.62249755859375, + 116.10867309570312, + 115.35679626464844, + 114.52973175048828 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "contiguous": { + "step_ms": 115.4524154663086, + "prepare_ms": 2.2480469197034836, + "one_use_ms": 117.70046238601208, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014863891656528205, + "dq": 0.0005041585809728006, + "dkv": 0.0005403167928287846, + "d_index_q": 0.0007725595847374566, + "d_weights": 0.0006259349484387492, + "d_pool_kv": 0.0007197438097901232, + "d_pool_score": 0.000715737343426364, + "d_ape": 1.7515138877326042e-05 + }, + "critical_step_samples_ms": [ + 115.4524154663086, + 115.16633605957031, + 115.15238189697266, + 116.03148651123047, + 115.69798278808594 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "zigzag": { + "step_ms": 105.18627166748047, + "prepare_ms": 2.860206994228065, + "one_use_ms": 108.04647866170853, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014852931124954782, + "dq": 0.0005038136738057022, + "dkv": 0.0004547442223581367, + "d_index_q": 0.0004776906182052158, + "d_weights": 0.0004588937679118803, + "d_pool_kv": 0.0006862380126650793, + "d_pool_score": 0.0006758094131979612, + "d_ape": 1.6993433973488575e-05 + }, + "critical_step_samples_ms": [ + 105.17571258544922, + 105.18627166748047, + 105.00962829589844, + 105.96454620361328, + 105.43711853027344 + ], + "candidate_workspace_bytes_per_rank": [ + 1073712188, + 1073660864, + 1073659492, + 1073631792, + 1073736736, + 1073737680, + 1073713936, + 1073713120 + ] + }, + "planner": { + "step_ms": 115.74569702148438, + "prepare_ms": 2.8671040199697018, + "one_use_ms": 118.61280104145408, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014850858958218448, + "dq": 0.0005039087119093382, + "dkv": 0.0006199094601327424, + "d_index_q": 0.0007125454770317754, + "d_weights": 0.0005555715209788749, + "d_pool_kv": 0.0009620010607453248, + "d_pool_score": 0.0009848719645593007, + "d_ape": 1.8798434764171715e-05 + }, + "critical_step_samples_ms": [ + 115.16841888427734, + 115.74569702148438, + 116.19100952148438, + 116.41241455078125, + 114.95225524902344 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "cca2fdad0f4585eca1da043c29c9689115a510d9af63c2079fd2f1abc9c36c52", + "policies": { + "legacy-contiguous": { + "step_ms": 18.9467830657959, + "prepare_ms": 12.605302035808563, + "one_use_ms": 31.55208510160446, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 19.008895874023438, + 18.9467830657959, + 18.906911849975586, + 18.95804786682129, + 18.89014434814453 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "contiguous": { + "step_ms": 18.971071243286133, + "prepare_ms": 8.725322084501386, + "one_use_ms": 27.69639332778752, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.668157086472552e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 3.444373471802985e-05, + "d_pool_score": 3.4315328261516696e-05, + "d_ape": 1.2737882398570104e-06 + }, + "critical_step_samples_ms": [ + 18.968896865844727, + 19.191680908203125, + 18.971071243286133, + 18.9803524017334, + 18.957279205322266 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "zigzag": { + "step_ms": 22.57574462890625, + "prepare_ms": 59.226449113339186, + "one_use_ms": 81.80219374224544, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.542628572706754e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.743842138582279e-05, + "d_pool_score": 5.184653523102845e-05, + "d_ape": 1.688316934838138e-06 + }, + "critical_step_samples_ms": [ + 22.599584579467773, + 22.599424362182617, + 22.55353546142578, + 22.539520263671875, + 22.57574462890625 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "planner": { + "step_ms": 18.966144561767578, + "prepare_ms": 10.67610492464155, + "one_use_ms": 29.642249486409128, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.695941042091729e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.221088970557776e-05, + "d_pool_score": 4.9402061088101224e-05, + "d_ape": 1.3945307090872565e-06 + }, + "critical_step_samples_ms": [ + 19.05878448486328, + 18.966144561767578, + 19.218975067138672, + 18.93948745727539, + 18.935871124267578 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + } + } + } + } + } + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "34c2d866f81c1127623ba482adce4ff365a7a1663f66bfdbf717a39cbb7c3beb", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "native_subgroup_checks": { + "subgroups-final": { + "checks": 64, + "all_ids_equal": true, + "all_results_retained": true, + "max_score_relative_rms": 0.0, + "source_record_sha256": "674df06728eb4c8a379f496018ad03db108b684c8a7be8adf90ea8d8571429ea" + }, + "subgroups-fp32": { + "checks": 64, + "all_ids_equal": true, + "all_results_retained": true, + "max_score_relative_rms": 0.0, + "source_record_sha256": "20991c9a3c1cd5abbb223067c4d71f345116c2a72d57f7d2b4d2e30929d9694d" + } + }, + "fp32_checks": { + "check-many": { + "max_relative_rms": { + "output": 0.001414975617080927, + "dq": 0.002863918896764517, + "dkv": 0.002098844153806567, + "teacher": 0, + "d_index_q": 0, + "d_weights": 0, + "d_kbar": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_error": 0, + "source_record_sha256": "fee8df5ea2b612f3b5e7825d6b3a1522bd6be4fcc7b5d40fcd2fae8c0f9d182d" + }, + "check-mixed": { + "max_relative_rms": { + "output": 0.0023773587308824062, + "dq": 0.0028181811794638634, + "dkv": 0.0024023125879466534, + "teacher": 1.22032631111324e-07, + "d_index_q": 0.002322548534721136, + "d_weights": 0.0016673828940838575, + "d_kbar": 0.0019576919730752707, + "d_pool_kv": 0.002863020868971944, + "d_pool_score": 0.002832910744473338, + "d_ape": 0.0004040832573082298 + }, + "max_loss_abs_error": 7.4078343459405e-10, + "source_record_sha256": "3733c8a18429571b586faf401c2f5ba3e2b9ca2e30a70b924b06a501e1923db9" + }, + "check-single": { + "max_relative_rms": { + "output": 0.002390037290751934, + "dq": 0.0023644936736673117, + "dkv": 0.0024435562081635, + "teacher": 1.2636910184937733e-07, + "d_index_q": 0.002317695878446102, + "d_weights": 0.0016715021338313818, + "d_kbar": 0.002232952509075403, + "d_pool_kv": 0.0030496628023684025, + "d_pool_score": 0.0030512649100273848, + "d_ape": 0.0003674851031973958 + }, + "max_loss_abs_error": 1.3108092389302328e-09, + "source_record_sha256": "e2bd8016a3fdc9572c2ce76afce91cc30ed88e594916821215b5c62cd9aafd98" + } + } +} diff --git a/tests/test_csa_cp_selection.py b/tests/test_csa_cp_selection.py new file mode 100644 index 00000000..7b0fec8e --- /dev/null +++ b/tests/test_csa_cp_selection.py @@ -0,0 +1,238 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Host planning and execution-contract checks without a native dependency.""" + +from dataclasses import replace +from itertools import accumulate +from types import SimpleNamespace + +import pytest +import torch + +from benchmarks.cp_layouts import query_layouts +from cula.distributed import THDPlannerConfig, THDSelectionCostModel, plan_thd_selection +from cula.hybrid import CsaCPSelectionPlan, CsaCPSelectionWorkspace, csa_cp, cudnn_csa_select_cp +from cula.hybrid.dsa import DsaSelectionPlan + + +def costs(world): + return THDSelectionCostModel( + candidate_ns=0.1, + query_ns=8, + span_us=5, + bytes_per_query=2 * 4 * 2 + 2 * 2 + 8 * 8, + intra_node_gbps=40, + inter_node_gbps=20, + rank_node_ids=(0,) * world, + ) + + +@pytest.mark.parametrize("tokens", [128 << 10, 256 << 10, 1 << 20]) +@pytest.mark.parametrize("world", [8, 16]) +@pytest.mark.parametrize("kind", ["single", "uniform", "near-uniform", "mixed", "long-tail"]) +@pytest.mark.parametrize("policy", ["contiguous", "zigzag", "planner"]) +def test_plans_match_existing_execution_metadata(tokens, world, kind, policy): + if kind == "single": + lengths = [tokens] + elif kind == "uniform": + lengths = [tokens // 1024] * 1024 + elif kind == "near-uniform": + lengths = [tokens // 512 - 4, tokens // 512 + 4] * 256 + elif kind == "mixed": + lengths = [64] * 1000 + [(tokens - 64000) // 2] * 2 + else: + lengths = [64] * 500 + [256] * 12 + [tokens - 35072] + cu = (0, *accumulate(lengths)) + model = costs(world) + layouts, selected = query_layouts(cu, world, policy, cost_model=model, planner_strategy="selection") + # First, interior and last source ranks cover both complete and split docs. + for rank in (0, world // 2, world - 1): + p = CsaCPSelectionPlan.from_cu_seqlens(cu, rank=rank, world_size=world, top_k=8, policy=policy, cost_model=model) + assert p.ranks == layouts and p.policy == selected + local = layouts[rank] + assert p.exchange.source_rows == tokens // world + assert p.exchange.target_rows == local.local_tokens + ranges = tuple((s.sequence_start // 4, s.sequence_end // 4) for s in local.spans) + assert p.key_ranges == ranges + if not local.local_tokens: + assert p.selection is None + continue + prefix = (0, *accumulate(z - a for a, z in ranges)) + expected = DsaSelectionPlan.from_cu_seqlens( + local.local_cu_seqlens, + prefix, + top_k=8, + q_causal_offsets=local.causal_offsets, + ) + assert p.selection == expected + + +@pytest.mark.parametrize( + "options", + [ + {"world_size": 0}, + {"rank": True}, + {"rank": 8}, + {"top_k": 0}, + {"top_k": 2049}, + {"top_k": True}, + {"policy": "unknown"}, + {"max_tokens_per_rank": 15}, + {"candidate_budget_bytes": 0}, + {"candidate_budget_bytes": True}, + {"cost_model": costs(4)}, + ], +) +def test_invalid_plan_options(options): + kwargs = dict(rank=0, world_size=8, top_k=8, policy="contiguous") + kwargs.update(options) + with pytest.raises((ValueError, TypeError)): + CsaCPSelectionPlan.from_cu_seqlens((0, 128), **kwargs) + + +@pytest.mark.parametrize("cu", [(0, 5, 128), (0, 16, 12), (1, 128), (0, 2**31), (0, 4)]) +def test_invalid_global_metadata_fails_on_every_rank(cu): + for rank in range(8): + with pytest.raises(ValueError): + CsaCPSelectionPlan.from_cu_seqlens(cu, rank=rank, world_size=8, top_k=8, policy="contiguous") + + +def test_planner_needs_explicit_costs_and_never_reads_device_metadata(): + with pytest.raises(ValueError, match="cost_model"): + CsaCPSelectionPlan.from_cu_seqlens((0, 128), rank=0, world_size=8, top_k=8) + with pytest.raises(ValueError, match="CPU tensor"): + CsaCPSelectionPlan.from_cu_seqlens(torch.empty(2, device="meta"), rank=0, world_size=8, top_k=8, policy="contiguous") + + +def test_empty_zigzag_destination_and_block_balanced_sources(): + plans = [CsaCPSelectionPlan.from_cu_seqlens((0, 4, 36), rank=r, world_size=8, top_k=8, policy="zigzag") for r in range(8)] + assert all(p.exchange.source_rows % 4 == 0 for p in plans) + assert sum(p.exchange.target_rows for p in plans) == 36 + # Unequal source counts are intentional: nine blocks across eight ranks. + assert {p.exchange.source_rows for p in plans} == {4, 8} + tiny = [ + CsaCPSelectionPlan.from_cu_seqlens( + tuple(range(0, 36, 4)), rank=r, world_size=8, top_k=8, policy="zigzag", max_tokens_per_rank=32 + ) + for r in range(8) + ] + assert any(p.selection is None for p in tiny) + + +@pytest.mark.parametrize("identity", [True, False]) +def test_source_order_result_lifetime_and_validation(monkeypatch, identity): + plan = CsaCPSelectionPlan.from_cu_seqlens((0, 16), rank=0, world_size=1, top_k=8, policy="contiguous", cost_model=costs(1)) + plan = replace(plan, exchange=replace(plan.exchange, identity=identity)) + calls = [] + + def route(tensor): + calls.append("route") + return tensor if identity else tensor.flip(0) + + def restore(tensor): + calls.append("restore") + return tensor if identity else tensor.flip(0) + + exchange = SimpleNamespace(device=torch.device("cpu"), group=None, distribute=route, restore=restore) + native = SimpleNamespace(block_ids=torch.empty(16, 8, dtype=torch.int32), scores=torch.empty(16, 8)) + workspace = CsaCPSelectionWorkspace(plan, exchange, native, slice(0, 4), None) + monkeypatch.setattr(csa_cp.dist, "get_rank", lambda group: 0) + monkeypatch.setattr(csa_cp.dist, "get_world_size", lambda group: 1) + seen = [] + + def select(q, k, w, **kwargs): + seen.append(kwargs) + native.block_ids.copy_(q[:, 0, 0, None].expand(-1, 8).int()) + native.scores.copy_(q[:, 0, 0, None].expand(-1, 8).float()) + return native + + monkeypatch.setattr(csa_cp, "cudnn_dsa_select", select) + qi = torch.arange(16, dtype=torch.bfloat16)[:, None, None].expand(-1, 2, 4).clone().requires_grad_() + keys, weights = torch.ones(4, 1, 4, dtype=torch.bfloat16), torch.ones(16, 2, dtype=torch.bfloat16) + result = cudnn_csa_select_cp(qi, keys, weights, workspace=workspace, sm_scale=0.5) + assert torch.equal(result.block_ids[:, 0], torch.arange(16)) + assert torch.equal(result.scores[:, 0], torch.arange(16).float()) + native.block_ids.fill_(-1) + native.scores.zero_() + assert torch.equal(result.block_ids[:, 0], torch.arange(16)) + assert torch.equal(result.scores[:, 0], torch.arange(16).float()) + assert not result.scores.requires_grad + assert calls == ["route", "route", "restore", "restore"] + assert seen[0]["deterministic"] is False and seen[0]["sm_scale"] == 0.5 + for bad_q, bad_k, bad_w in [ + (qi.float(), keys, weights), + (qi, keys.float(), weights), + (qi[:, :, ::2], keys, weights), + (qi, keys, weights[:, :1]), + ]: + with pytest.raises((TypeError, ValueError)): + cudnn_csa_select_cp(bad_q, bad_k, bad_w, workspace=workspace) + with pytest.raises(ValueError, match="bytes_per_query"): + cudnn_csa_select_cp(qi, keys, weights.float(), workspace=workspace) + with pytest.raises(ValueError, match="finite"): + cudnn_csa_select_cp(qi, keys, weights, workspace=workspace, sm_scale=float("nan")) + with pytest.raises(TypeError, match="bool"): + cudnn_csa_select_cp(qi, keys, weights, workspace=workspace, deterministic=1) + assert len(calls) == 4 # Invalid calls fail before any collective. + + +@pytest.mark.parametrize("world", [8, 16, 64]) +@pytest.mark.parametrize("tokens", [128 << 10, 256 << 10]) +def test_effective_chunk_floor_limits_high_cp_fragmentation(world, tokens): + from cula.distributed.thd_planner import _adaptive_items + + config = THDPlannerConfig(min_chunk_tokens=1024) + pieces = _adaptive_items((0, tokens), world, config, chunks_per_rank=6, sequences={0}) + assert all(item.tokens >= 1024 for item in pieces) + assert len(pieces) <= tokens // 1024 + plan = plan_thd_selection( + (0, tokens), world_size=world, max_tokens_per_rank=2 * tokens // world, cost_model=costs(world), config=config + ) + assert all(span.tokens >= 1024 for rank in plan.ranks for span in rank.spans) + assert sum(rank.local_tokens for rank in plan.ranks) == tokens + + +def test_chunk_floor_preserves_short_source_fragments_and_fallback(): + cu = (0, 4, 128 << 10) + plan = plan_thd_selection( + cu, world_size=64, max_tokens_per_rank=4096, cost_model=costs(64), config=THDPlannerConfig(min_chunk_tokens=4096) + ) + assert sum(rank.local_tokens for rank in plan.ranks) == cu[-1] + assert any(s.tokens == 4 for r in plan.ranks for s in r.spans) + # An infeasible grain preference must not remove the contiguous fallback. + assert plan.policy == "contiguous" + + +@pytest.mark.parametrize("floor", [-4, 1, True, 1.5]) +def test_invalid_chunk_floor(floor): + with pytest.raises((ValueError, TypeError)): + plan_thd_selection( + (0, 128), + world_size=8, + max_tokens_per_rank=32, + cost_model=costs(8), + config=THDPlannerConfig(min_chunk_tokens=floor), + ) + + +@pytest.mark.parametrize("world", [1, 8, 16, 64]) +@pytest.mark.parametrize("lengths", [[4] * 1024, [4, 128, 512, 65536], [131072], [64] * 998 + [33600, 33600]]) +def test_identity_metrics_match_general_cost_model(world, lengths): + from cula.distributed.thd_planner import _contiguous_items, _metrics, _source_selection_candidate + + cu = (0, *accumulate(lengths)) + config, model = THDPlannerConfig(), costs(world) + ranks, metrics = _source_selection_candidate(_contiguous_items(cu, world, 4), world, model) + expected = _metrics([s for r in ranks for s in r.spans], world_size=world, block_size=4, config=config) + score = model.estimate_ms(ranks, block_size=4) + assert metrics == replace(expected, score=score) + + +@pytest.mark.parametrize("ranges", [[], [(0, 0)], [(3, 9)], [(7, 11), (0, 3), (7, 11)], [(5, 5), (12, 16)]]) +def test_range_indices_preserve_duplicates_gaps_and_order(ranges): + from cula._metadata import _range_indices + + actual = _range_indices(ranges, torch.device("cpu")) + expected = torch.tensor([i for a, z in ranges for i in range(a, z)], dtype=torch.int64) + assert torch.equal(actual, expected) From ea71df3b3bfad69141d1ee3ddd884ac9ccdb27ce Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Mon, 16 Nov 2026 20:53:23 +0800 Subject: [PATCH 7/9] test(csa): compare CP policies across document counts and contexts Signed-off-by: shuyan.ycf --- benchmarks/bench_csa_cp_windows.py | 7 +- docs/CSA_CP_SELECTION.md | 4 +- docs/CSA_CP_THREE_WAY_RESULTS.md | 228 ++ docs/CSA_CP_WINDOW_RESULTS.md | 5 + docs/results/csa_cp_three_way_20261116.json | 3681 +++++++++++++++++++ tests/test_csa_cp_selection.py | 25 + 6 files changed, 3947 insertions(+), 3 deletions(-) create mode 100644 docs/CSA_CP_THREE_WAY_RESULTS.md create mode 100644 docs/results/csa_cp_three_way_20261116.json diff --git a/benchmarks/bench_csa_cp_windows.py b/benchmarks/bench_csa_cp_windows.py index 4342338e..e5c10a27 100644 --- a/benchmarks/bench_csa_cp_windows.py +++ b/benchmarks/bench_csa_cp_windows.py @@ -27,7 +27,7 @@ def cases(tokens): - """Five block-aligned layouts, including 512/1000-document packs.""" + """Block-aligned layouts with one, dozens, and hundreds of documents.""" short = 64 mixed = [short] * 998 + [(tokens - 998 * short) // 2] * 2 # The two long documents have different positions in a reproducible pack. @@ -35,6 +35,9 @@ def cases(tokens): tail = [64] * 800 + [256] * 199 + [tokens - 800 * 64 - 199 * 256] return { "single": [tokens], + "uniform-64": [tokens // 64] * 64, + "near-uniform-64": [tokens // 64 - tokens // 1024, tokens // 64 + tokens // 1024] * 32, + "mixed-22": [1024] * 20 + [(tokens - 20 * 1024) // 2] * 2, "uniform-1024": [tokens // 1024] * 1024, "near-uniform-512": [tokens // 512 - 4, tokens // 512 + 4] * 256, "mixed-1000": mixed, @@ -81,7 +84,7 @@ def main(): parser.add_argument("--profile", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--tokens", type=int, nargs="+", default=[128 << 10, 256 << 10]) - parser.add_argument("--cases", nargs="+", help="Subset of the five named distributions") + parser.add_argument("--cases", nargs="+", help="Subset of the named distributions") parser.add_argument("--warmups", type=int, default=2) parser.add_argument("--repeats", type=int, default=5) parser.add_argument("--prepare-repeats", type=int, default=8) diff --git a/docs/CSA_CP_SELECTION.md b/docs/CSA_CP_SELECTION.md index 695341f7..9f5973b1 100644 --- a/docs/CSA_CP_SELECTION.md +++ b/docs/CSA_CP_SELECTION.md @@ -141,4 +141,6 @@ wall time includes workspace allocation, H2D metadata and a completion sync; there is no implicit cache or plan-reuse credit. Host-time outliers are retained. Measured B300 CP8 results and the preparation limitations are documented in -[the 128K/256K study](CSA_CP_WINDOW_RESULTS.md). +[the 128K/256K study](CSA_CP_WINDOW_RESULTS.md) and the expanded +[128K/256K/1M three-way comparison](CSA_CP_THREE_WAY_RESULTS.md), including +22- and 64-document packs. diff --git a/docs/CSA_CP_THREE_WAY_RESULTS.md b/docs/CSA_CP_THREE_WAY_RESULTS.md new file mode 100644 index 00000000..13448cb2 --- /dev/null +++ b/docs/CSA_CP_THREE_WAY_RESULTS.md @@ -0,0 +1,228 @@ +# CSA CP: three policies across 128K, 256K and 1M + +On B300 CP8, adaptive selection routing stays within **0.576%** of the faster +fixed contiguous/zigzag policy in steady execution across these 24 cases. The +largest gains occur at 1M with unequal long documents. Uniform packs usually +select contiguous; a single long document performs similarly to zigzag. +This is measured coverage, not a guarantee over arbitrary THD inputs. + +The workload is **one complete CSA operator forward/backward**: ratio-4 pooling, +pooled-key and attention-KV gathers, routed selection and inverse exchange, +sparse attention, teacher/KL and all activation/parameter gradients. Runtime +layout conversions and communication are included. KDA, projections, MoE, +optimizer and checkpoint recomputation are excluded. These are not timings of +a connected seven-KDA/one-CSA training block. + +## Measurement and workloads + +- Eight B300 GPUs, CSA H64/D576/V512, indexer H64/D128, Top-K512. +- BF16 Q/K/weights; FP32 scores, nonlinearities and communication reductions. + TF32 and BF16 reduced-precision GEMM intermediate reductions are disabled. + Native nondeterministic mode is unchanged. +- PyTorch 2.12.1+cu130, CUDA runtime 13.0, CuTeDSL 4.8.1, cuDNN 9.20.0.48. + cuDNN Frontend source commit `855ccd77ba9efed22441fd454f7cc0206a068e94`. + Frontend wheel version metadata does not identify the imported source tree. +- Two warmups, seven timed execution samples and eight fresh preparation samples. + Policy order rotates and reverses. Each reported duration is the median of + the per-sample maximum across all ranks. Compilation, synthetic input creation + and correctness reference work are outside timing. Host-time outliers remain. +- The existing compute/transport profile is used with explicit `min_gain_ms=0`, + `max_chunks_per_rank=6`, `min_chunk_tokens=1024`. This is a reuse-oriented + throughput experiment, not a newly calibrated profile or changed default. +- Public contiguous, full per-document zigzag and planner share the same source + ownership. Zigzag/planner redistribute selection Q/weights and restore results; + attention retains the original ownership. A fourth legacy-contiguous fixture + is the correctness reference and is not a table column. + +Here K means 1,024 tokens and M means 1,048,576 tokens. All lengths are divisible +by four, and each pack sums exactly to the context size T. + +| Distribution | Document lengths | +|---|---| +| 1 long document | T | +| 64 uniform | 64 x T/64 | +| 64 near-uniform | Alternating T/64 - T/1024 and T/64 + T/1024, 32 pairs | +| 20 short + 2 long | 20 x 1,024, followed by 2 x (T - 20,480)/2 | +| 1,024 uniform | 1,024 x T/1,024 | +| 512 near-uniform | Alternating T/512 - 4 and T/512 + 4, 256 pairs | +| 998 short + 2 long | 998 x 64 and 2 x (T - 63,872)/2; long docs after 199 shorts and at the end | +| 1,000 long-tail | 800 x 64, 199 x 256 and one remaining long document | + +All tables report milliseconds; lower is better. Percentage columns are +`100 * (planner / baseline - 1)`; negative means less time. Subpercent differences +should be treated cautiously, rather than as strong evidence of a speedup. + +## 128K steady execution + +| Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | +|---|---:|---:|---:|---:|---:| +| 1 long document | 47.60 | 46.23 | 45.82 | -3.74% | -0.88% | +| 64 uniform | 20.11 | 20.66 | 20.04 | -0.37% | -3.02% | +| 64 near-uniform | 20.08 | 20.79 | 20.06 | -0.13% | -3.53% | +| 20 short + 2 long | 35.44 | 35.23 | 35.44 | -0.00% | +0.58% | +| 1,024 uniform | 9.29 | 11.10 | 9.30 | +0.09% | -16.28% | +| 512 near-uniform | 10.01 | 11.21 | 10.03 | +0.14% | -10.52% | +| 998 short + 2 long | 33.07 | 38.75 | 33.07 | -0.00% | -14.64% | +| 1,000 long-tail | 32.37 | 37.90 | 32.35 | -0.06% | -14.64% | + +## 256K steady execution + +| Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | +|---|---:|---:|---:|---:|---:| +| 1 long document | 115.29 | 105.58 | 104.15 | -9.67% | -1.36% | +| 64 uniform | 50.34 | 51.73 | 50.33 | -0.00% | -2.70% | +| 64 near-uniform | 50.34 | 51.77 | 50.35 | +0.01% | -2.75% | +| 20 short + 2 long | 90.69 | 87.95 | 87.39 | -3.64% | -0.64% | +| 1,024 uniform | 18.96 | 22.56 | 18.94 | -0.10% | -16.02% | +| 512 near-uniform | 21.79 | 24.21 | 21.76 | -0.12% | -10.12% | +| 998 short + 2 long | 83.64 | 96.05 | 82.15 | -1.78% | -14.47% | +| 1,000 long-tail | 99.36 | 115.25 | 92.20 | -7.20% | -20.00% | + +## 1M steady execution + +| Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | +|---|---:|---:|---:|---:|---:| +| 1 long document | 853.46 | 638.50 | 637.55 | -25.30% | -0.15% | +| 64 uniform | 239.31 | 244.20 | 239.05 | -0.11% | -2.11% | +| 64 near-uniform | 239.06 | 244.37 | 239.25 | +0.08% | -2.10% | +| 20 short + 2 long | 576.29 | 493.97 | 491.52 | -14.71% | -0.50% | +| 1,024 uniform | 106.86 | 121.14 | 106.80 | -0.05% | -11.84% | +| 512 near-uniform | 153.78 | 162.94 | 153.50 | -0.18% | -5.79% | +| 998 short + 2 long | 566.17 | 549.72 | 493.44 | -12.85% | -10.24% | +| 1,000 long-tail | 801.37 | 715.45 | 591.85 | -26.14% | -17.28% | + +## Preparation plus one use + +The following is **median fresh preparation + median execution**, not a directly +timed cold end-to-end sample. Preparation includes common metadata, host planning, +workspace/index construction, H2D copies and a completion sync. It is rebuilt, +not replaced by a cache lookup. First compilation is still excluded. + +### 128K + +| Distribution | Contiguous | Zigzag | Planner | +|---|---:|---:|---:| +| 1 long document | 48.33 | 47.14 | 53.37 | +| 64 uniform | 21.07 | 24.84 | 21.28 | +| 64 near-uniform | 21.06 | 24.95 | 21.24 | +| 20 short + 2 long | 36.25 | 37.16 | 43.04 | +| 1,024 uniform | 16.07 | 58.33 | 18.27 | +| 512 near-uniform | 13.76 | 35.29 | 14.98 | +| 998 short + 2 long | 40.72 | 84.47 | 61.39 | +| 1,000 long-tail | 39.95 | 83.74 | 44.48 | + +### 256K + +| Distribution | Contiguous | Zigzag | Planner | +|---|---:|---:|---:| +| 1 long document | 116.41 | 106.72 | 111.94 | +| 64 uniform | 51.40 | 55.87 | 51.61 | +| 64 near-uniform | 51.40 | 56.02 | 51.62 | +| 20 short + 2 long | 91.84 | 90.29 | 96.16 | +| 1,024 uniform | 25.87 | 69.42 | 27.91 | +| 512 near-uniform | 25.73 | 48.89 | 26.78 | +| 998 short + 2 long | 93.22 | 188.33 | 114.82 | +| 1,000 long-tail | 109.06 | 262.29 | 124.02 | + +### 1M + +| Distribution | Contiguous | Zigzag | Planner | +|---|---:|---:|---:| +| 1 long document | 855.70 | 640.79 | 646.87 | +| 64 uniform | 240.80 | 248.86 | 240.77 | +| 64 near-uniform | 240.57 | 249.16 | 241.00 | +| 20 short + 2 long | 578.53 | 497.20 | 501.32 | +| 1,024 uniform | 114.35 | 169.54 | 116.43 | +| 512 near-uniform | 158.33 | 188.80 | 159.24 | +| 998 short + 2 long | 576.80 | 597.98 | 528.35 | +| 1,000 long-tail | 815.55 | 867.25 | 630.53 | + +## What the comparison shows + +The aggregate below sums one execution of each of the eight distributions at +each context length. It assumes equal case frequency, not equal runtime weights +or a measured production mix. Values are percentage time reductions relative +to the specified fixed policy; a negative reduction means a slowdown. + +| Context | Steady vs contiguous | Steady vs zigzag | Prepare + one use vs contiguous | Prepare + one use vs zigzag | +|---|---:|---:|---:|---:| +| 128K | 0.90% | 7.11% | -17.22% | 29.77% | +| 256K | 4.36% | 8.62% | -7.07% | 31.10% | +| 1M | 16.50% | 6.85% | 14.41% | 11.42% | + +At 1M, the two-long/998-short mixture improves steady time by **12.85% versus +contiguous and 10.24% versus zigzag**. The 1,000-document long tail improves by +**26.14% and 17.28%**, respectively. These two cases choose locality-preserving +six-piece-per-rank candidates. Balancing causal selection work while avoiding +unnecessary movement of short documents beats applying full zigzag everywhere. + +The dozens-of-documents cases add a useful distinction: 64 uniform/near-uniform +documents choose contiguous at all three lengths, avoiding the full-zigzag cost +for already-balanced work. The 20-short/two-long pack at 1M improves **14.71%** +over contiguous and is close to zigzag. At 128K, however, the same document-count +pattern is **0.576% slower than zigzag**. Layout choice is not perfectly predicted; +the larger aggregate gain at 1M does not establish monotonic improvement per case. + +Fresh preparation still matters. At 1M, including it leaves planner slightly +slower than zigzag for the single document and the 22-document pack, despite +similar steady execution. At 128K/256K the aggregate one-use result remains +slower than contiguous. The worst one-use gap to the faster fixed policy across +all 24 cases is **50.77%** for the 128K two-long/998-short mixture. Thus this study +does not establish universally optimal one-use selection or a <3% preparation +budget. A reused layout and a layout used only once need different preparation +budgets. Some preparation samples have large host-time spikes; the raw minimum, +maximum and all critical samples are retained in the linked JSON. + +## Precision, review and provenance + +All **24 cases x four paths x eight ranks** pass row-identity/inverse checks and +full-tensor output/seven-gradient comparisons against legacy contiguous. Maximum +relative RMS across all public paths/ranks/cases is shown below; it is not a +bitwise-equivalence claim for nondeterministic native kernels. + +| Tensor | Maximum relative RMS | +|---|---:| +| Output | 0.21361% | +| dQ | 0.06709% | +| dKV | 0.10311% | +| Indexer dQ | 0.07967% | +| Indexer dWeights | 0.06914% | +| Pool dKV | 0.13756% | +| Pool dScore | 0.13780% | +| Pool dAPE | 0.00265% | + +Maximum auxiliary loss absolute difference is **8.15e-10**. No runtime math, +dtype, kernel or precision option changed for this coverage extension. +**444 focused CPU tests pass**, including nine new document-count/context +coverage tests. The unchanged runtime had also passed 128 native subgroup +selection checks and three small independent FP32 oracle fixtures under all +four paths, as detailed in the [original study](CSA_CP_WINDOW_RESULTS.md). + +The original ten 128K/256K records are reused from that study; fourteen additional +records cover six dozens-of-documents cases at 128K/256K and all eight 1M +distributions. Runtime source is `d2cf4fd`; only benchmark fixture +construction/help changed between timing cohorts. All 125 source hashes match +for the new cohorts. The only differing source hash in the original cohort is +that benchmark file. No algorithm or profile was tuned between these cohorts. + +The report preserves per-case timing/error envelopes, selected policies, all +critical timing/preparation samples, source-record hashes, and cohort source +manifests in [machine-readable results](results/csa_cp_three_way_20261116.json). +Diagnostic allocator peaks in the underlying harness include four coexisting +fixtures; they must not be interpreted as isolated per-policy memory usage. + +## Reproduction + +```bash +python -m torch.distributed.run --standalone --nproc-per-node=8 \ + -m benchmarks.bench_csa_cp_windows \ + --profile benchmarks/profiles/thd_selection_b300_cp8.json \ + --output-dir results/csa-three-way --tokens 131072 262144 1048576 \ + --cases single uniform-64 near-uniform-64 mixed-22 \ + uniform-1024 near-uniform-512 mixed-1000 long-tail-1000 \ + --min-gain-ms 0 --min-chunk-tokens 1024 --max-chunks-per-rank 6 \ + --warmups 2 --repeats 7 --prepare-repeats 8 +``` + +See the [public API contract](CSA_CP_SELECTION.md) for explicit plan/workspace +lifetimes, source ownership and downstream integration. diff --git a/docs/CSA_CP_WINDOW_RESULTS.md b/docs/CSA_CP_WINDOW_RESULTS.md index b6d27a7b..ef596669 100644 --- a/docs/CSA_CP_WINDOW_RESULTS.md +++ b/docs/CSA_CP_WINDOW_RESULTS.md @@ -1,5 +1,9 @@ # CSA CP selection at 128K and 256K +The [expanded three-way comparison](CSA_CP_THREE_WAY_RESULTS.md) retains these +ten measurements and adds 1M contexts and 22-/64-document packs, for 24 cases. +The historical ten-case statistics below remain scoped to this original study. + This tests the public `cudnn_csa_select_cp` API on eight B300 GPUs. The workload is **one complete CSA operator forward/backward**, including ratio-4 pooling, pooled-key and attention-KV gathers, selection routing/restoration, sparse @@ -127,6 +131,7 @@ python -m torch.distributed.run --standalone --nproc-per-node=8 \ -m benchmarks.bench_csa_cp_windows \ --profile benchmarks/profiles/thd_selection_b300_cp8.json \ --output-dir results/csa-windows --tokens 131072 262144 \ + --cases single uniform-1024 near-uniform-512 mixed-1000 long-tail-1000 \ --min-gain-ms 0 --min-chunk-tokens 1024 --max-chunks-per-rank 6 \ --warmups 2 --repeats 7 --prepare-repeats 8 diff --git a/docs/results/csa_cp_three_way_20261116.json b/docs/results/csa_cp_three_way_20261116.json new file mode 100644 index 00000000..9ad4f8da --- /dev/null +++ b/docs/results/csa_cp_three_way_20261116.json @@ -0,0 +1,3681 @@ +{ + "scope": "Complete CSA operator forward/backward, including layouts and communication; no KDA/projections/MoE", + "platform": "B300 CP8", + "native_source_commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "runtime_commit": "d2cf4fdf7b7ec8fc4e22ecd780dd4a48d0a61670", + "cases": { + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "4ffc0c811647416d205412593d727352a535ab6cd11c26ad666b219d87fde48f", + "policies": { + "contiguous": { + "step_ms": 47.59836959838867, + "prepare_ms": 0.7295145187526941, + "one_use_ms": 48.327884117141366, + "prepare_min_ms": 0.7136678323149681, + "prepare_max_ms": 0.8075819350779057, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015765946646741124, + "dq": 0.0005790440951112175, + "dkv": 0.0006679192583025653, + "d_index_q": 0.0007036192249884126, + "d_weights": 0.0006587839768356234, + "d_pool_kv": 0.0008462754770025072, + "d_pool_score": 0.000887909900268915, + "d_ape": 2.647273742900797e-05 + }, + "max_loss_abs_diff": 1.4551915228366852e-10, + "critical_step_samples_ms": [ + 47.504417419433594, + 47.59836959838867, + 47.79350280761719, + 47.29996871948242, + 47.922752380371094, + 47.66716766357422, + 47.468990325927734 + ], + "critical_prepare_samples_ms": [ + 0.8075819350779057, + 0.7368980441242456, + 0.7332341047003865, + 0.7257949328050017, + 0.7136678323149681, + 0.7397379959002137, + 0.7191249169409275, + 0.7216879166662693 + ] + }, + "zigzag": { + "step_ms": 46.22537612915039, + "prepare_ms": 0.9122525225393474, + "one_use_ms": 47.13762865168974, + "prepare_min_ms": 0.8884459966793656, + "prepare_max_ms": 2.3740038741379976, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015766157105562468, + "dq": 0.0005791250183306168, + "dkv": 0.0005826837325579235, + "d_index_q": 0.0006550745397822342, + "d_weights": 0.0005112390665302352, + "d_pool_kv": 0.0007098261716766501, + "d_pool_score": 0.0007224017331145301, + "d_ape": 2.402751970718015e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 46.06342315673828, + 46.19878387451172, + 46.51465606689453, + 46.22537612915039, + 46.21033477783203, + 46.31951904296875, + 46.25299072265625 + ], + "critical_prepare_samples_ms": [ + 2.3740038741379976, + 0.9390920167788863, + 0.9824440348893404, + 0.9092459222301841, + 0.9014920797199011, + 0.9148720419034362, + 0.8884459966793656, + 0.9096330031752586 + ] + }, + "planner": { + "step_ms": 45.816734313964844, + "prepare_ms": 7.54898099694401, + "one_use_ms": 53.365715310908854, + "prepare_min_ms": 7.393982145003974, + "prepare_max_ms": 7.698302972130477, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0015837265043694255, + "dq": 0.0005816460263006044, + "dkv": 0.0007768235696223359, + "d_index_q": 0.0007047741714831017, + "d_weights": 0.000687690452839031, + "d_pool_kv": 0.0008500666348145983, + "d_pool_score": 0.0009004768884284642, + "d_ape": 2.473088932251163e-05 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 45.82166290283203, + 45.84272003173828, + 45.73017501831055, + 46.25267028808594, + 45.816734313964844, + 45.81379318237305, + 45.737918853759766 + ], + "critical_prepare_samples_ms": [ + 7.6499179704114795, + 7.698302972130477, + 7.51110992860049, + 7.575499941594899, + 7.636827998794615, + 7.517814985476434, + 7.522462052293122, + 7.393982145003974 + ] + } + } + }, + "128k-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "3a1a1ac62ea766a15b80829e415a9660c79cea6aed1be6fb755c4527f4bef8d4", + "policies": { + "contiguous": { + "step_ms": 20.111711502075195, + "prepare_ms": 0.9577220189385116, + "one_use_ms": 21.069433521013707, + "prepare_min_ms": 0.9080220479518175, + "prepare_max_ms": 1.176033983938396, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.589457181863983e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.33998632212166e-05, + "d_pool_score": 6.779927803572065e-05, + "d_ape": 1.564723126474068e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.175872802734375, + 20.117664337158203, + 20.179584503173828, + 20.060287475585938, + 20.016511917114258, + 20.111711502075195, + 20.057247161865234 + ], + "critical_prepare_samples_ms": [ + 1.0195919312536716, + 0.9686399716883898, + 0.9425180032849312, + 1.176033983938396, + 0.9080220479518175, + 0.9982179617509246, + 0.9080249583348632, + 0.9468040661886334 + ] + }, + "zigzag": { + "step_ms": 20.662303924560547, + "prepare_ms": 4.178662959020585, + "one_use_ms": 24.84096688358113, + "prepare_min_ms": 3.9806190179660916, + "prepare_max_ms": 4.345727036707103, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.160216835107211e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.664592625392898e-05, + "d_pool_score": 6.241826443094128e-05, + "d_ape": 1.4828357064696632e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.99295997619629, + 20.976959228515625, + 20.812480926513672, + 20.662303924560547, + 20.6615047454834, + 20.65987205505371, + 20.65795135498047 + ], + "critical_prepare_samples_ms": [ + 4.096206976100802, + 4.345727036707103, + 4.321377025917172, + 4.138701944611967, + 4.223175114020705, + 3.98633093573153, + 4.218623973429203, + 3.9806190179660916 + ] + }, + "planner": { + "step_ms": 20.038047790527344, + "prepare_ms": 1.2417834950610995, + "one_use_ms": 21.279831285588443, + "prepare_min_ms": 1.1157230474054813, + "prepare_max_ms": 1.3633579947054386, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.742825463536053e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.19493785934264e-05, + "d_pool_score": 6.324612395848447e-05, + "d_ape": 1.6624866979508424e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.352832794189453, + 20.19059181213379, + 20.023487091064453, + 20.057823181152344, + 20.014240264892578, + 20.036767959594727, + 20.038047790527344 + ], + "critical_prepare_samples_ms": [ + 1.2311399914324284, + 1.2524269986897707, + 1.3623570557683706, + 1.3633579947054386, + 1.257814932614565, + 1.134752994403243, + 1.1582979932427406, + 1.1157230474054813 + ] + } + } + }, + "128k-near-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "5ef7ec8fee87197b0c117b0eaf0008be59967fe8c900e7161fd432c33f97ea77", + "policies": { + "contiguous": { + "step_ms": 20.080768585205078, + "prepare_ms": 0.9788154857233167, + "one_use_ms": 21.059584070928395, + "prepare_min_ms": 0.9078129660338163, + "prepare_max_ms": 1.02512096054852, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016355387131441923, + "dq": 8.456624756602418e-05, + "dkv": 0.0001043296667775725, + "d_index_q": 2.4571583304236613e-05, + "d_weights": 2.937356095947118e-05, + "d_pool_kv": 9.235941677369348e-05, + "d_pool_score": 0.00012162571635460319, + "d_ape": 3.316459358658715e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.122495651245117, + 20.08835220336914, + 20.119455337524414, + 20.080768585205078, + 20.055072784423828, + 20.056095123291016, + 20.064416885375977 + ], + "critical_prepare_samples_ms": [ + 1.02512096054852, + 0.9624578524380922, + 0.9632399305701256, + 1.0062160436064005, + 0.9819930419325829, + 0.9756379295140505, + 0.9078129660338163, + 0.9824160952121019 + ] + }, + "zigzag": { + "step_ms": 20.79020881652832, + "prepare_ms": 4.162916971836239, + "one_use_ms": 24.95312578836456, + "prepare_min_ms": 4.007034818641841, + "prepare_max_ms": 4.921410931274295, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.000167355533096453, + "dq": 8.740554038293048e-05, + "dkv": 0.0001053261127085974, + "d_index_q": 2.491827918171799e-05, + "d_weights": 2.5499445570320522e-05, + "d_pool_kv": 8.765414636157637e-05, + "d_pool_score": 7.288222095933882e-05, + "d_ape": 2.6757349713227653e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.79020881652832, + 20.970272064208984, + 20.883359909057617, + 20.743200302124023, + 20.754911422729492, + 20.835552215576172, + 20.762752532958984 + ], + "critical_prepare_samples_ms": [ + 4.25333296880126, + 4.007034818641841, + 4.1503519751131535, + 4.179439041763544, + 4.06482198741287, + 4.118405049666762, + 4.921410931274295, + 4.175481968559325 + ] + }, + "planner": { + "step_ms": 20.055456161499023, + "prepare_ms": 1.181907020509243, + "one_use_ms": 21.237363182008266, + "prepare_min_ms": 1.1588750639930367, + "prepare_max_ms": 1.244336017407477, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016034442556637804, + "dq": 8.377134480427006e-05, + "dkv": 0.00012869833642443928, + "d_index_q": 2.4304393144673242e-05, + "d_weights": 3.1874338742783715e-05, + "d_pool_kv": 7.765903597090855e-05, + "d_pool_score": 6.551608904168383e-05, + "d_ape": 2.6567923612493613e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.058752059936523, + 20.152511596679688, + 20.14681625366211, + 20.055456161499023, + 20.03536033630371, + 20.055200576782227, + 20.035104751586914 + ], + "critical_prepare_samples_ms": [ + 1.244336017407477, + 1.1704741045832634, + 1.1588750639930367, + 1.1804300593212247, + 1.1833839816972613, + 1.1672999244183302, + 1.1852069292217493, + 1.2041899608448148 + ] + } + } + }, + "128k-mixed-22": { + "documents": 22, + "tokens": 131072, + "source_record_sha256": "d54ce7bb5382e15f3d8a885c6e53f2d59fb252c6dbc73a8e030ff30ea58fd017", + "policies": { + "contiguous": { + "step_ms": 35.437599182128906, + "prepare_ms": 0.8113369694910944, + "one_use_ms": 36.24893615162, + "prepare_min_ms": 0.7874679286032915, + "prepare_max_ms": 11.578752077184618, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0019160342752732016, + "dq": 0.0005784768780049561, + "dkv": 0.0004627341121441643, + "d_index_q": 0.0003634974114396893, + "d_weights": 0.0003493773692632088, + "d_pool_kv": 0.0006592622470164842, + "d_pool_score": 0.0006617473534238676, + "d_ape": 1.5070293250763499e-05 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 35.15270233154297, + 34.845760345458984, + 35.5722541809082, + 35.437599182128906, + 36.6080322265625, + 35.424896240234375, + 35.540000915527344 + ], + "critical_prepare_samples_ms": [ + 0.8466748986393213, + 0.8107968606054783, + 11.578752077184618, + 0.8092040661722422, + 0.7907270919531584, + 0.8135039824992418, + 0.7874679286032915, + 0.8118770783767104 + ] + }, + "zigzag": { + "step_ms": 35.23379135131836, + "prepare_ms": 1.9275369704701006, + "one_use_ms": 37.16132832178846, + "prepare_min_ms": 1.8797999946400523, + "prepare_max_ms": 3.936605993658304, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0019299163049260265, + "dq": 0.0005823918678050893, + "dkv": 0.0004619453030516524, + "d_index_q": 0.0003622360991686167, + "d_weights": 0.0003449186778998967, + "d_pool_kv": 0.0006607552043101183, + "d_pool_score": 0.0006699460238477643, + "d_ape": 1.4678422801075643e-05 + }, + "max_loss_abs_diff": 8.149072527885437e-10, + "critical_step_samples_ms": [ + 35.07795333862305, + 35.11782455444336, + 35.7696647644043, + 35.212799072265625, + 35.694305419921875, + 35.23379135131836, + 36.08281707763672 + ], + "critical_prepare_samples_ms": [ + 3.936605993658304, + 1.9458660390228033, + 1.9063919316977262, + 1.9079309422522783, + 1.9645949359983206, + 1.9459249451756477, + 1.909207901917398, + 1.8797999946400523 + ] + }, + "planner": { + "step_ms": 35.436702728271484, + "prepare_ms": 7.6061730505898595, + "one_use_ms": 43.042875778861344, + "prepare_min_ms": 7.441366091370583, + "prepare_max_ms": 7.997068925760686, + "selected_policy": "selection-local-3x", + "max_relative_rms": { + "output": 0.001919035177989802, + "dq": 0.0005792319248342236, + "dkv": 0.00046265994814852977, + "d_index_q": 0.00036264936861235027, + "d_weights": 0.00034889076944032783, + "d_pool_kv": 0.0006608845275100428, + "d_pool_score": 0.0006599980949303307, + "d_ape": 1.2610507503426976e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-10, + "critical_step_samples_ms": [ + 34.854976654052734, + 35.8928337097168, + 35.35036849975586, + 35.62351989746094, + 35.61305618286133, + 35.436702728271484, + 35.13798522949219 + ], + "critical_prepare_samples_ms": [ + 7.997068925760686, + 7.598275085911155, + 7.614071015268564, + 7.47464899905026, + 7.661403971724212, + 7.441366091370583, + 7.570564979687333, + 7.846773136407137 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "38e8fdad2096841ce9b57196cdb05dfe4cd17b7c2311cf5bede714c90c5d3a91", + "policies": { + "contiguous": { + "step_ms": 9.287199974060059, + "prepare_ms": 6.7794660571962595, + "one_use_ms": 16.066666031256318, + "prepare_min_ms": 6.7019599955528975, + "prepare_max_ms": 7.129473960958421, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7034465202561103e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.80841349081689e-05, + "d_pool_score": 4.969745146211584e-05, + "d_ape": 2.463488004291754e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.327584266662598, + 9.295040130615234, + 9.287199974060059, + 9.278176307678223, + 9.278143882751465, + 9.258208274841309, + 9.473567962646484 + ], + "critical_prepare_samples_ms": [ + 6.837799912318587, + 6.7019599955528975, + 6.714918999932706, + 6.821699091233313, + 7.129473960958421, + 6.737233023159206, + 6.707626045681536, + 6.954675889573991 + ] + }, + "zigzag": { + "step_ms": 11.102463722229004, + "prepare_ms": 47.22392506664619, + "one_use_ms": 58.32638878887519, + "prepare_min_ms": 45.078030903823674, + "prepare_max_ms": 156.1478169169277, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.723088398789181e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.7408244692422145e-05, + "d_pool_score": 6.371221934010526e-05, + "d_ape": 3.3825512629530646e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.100447654724121, + 11.142175674438477, + 11.123104095458984, + 11.102463722229004, + 11.112383842468262, + 11.097536087036133, + 11.057215690612793 + ], + "critical_prepare_samples_ms": [ + 48.150649992749095, + 46.42646899446845, + 45.078030903823674, + 151.0155029827729, + 47.763709095306695, + 46.68414103798568, + 45.34025501925498, + 156.1478169169277 + ] + }, + "planner": { + "step_ms": 9.295295715332031, + "prepare_ms": 8.97316396003589, + "one_use_ms": 18.26845967536792, + "prepare_min_ms": 8.708094945177436, + "prepare_max_ms": 123.31375910434872, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.9905925067018862e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.216577144844658e-05, + "d_pool_score": 6.720340578861908e-05, + "d_ape": 3.3511195947336086e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.287327766418457, + 9.30742359161377, + 9.270848274230957, + 9.30361557006836, + 9.306976318359375, + 9.284768104553223, + 9.295295715332031 + ], + "critical_prepare_samples_ms": [ + 8.869540877640247, + 8.825646946206689, + 123.31375910434872, + 9.473679936490953, + 8.962570922449231, + 8.708094945177436, + 9.538239915855229, + 8.98375699762255 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "be33650d531537cf859041aa184f7846ca91ee7d6f07866e13f5eff3df7ff982", + "policies": { + "contiguous": { + "step_ms": 10.01318359375, + "prepare_ms": 3.743164998013526, + "one_use_ms": 13.756348591763526, + "prepare_min_ms": 3.6540150176733732, + "prepare_max_ms": 3.838455071672797, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 4.31067005302863e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.485686835600299e-05, + "d_pool_score": 3.734317152918191e-05, + "d_ape": 1.5884632522867948e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 10.00160026550293, + 10.01318359375, + 9.96771240234375, + 10.067392349243164, + 9.997440338134766, + 10.017472267150879, + 10.042176246643066 + ], + "critical_prepare_samples_ms": [ + 3.793359035626054, + 3.752059070393443, + 3.740546992048621, + 3.739589941687882, + 3.700686036609113, + 3.7457830039784312, + 3.6540150176733732, + 3.838455071672797 + ] + }, + "zigzag": { + "step_ms": 11.205792427062988, + "prepare_ms": 24.085226003080606, + "one_use_ms": 35.291018430143595, + "prepare_min_ms": 23.83457194082439, + "prepare_max_ms": 139.23344609793276, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7925220371458967e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 7.136588948385061e-05, + "d_pool_score": 4.065888036840021e-05, + "d_ape": 1.7333242400394557e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.202655792236328, + 11.212991714477539, + 11.187968254089355, + 11.230527877807617, + 11.205792427062988, + 11.16857624053955, + 11.311712265014648 + ], + "critical_prepare_samples_ms": [ + 27.768890955485404, + 25.140583980828524, + 24.11484904587269, + 23.88202305883169, + 24.055602960288525, + 24.04588391073048, + 23.83457194082439, + 139.23344609793276 + ] + }, + "planner": { + "step_ms": 10.027359962463379, + "prepare_ms": 4.955698561388999, + "one_use_ms": 14.983058523852378, + "prepare_min_ms": 4.824013100005686, + "prepare_max_ms": 5.206209025345743, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3273367675160776e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.542720423634009e-05, + "d_pool_score": 5.10650784981466e-05, + "d_ape": 2.1403371983524285e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 10.017024040222168, + 10.369183540344238, + 10.027359962463379, + 10.05395221710205, + 10.025664329528809, + 10.148415565490723, + 10.026752471923828 + ], + "critical_prepare_samples_ms": [ + 5.206209025345743, + 5.033118999563158, + 4.875925951637328, + 4.824013100005686, + 4.920054925605655, + 4.9812550423666835, + 4.930142080411315, + 5.035332054831088 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "a42d7db2614d9e21c3bd24a8de34199977030c3a75e1b28856d51a604529dfdb", + "policies": { + "contiguous": { + "step_ms": 33.07494354248047, + "prepare_ms": 7.641692471224815, + "one_use_ms": 40.71663601370528, + "prepare_min_ms": 7.504745037294924, + "prepare_max_ms": 110.33575492911041, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002124358368844286, + "dq": 0.0006708973533251172, + "dkv": 0.0006748384557586351, + "d_index_q": 0.000419026902945799, + "d_weights": 0.0003304557285110155, + "d_pool_kv": 0.0005772261277278353, + "d_pool_score": 0.0005797688319552085, + "d_ape": 7.489935465690975e-06 + }, + "max_loss_abs_diff": 3.4924596548080444e-10, + "critical_step_samples_ms": [ + 33.100799560546875, + 33.073055267333984, + 33.057918548583984, + 33.08787155151367, + 33.07494354248047, + 33.04643249511719, + 33.11958312988281 + ], + "critical_prepare_samples_ms": [ + 8.83635098580271, + 7.582117919810116, + 110.33575492911041, + 7.812974974513054, + 7.556077907793224, + 7.6242099748924375, + 7.504745037294924, + 7.659174967557192 + ] + }, + "zigzag": { + "step_ms": 38.74604797363281, + "prepare_ms": 45.72551843011752, + "one_use_ms": 84.47156640375033, + "prepare_min_ms": 44.49985595420003, + "prepare_max_ms": 152.91648893617094, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.002136063325688249, + "dq": 0.0006448208867270936, + "dkv": 0.0005397046574182863, + "d_index_q": 0.0003537235166520844, + "d_weights": 0.0003170419424130787, + "d_pool_kv": 0.0005661726253782145, + "d_pool_score": 0.0005473705393115235, + "d_ape": 7.236730944585608e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.72604751586914, + 38.74604797363281, + 38.78224182128906, + 38.78335952758789, + 38.72016143798828, + 38.75788879394531, + 38.730464935302734 + ], + "critical_prepare_samples_ms": [ + 47.61352890636772, + 45.467405929230154, + 44.49985595420003, + 152.91648893617094, + 45.98363093100488, + 45.106612145900726, + 44.917382998391986, + 145.95098292920738 + ] + }, + "planner": { + "step_ms": 33.0742073059082, + "prepare_ms": 28.312876995187253, + "one_use_ms": 61.387084301095456, + "prepare_min_ms": 28.01844000350684, + "prepare_max_ms": 123.6400690395385, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0020962956426669433, + "dq": 0.0006347467076768539, + "dkv": 0.0005276886282440079, + "d_index_q": 0.00035682941768751356, + "d_weights": 0.00032945679607176856, + "d_pool_kv": 0.0005648985794352423, + "d_pool_score": 0.0005625030405139739, + "d_ape": 6.634011369785548e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 33.05827331542969, + 33.0742073059082, + 33.59049606323242, + 33.06959915161133, + 33.093441009521484, + 33.08927917480469, + 33.03887939453125 + ], + "critical_prepare_samples_ms": [ + 28.298633988015354, + 28.01844000350684, + 123.6400690395385, + 28.612435911782086, + 28.067155042663217, + 28.327120002359152, + 28.597721946425736, + 28.228612965904176 + ] + } + } + }, + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "1ce20ae25f95e468a8ceb74f0607b96cd4b3c96afba3f565edacffb45121c460", + "policies": { + "contiguous": { + "step_ms": 32.37452697753906, + "prepare_ms": 7.578457531053573, + "one_use_ms": 39.952984508592635, + "prepare_min_ms": 7.474332000128925, + "prepare_max_ms": 12.586126918904483, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015154178483681854, + "dq": 0.0004590856367981372, + "dkv": 0.00046865559368039917, + "d_index_q": 0.0003544140670990448, + "d_weights": 0.0003176278618649191, + "d_pool_kv": 0.0005387196908820256, + "d_pool_score": 0.0005430168708674836, + "d_ape": 4.7473252849530434e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.37452697753906, + 32.392513275146484, + 32.36988830566406, + 32.39535903930664, + 32.35891342163086, + 32.35808181762695, + 32.382911682128906 + ], + "critical_prepare_samples_ms": [ + 7.727506919763982, + 7.595576927997172, + 7.546272943727672, + 12.586126918904483, + 7.49861600343138, + 7.561338134109974, + 7.474332000128925, + 7.61300593148917 + ] + }, + "zigzag": { + "step_ms": 37.90224075317383, + "prepare_ms": 45.84232851630077, + "one_use_ms": 83.7445692694746, + "prepare_min_ms": 44.70354691147804, + "prepare_max_ms": 155.66171507816762, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016252246640432245, + "dq": 0.0004916736119971625, + "dkv": 0.0004861918573380407, + "d_index_q": 0.0003569257876233377, + "d_weights": 0.0003177650459406723, + "d_pool_kv": 0.0005486278732828732, + "d_pool_score": 0.0005445220484194565, + "d_ape": 3.860767227143574e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 37.866241455078125, + 37.90224075317383, + 37.91574478149414, + 37.9218864440918, + 37.847808837890625, + 37.96307373046875, + 37.85862350463867 + ], + "critical_prepare_samples_ms": [ + 47.00984014198184, + 45.00916204415262, + 44.70354691147804, + 153.49186910316348, + 46.21615505311638, + 45.468501979485154, + 44.9322130298242, + 155.66171507816762 + ] + }, + "planner": { + "step_ms": 32.353633880615234, + "prepare_ms": 12.12866249261424, + "one_use_ms": 44.482296373229474, + "prepare_min_ms": 11.894998839125037, + "prepare_max_ms": 123.44329594634473, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015138876329661513, + "dq": 0.00045860825477340696, + "dkv": 0.00046849135874668684, + "d_index_q": 0.0003562482112602212, + "d_weights": 0.00032118335948173643, + "d_pool_kv": 0.0005438274579043697, + "d_pool_score": 0.0005519648229923422, + "d_ape": 4.258189850384637e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.353633880615234, + 32.412353515625, + 32.39923095703125, + 32.37456130981445, + 32.34396743774414, + 32.31990432739258, + 32.341121673583984 + ], + "critical_prepare_samples_ms": [ + 12.077874038368464, + 12.179450946860015, + 123.44329594634473, + 12.385182897560298, + 11.991281062364578, + 12.406638008542359, + 11.894998839125037, + 11.912805959582329 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "27e077b4ff55aa00c71a035377fafcb97d9db8bfb198c213673c877e5ded916b", + "policies": { + "contiguous": { + "step_ms": 115.2938232421875, + "prepare_ms": 1.1161149595864117, + "one_use_ms": 116.40993820177391, + "prepare_min_ms": 1.087293028831482, + "prepare_max_ms": 1.304103061556816, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014842079275755727, + "dq": 0.0005035188023387811, + "dkv": 0.0010278174526612903, + "d_index_q": 0.0006256382167037119, + "d_weights": 0.000563762569940494, + "d_pool_kv": 0.0013303364084540658, + "d_pool_score": 0.001377639655655214, + "d_ape": 2.0452408219956506e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 114.4939193725586, + 115.84454345703125, + 115.2938232421875, + 114.8194580078125, + 116.123779296875, + 114.58399963378906, + 115.50758361816406 + ], + "critical_prepare_samples_ms": [ + 1.304103061556816, + 1.1089229956269264, + 1.0957310441881418, + 1.1007870780304074, + 1.1440730886533856, + 1.123306923545897, + 1.087293028831482, + 1.187960966490209 + ] + }, + "zigzag": { + "step_ms": 105.58483123779297, + "prepare_ms": 1.1357140028849244, + "one_use_ms": 106.7205452406779, + "prepare_min_ms": 1.1315711308270693, + "prepare_max_ms": 2.692296984605491, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014841542205465054, + "dq": 0.00050380319214946, + "dkv": 0.0010287937173898806, + "d_index_q": 0.0006482205309808245, + "d_weights": 0.000583151648179517, + "d_pool_kv": 0.0013331964396419756, + "d_pool_score": 0.0013779578045436441, + "d_ape": 1.9963974118421826e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 106.04412841796875, + 105.58483123779297, + 105.72246551513672, + 105.1151351928711, + 105.20396423339844, + 106.02706909179688, + 105.4378890991211 + ], + "critical_prepare_samples_ms": [ + 2.692296984605491, + 1.1655869893729687, + 1.1315711308270693, + 1.1326848762109876, + 1.1354410089552402, + 1.15663290489465, + 1.1359869968146086, + 1.1334370356053114 + ] + }, + "planner": { + "step_ms": 104.14985656738281, + "prepare_ms": 7.791060954332352, + "one_use_ms": 111.94091752171516, + "prepare_min_ms": 7.6777610229328275, + "prepare_max_ms": 8.784707984887064, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014818148634560368, + "dq": 0.0005030012855905718, + "dkv": 0.0010310600226582155, + "d_index_q": 0.0007705202387904052, + "d_weights": 0.0006243137936169513, + "d_pool_kv": 0.0013332458370744855, + "d_pool_score": 0.0013751792972335336, + "d_ape": 2.096916046719033e-05 + }, + "max_loss_abs_diff": 2.0372681319713593e-10, + "critical_step_samples_ms": [ + 103.95587158203125, + 104.33695983886719, + 103.87484741210938, + 104.14985656738281, + 104.16598510742188, + 104.65894317626953, + 104.02249908447266 + ], + "critical_prepare_samples_ms": [ + 7.850671885535121, + 8.708197972737253, + 7.70237494725734, + 7.6777610229328275, + 8.784707984887064, + 7.718592998571694, + 7.731450023129582, + 8.389059104956686 + ] + } + } + }, + "256k-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "b206d1914825ce27ceb8f280beaa9efb8b8c2d6e09e8192df5fe030a36b95f42", + "policies": { + "contiguous": { + "step_ms": 50.3370246887207, + "prepare_ms": 1.0602579568512738, + "one_use_ms": 51.39728264557198, + "prepare_min_ms": 1.0142670944333076, + "prepare_max_ms": 1.16502505261451, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008995063625665686, + "dq": 0.000417439995711948, + "dkv": 0.00029408694183826416, + "d_index_q": 0.00010925775240076204, + "d_weights": 9.570165139064877e-05, + "d_pool_kv": 0.0002036231762018183, + "d_pool_score": 0.00023355335680617782, + "d_ape": 7.1891911871349595e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.318817138671875, + 50.3370246887207, + 50.29235076904297, + 50.279136657714844, + 50.51945495605469, + 50.36774444580078, + 50.34364700317383 + ], + "critical_prepare_samples_ms": [ + 1.16502505261451, + 1.0669470066204667, + 1.0217679664492607, + 1.0694119846448302, + 1.0273349471390247, + 1.0535689070820808, + 1.0142670944333076, + 1.0685940505936742 + ] + }, + "zigzag": { + "step_ms": 51.72915267944336, + "prepare_ms": 4.144389531575143, + "one_use_ms": 55.8735422110185, + "prepare_min_ms": 4.095686017535627, + "prepare_max_ms": 4.8910570330917835, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009047350578015278, + "dq": 0.00041966050461122436, + "dkv": 0.00029402900005215225, + "d_index_q": 0.00010933438743714884, + "d_weights": 9.557770476272342e-05, + "d_pool_kv": 0.0001933948013080206, + "d_pool_score": 0.0001997392626168786, + "d_ape": 6.032497424065086e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 51.72719955444336, + 51.7239990234375, + 51.834590911865234, + 51.799041748046875, + 51.7083854675293, + 51.72915267944336, + 51.82553482055664 + ], + "critical_prepare_samples_ms": [ + 4.361170926131308, + 4.214552929624915, + 4.100008052773774, + 4.132551024667919, + 4.152706009335816, + 4.095686017535627, + 4.8910570330917835, + 4.136073053814471 + ] + }, + "planner": { + "step_ms": 50.334686279296875, + "prepare_ms": 1.275484450161457, + "one_use_ms": 51.61017072945833, + "prepare_min_ms": 1.2338400119915605, + "prepare_max_ms": 1.3704260345548391, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008968787811652317, + "dq": 0.00041599997551765766, + "dkv": 0.0002978264449580809, + "d_index_q": 0.00010914478084630114, + "d_weights": 9.54750575422348e-05, + "d_pool_kv": 0.00019495316721958097, + "d_pool_score": 0.00020016815004170784, + "d_ape": 6.878236502860825e-06 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 50.27433776855469, + 50.653377532958984, + 50.30992126464844, + 50.319297790527344, + 50.40998458862305, + 50.47235107421875, + 50.334686279296875 + ], + "critical_prepare_samples_ms": [ + 1.3704260345548391, + 1.2794609647244215, + 1.2759730452671647, + 1.2749958550557494, + 1.2793471105396748, + 1.2338400119915605, + 1.27033784519881, + 1.248182961717248 + ] + } + } + }, + "256k-near-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "ccc7f36371b2ca093244df96efefd898c4c05ac4599ade1e2027012193b8ea43", + "policies": { + "contiguous": { + "step_ms": 50.341758728027344, + "prepare_ms": 1.0578265646472573, + "one_use_ms": 51.3995852926746, + "prepare_min_ms": 1.0304378811269999, + "prepare_max_ms": 2.42595502641052, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009040294318661402, + "dq": 0.0004176309327453784, + "dkv": 0.00028700438317877033, + "d_index_q": 0.00010909463263037384, + "d_weights": 9.385798875582676e-05, + "d_pool_kv": 0.0002099997308788692, + "d_pool_score": 0.00021174261363919708, + "d_ape": 6.975002356106306e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.3015022277832, + 50.3072624206543, + 50.37491226196289, + 50.39254379272461, + 50.31679916381836, + 50.34892654418945, + 50.341758728027344 + ], + "critical_prepare_samples_ms": [ + 2.42595502641052, + 1.056926092132926, + 1.0342529276385903, + 1.0686619207262993, + 1.0407939553260803, + 1.0587270371615887, + 1.0304378811269999, + 1.0622970294207335 + ] + }, + "zigzag": { + "step_ms": 51.770111083984375, + "prepare_ms": 4.253186983987689, + "one_use_ms": 56.023298067972064, + "prepare_min_ms": 4.211014020256698, + "prepare_max_ms": 4.994260962121189, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009021376190146195, + "dq": 0.00042020065818351493, + "dkv": 0.0003025562304355097, + "d_index_q": 0.00010794658869771671, + "d_weights": 9.589076007313737e-05, + "d_pool_kv": 0.00021834571632293082, + "d_pool_score": 0.00021545790675349828, + "d_ape": 7.255162179958939e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 51.72880172729492, + 51.770111083984375, + 51.73446273803711, + 51.797088623046875, + 51.87964630126953, + 51.770816802978516, + 51.68940734863281 + ], + "critical_prepare_samples_ms": [ + 4.418599070049822, + 4.292018827982247, + 4.211014020256698, + 4.258326953276992, + 4.248047014698386, + 4.212184925563633, + 4.994260962121189, + 4.242594935931265 + ] + }, + "planner": { + "step_ms": 50.34764862060547, + "prepare_ms": 1.2725520064122975, + "one_use_ms": 51.620200627017766, + "prepare_min_ms": 1.2156248558312654, + "prepare_max_ms": 1.4591740909963846, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009001018368343759, + "dq": 0.0004180679445311549, + "dkv": 0.0002908204232464405, + "d_index_q": 0.00010824020032508636, + "d_weights": 9.559161306044745e-05, + "d_pool_kv": 0.0002088096055712727, + "d_pool_score": 0.00022862128266242155, + "d_ape": 7.098986447802436e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 50.308414459228516, + 50.31196975708008, + 50.657249450683594, + 50.34764862060547, + 50.37120056152344, + 50.28732681274414, + 50.731998443603516 + ], + "critical_prepare_samples_ms": [ + 1.3486190000548959, + 1.4591740909963846, + 1.2668580748140812, + 1.2407059548422694, + 1.282125012949109, + 1.2156248558312654, + 1.2782459380105138, + 1.236111973412335 + ] + } + } + }, + "256k-mixed-22": { + "documents": 22, + "tokens": 262144, + "source_record_sha256": "db3e840999278944a7dc0c815b5832e7cf601e9dcaa7db9db4627377f326570b", + "policies": { + "contiguous": { + "step_ms": 90.6918716430664, + "prepare_ms": 1.150823023635894, + "one_use_ms": 91.8426946667023, + "prepare_min_ms": 1.115339924581349, + "prepare_max_ms": 1.3260958949103951, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014396839839641682, + "dq": 0.0004912932869663864, + "dkv": 0.000782527660447053, + "d_index_q": 0.0007096421066149192, + "d_weights": 0.0004776414729478394, + "d_pool_kv": 0.001229747392195446, + "d_pool_score": 0.0012867899272477345, + "d_ape": 1.346208842230701e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 91.04662322998047, + 90.35794830322266, + 90.62847900390625, + 90.47663879394531, + 92.35906982421875, + 90.6918716430664, + 91.08707427978516 + ], + "critical_prepare_samples_ms": [ + 1.3260958949103951, + 1.1735139414668083, + 1.1378390481695533, + 1.1485170107334852, + 1.115339924581349, + 1.1682850308716297, + 1.1497350642457604, + 1.1519109830260277 + ] + }, + "zigzag": { + "step_ms": 87.94976043701172, + "prepare_ms": 2.339682076126337, + "one_use_ms": 90.28944251313806, + "prepare_min_ms": 2.3260059533640742, + "prepare_max_ms": 5.5571189150214195, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014655759967181566, + "dq": 0.0005149950687110133, + "dkv": 0.0007858004056596082, + "d_index_q": 0.0007967333850337871, + "d_weights": 0.0006135588148793877, + "d_pool_kv": 0.0012313505882381754, + "d_pool_score": 0.0012816470119322591, + "d_ape": 1.3536862404843282e-05 + }, + "max_loss_abs_diff": 5.238689482212067e-10, + "critical_step_samples_ms": [ + 87.89424133300781, + 87.9170913696289, + 88.21635437011719, + 87.94976043701172, + 87.63407897949219, + 88.22838592529297, + 88.08528137207031 + ], + "critical_prepare_samples_ms": [ + 3.682921058498323, + 5.5571189150214195, + 2.3260059533640742, + 2.33569392003119, + 2.3404060630127788, + 2.3389580892398953, + 2.3376650642603636, + 2.347580040805042 + ] + }, + "planner": { + "step_ms": 87.38617706298828, + "prepare_ms": 8.772762026637793, + "one_use_ms": 96.15893908962607, + "prepare_min_ms": 8.418329060077667, + "prepare_max_ms": 9.713605977594852, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0014719262651841122, + "dq": 0.0005169495137705205, + "dkv": 0.0005843813325958284, + "d_index_q": 0.000568126948707376, + "d_weights": 0.0005875102674471812, + "d_pool_kv": 0.0006705463099119529, + "d_pool_score": 0.0006839089222165456, + "d_ape": 1.3325801383008022e-05 + }, + "max_loss_abs_diff": 4.0745362639427185e-10, + "critical_step_samples_ms": [ + 87.38617706298828, + 87.18109130859375, + 87.44198608398438, + 87.81423950195312, + 87.28131103515625, + 87.66595458984375, + 87.18303680419922 + ], + "critical_prepare_samples_ms": [ + 8.892927085980773, + 8.418329060077667, + 9.39794396981597, + 8.462049067020416, + 9.713605977594852, + 8.53787490632385, + 9.19189490377903, + 8.652596967294812 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "a3e04775cd4674a149c44452e86572771d80fd867f99c915f4d8703ea7cf55c1", + "policies": { + "contiguous": { + "step_ms": 18.96121597290039, + "prepare_ms": 6.907838920596987, + "one_use_ms": 25.869054893497378, + "prepare_min_ms": 6.844045943580568, + "prepare_max_ms": 7.187637034803629, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.724041124260225e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.511801279303253e-05, + "d_pool_score": 5.232610864426897e-05, + "d_ape": 1.4967124181438183e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 19.019712448120117, + 18.97760009765625, + 18.96121597290039, + 18.9564151763916, + 18.95644760131836, + 18.9434871673584, + 19.144479751586914 + ], + "critical_prepare_samples_ms": [ + 7.187637034803629, + 6.97691401001066, + 6.844045943580568, + 6.993726012296975, + 6.876001018099487, + 6.887514842674136, + 6.885394104756415, + 6.928162998519838 + ] + }, + "zigzag": { + "step_ms": 22.557504653930664, + "prepare_ms": 46.86611460056156, + "one_use_ms": 69.42361925449222, + "prepare_min_ms": 46.30116908811033, + "prepare_max_ms": 167.44421899784356, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8637480027866625e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.271880161331187e-05, + "d_pool_score": 5.610531234471807e-05, + "d_ape": 1.5175327931420397e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 22.557504653930664, + 22.60767936706543, + 22.548383712768555, + 22.567264556884766, + 22.551103591918945, + 22.51372718811035, + 22.592575073242188 + ], + "critical_prepare_samples_ms": [ + 47.66656889114529, + 46.35808803141117, + 46.458124998025596, + 167.44421899784356, + 47.36804007552564, + 47.025946085341275, + 46.30116908811033, + 46.706283115781844 + ] + }, + "planner": { + "step_ms": 18.94291114807129, + "prepare_ms": 8.967155939899385, + "one_use_ms": 27.910067087970674, + "prepare_min_ms": 8.858141023665667, + "prepare_max_ms": 14.381637098267674, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.590593616004352e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.534328689387801e-05, + "d_pool_score": 5.3623912772759155e-05, + "d_ape": 1.7238388124362549e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 18.938079833984375, + 18.94291114807129, + 18.93289566040039, + 18.952415466308594, + 18.924928665161133, + 18.946847915649414, + 19.089344024658203 + ], + "critical_prepare_samples_ms": [ + 9.043016005307436, + 8.858141023665667, + 8.99953895714134, + 8.93477292265743, + 9.00569697842002, + 8.906646049581468, + 8.866342948749661, + 14.381637098267674 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "3abdfc48993cc3747d2f08fcfbd801e8c6fa1abaa8a607d585da37c318d9b529", + "policies": { + "contiguous": { + "step_ms": 21.785919189453125, + "prepare_ms": 3.9414139464497566, + "one_use_ms": 25.72733313590288, + "prepare_min_ms": 3.8159850519150496, + "prepare_max_ms": 4.042547079734504, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.297053926215977e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.412638872669938e-05, + "d_pool_score": 4.495461915173074e-05, + "d_ape": 1.4301480341692518e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.769792556762695, + 21.76304054260254, + 22.114463806152344, + 21.785919189453125, + 21.940128326416016, + 21.767871856689453, + 21.940895080566406 + ], + "critical_prepare_samples_ms": [ + 4.042547079734504, + 3.9293068693950772, + 3.8159850519150496, + 3.9682361530140042, + 3.8398168981075287, + 3.9214659482240677, + 3.953521023504436, + 3.995029954239726 + ] + }, + "zigzag": { + "step_ms": 24.209856033325195, + "prepare_ms": 24.67734646052122, + "one_use_ms": 48.88720249384642, + "prepare_min_ms": 24.372468004003167, + "prepare_max_ms": 142.0474809128791, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2708240134177204e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.3500598722775094e-05, + "d_pool_score": 5.347177795761659e-05, + "d_ape": 1.327084474519006e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 24.196128845214844, + 24.229408264160156, + 24.214399337768555, + 24.209856033325195, + 24.24527931213379, + 24.169696807861328, + 24.205184936523438 + ], + "critical_prepare_samples_ms": [ + 26.387990918010473, + 24.77585698943585, + 24.372468004003167, + 24.47995904367417, + 24.4268769165501, + 24.750579963438213, + 24.60411295760423, + 142.0474809128791 + ] + }, + "planner": { + "step_ms": 21.7589111328125, + "prepare_ms": 5.023348960094154, + "one_use_ms": 26.782260092906654, + "prepare_min_ms": 4.9473929684609175, + "prepare_max_ms": 5.072213942185044, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.554213974193077e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.1905810341815174e-05, + "d_pool_score": 4.350734098547027e-05, + "d_ape": 1.4490775323448985e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.780927658081055, + 21.7589111328125, + 21.778976440429688, + 21.859455108642578, + 21.743295669555664, + 21.751680374145508, + 21.750688552856445 + ], + "critical_prepare_samples_ms": [ + 5.0161018734797835, + 5.0161059480160475, + 5.0472968723624945, + 5.041101947426796, + 4.989685956388712, + 5.03059197217226, + 5.072213942185044, + 4.9473929684609175 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "299ffffd41e8b2e4c2be973c6bdb05ae54dc4a991bc3051df8ee4e0a6127615f", + "policies": { + "contiguous": { + "step_ms": 83.6385269165039, + "prepare_ms": 9.585648542270064, + "one_use_ms": 93.22417545877397, + "prepare_min_ms": 9.470761171542108, + "prepare_max_ms": 109.36465603299439, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017248119636322726, + "dq": 0.0005231477545538955, + "dkv": 0.0008859578942512667, + "d_index_q": 0.0003679247450818875, + "d_weights": 0.0003583038720109353, + "d_pool_kv": 0.0013756446763457486, + "d_pool_score": 0.0013569902061103135, + "d_ape": 9.213348057789348e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 83.79923248291016, + 83.6385269165039, + 83.57939147949219, + 83.14595031738281, + 84.14755249023438, + 83.68710327148438, + 83.60105895996094 + ], + "critical_prepare_samples_ms": [ + 11.086768005043268, + 9.873543865978718, + 9.49363096151501, + 9.626674116589129, + 9.470761171542108, + 9.488550946116447, + 109.36465603299439, + 9.544622967951 + ] + }, + "zigzag": { + "step_ms": 96.04755401611328, + "prepare_ms": 92.28663396788761, + "one_use_ms": 188.3341879840009, + "prepare_min_ms": 46.753630973398685, + "prepare_max_ms": 166.36592405848205, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017395277068004997, + "dq": 0.0005277805775999957, + "dkv": 0.0006782649359844867, + "d_index_q": 0.0003691170318294211, + "d_weights": 0.0003624685324509999, + "d_pool_kv": 0.0007577977882457726, + "d_pool_score": 0.0007313251119719116, + "d_ape": 8.559385750338412e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 96.06777954101562, + 96.25202941894531, + 96.04755401611328, + 95.81129455566406, + 95.63484954833984, + 96.13632202148438, + 95.58025360107422 + ], + "critical_prepare_samples_ms": [ + 48.80867002066225, + 166.36592405848205, + 149.53875401988626, + 48.5578749794513, + 46.753630973398685, + 135.76459791511297, + 152.24266005679965, + 47.11186105851084 + ] + }, + "planner": { + "step_ms": 82.14995574951172, + "prepare_ms": 32.668465457390994, + "one_use_ms": 114.81842120690271, + "prepare_min_ms": 32.24971203599125, + "prepare_max_ms": 33.218687982298434, + "selected_policy": "selection-local-5x", + "max_relative_rms": { + "output": 0.0017299188701990226, + "dq": 0.0005249688471103487, + "dkv": 0.0006932649020557219, + "d_index_q": 0.0003879256230062098, + "d_weights": 0.0003670382435938008, + "d_pool_kv": 0.0013198355292136361, + "d_pool_score": 0.0013151740439955597, + "d_ape": 8.808128280757483e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 81.99295806884766, + 82.14995574951172, + 82.31292724609375, + 82.3354263305664, + 81.54524993896484, + 82.6478042602539, + 81.62854766845703 + ], + "critical_prepare_samples_ms": [ + 32.491752062924206, + 32.684653997421265, + 32.89240214508027, + 32.540695974603295, + 32.65227691736072, + 33.04584801662713, + 32.24971203599125, + 33.218687982298434 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "f6f2fb90fa19746799d7f723f65a57a1d55b4e768a2fbaff20ba8977a77dfb5e", + "policies": { + "contiguous": { + "step_ms": 99.36064147949219, + "prepare_ms": 9.696168475784361, + "one_use_ms": 109.05680995527655, + "prepare_min_ms": 9.520240011624992, + "prepare_max_ms": 10.234966059215367, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010563700585577058, + "dq": 0.00035831076606808053, + "dkv": 0.0004733622916756713, + "d_index_q": 0.0005939123996360771, + "d_weights": 0.0005024215185567405, + "d_pool_kv": 0.000752053852842638, + "d_pool_score": 0.0007688502326254845, + "d_ape": 6.434388548257363e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 98.40025329589844, + 99.20460510253906, + 100.11779022216797, + 99.36064147949219, + 99.45503997802734, + 98.65779113769531, + 99.67919921875 + ], + "critical_prepare_samples_ms": [ + 9.97902499511838, + 9.690101025626063, + 9.577112039551139, + 9.55722399521619, + 9.520240011624992, + 9.813543991185725, + 10.234966059215367, + 9.70223592594266 + ] + }, + "zigzag": { + "step_ms": 115.25299072265625, + "prepare_ms": 147.0409840112552, + "one_use_ms": 262.29397473391145, + "prepare_min_ms": 47.70176007878035, + "prepare_max_ms": 167.99743298906833, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001096030426138749, + "dq": 0.00036871769863560327, + "dkv": 0.0005522801000187844, + "d_index_q": 0.000646529994069314, + "d_weights": 0.0005200898687794669, + "d_pool_kv": 0.0007782778462285144, + "d_pool_score": 0.0007902708796079504, + "d_ape": 6.243475107842311e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 115.171875, + 115.26223754882812, + 115.35302734375, + 115.21993255615234, + 115.44630432128906, + 115.25299072265625, + 115.22217559814453 + ], + "critical_prepare_samples_ms": [ + 137.1572030475363, + 154.45262903813273, + 167.99743298906833, + 48.344484996050596, + 47.70176007878035, + 143.54554004967213, + 155.44524206779897, + 150.53642797283828 + ] + }, + "planner": { + "step_ms": 92.2047348022461, + "prepare_ms": 31.8146159988828, + "one_use_ms": 124.0193508011289, + "prepare_min_ms": 31.22708504088223, + "prepare_max_ms": 132.93269090354443, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.001048036358338339, + "dq": 0.00036006689300403495, + "dkv": 0.0004952787980932682, + "d_index_q": 0.0005006808620709585, + "d_weights": 0.00047204477185006934, + "d_pool_kv": 0.0006647742697551269, + "d_pool_score": 0.0006600247570603834, + "d_ape": 6.129309354351373e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 91.95887756347656, + 92.06761932373047, + 91.93360137939453, + 92.53968048095703, + 92.28575897216797, + 92.2804183959961, + 92.2047348022461 + ], + "critical_prepare_samples_ms": [ + 31.714363023638725, + 126.031952095218, + 31.22708504088223, + 31.914868974126875, + 31.32477297913283, + 31.713226926513016, + 132.93269090354443, + 32.04708802513778 + ] + } + } + }, + "1024k-single": { + "documents": 1, + "tokens": 1048576, + "source_record_sha256": "f8f59d7261da96dacfa2bc6680f2e607ee1272751d31448f68672deee7924dcc", + "policies": { + "contiguous": { + "step_ms": 853.462646484375, + "prepare_ms": 2.235381514765322, + "one_use_ms": 855.6980279991403, + "prepare_min_ms": 2.180504030548036, + "prepare_max_ms": 2.6538941310718656, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011390925500643692, + "dq": 0.00036677751548457586, + "dkv": 0.00045029686879957095, + "d_index_q": 0.0005475285484143609, + "d_weights": 0.0005234083334124328, + "d_pool_kv": 0.0006765433793383455, + "d_pool_score": 0.000669507534555031, + "d_ape": 1.201251494769134e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 848.7756958007812, + 851.5589599609375, + 854.1879272460938, + 853.2410888671875, + 853.76123046875, + 853.462646484375, + 854.3615112304688 + ], + "critical_prepare_samples_ms": [ + 2.6538941310718656, + 2.25232494994998, + 2.2514740703627467, + 2.2216499783098698, + 2.2194701014086604, + 2.229652018286288, + 2.180504030548036, + 2.2411110112443566 + ] + }, + "zigzag": { + "step_ms": 638.503173828125, + "prepare_ms": 2.2861745092086494, + "one_use_ms": 640.7893483373336, + "prepare_min_ms": 2.2653070045635104, + "prepare_max_ms": 5.670589045621455, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011415486958252505, + "dq": 0.00036769420295685456, + "dkv": 0.0004914265330137578, + "d_index_q": 0.000556942590103184, + "d_weights": 0.000536722297153423, + "d_pool_kv": 0.0006794240688639852, + "d_pool_score": 0.0006762446273154329, + "d_ape": 1.2332330725673362e-05 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 638.0089721679688, + 638.694091796875, + 640.7294921875, + 635.8269653320312, + 638.503173828125, + 637.2936401367188, + 639.0613403320312 + ], + "critical_prepare_samples_ms": [ + 5.670589045621455, + 2.431723987683654, + 2.2653070045635104, + 2.296129008755088, + 2.2838700097054243, + 2.2884790087118745, + 2.2718310356140137, + 2.270005992613733 + ] + }, + "planner": { + "step_ms": 637.5537719726562, + "prepare_ms": 9.320752928033471, + "one_use_ms": 646.8745249006897, + "prepare_min_ms": 8.959550876170397, + "prepare_max_ms": 10.955121018923819, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0011399811891549397, + "dq": 0.0003671549652483216, + "dkv": 0.0004688434719635236, + "d_index_q": 0.00054714713668071, + "d_weights": 0.0005235608730508459, + "d_pool_kv": 0.0007342648722923458, + "d_pool_score": 0.0007281192925473291, + "d_ape": 1.2245290306330182e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 637.5820922851562, + 637.1992797851562, + 637.2449951171875, + 637.8410034179688, + 636.2861938476562, + 637.5537719726562, + 641.8402709960938 + ], + "critical_prepare_samples_ms": [ + 10.955121018923819, + 10.685780085623264, + 9.343747980892658, + 9.170369943603873, + 9.297757875174284, + 8.959550876170397, + 9.293015929870307, + 9.915986098349094 + ] + } + } + }, + "1024k-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "467788be0454c5eb43ec073cd768122a5221cca5675c0bb30ce968068c6245a9", + "policies": { + "contiguous": { + "step_ms": 239.31231689453125, + "prepare_ms": 1.490010938141495, + "one_use_ms": 240.80232783267274, + "prepare_min_ms": 1.4407910639420152, + "prepare_max_ms": 1.7354709561914206, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015971718647385203, + "dq": 0.0005946386006291191, + "dkv": 0.0004893087559395691, + "d_index_q": 0.0002655780560894656, + "d_weights": 0.00020677860634936432, + "d_pool_kv": 0.0002909984083585693, + "d_pool_score": 0.00030601339199424316, + "d_ape": 6.1058968008983315e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 239.48367309570312, + 238.9126739501953, + 239.31820678710938, + 239.31231689453125, + 239.07810974121094, + 239.46946716308594, + 238.660888671875 + ], + "critical_prepare_samples_ms": [ + 1.7354709561914206, + 1.5307000139728189, + 1.4793298905715346, + 1.4931149780750275, + 1.4869068982079625, + 1.4774100854992867, + 1.4407910639420152, + 1.4997579855844378 + ] + }, + "zigzag": { + "step_ms": 244.19993591308594, + "prepare_ms": 4.6627100091427565, + "one_use_ms": 248.8626459222287, + "prepare_min_ms": 4.619401996023953, + "prepare_max_ms": 6.727989995852113, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.00161892792378639, + "dq": 0.00061183109697404, + "dkv": 0.0005255603590014857, + "d_index_q": 0.00027796173921266023, + "d_weights": 0.00021531215731985807, + "d_pool_kv": 0.00028469963192253694, + "d_pool_score": 0.00029151841170628354, + "d_ape": 6.315599964674472e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 244.90960693359375, + 243.77923583984375, + 245.494140625, + 244.19993591308594, + 243.84393310546875, + 244.16079711914062, + 244.61228942871094 + ], + "critical_prepare_samples_ms": [ + 6.727989995852113, + 4.855223116464913, + 4.65498398989439, + 4.665578017011285, + 4.734496003948152, + 4.624465946108103, + 4.659842001274228, + 4.619401996023953 + ] + }, + "planner": { + "step_ms": 239.0471649169922, + "prepare_ms": 1.7240439774468541, + "one_use_ms": 240.77120889443904, + "prepare_min_ms": 1.6765620093792677, + "prepare_max_ms": 1.8191479612141848, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.001606236638194518, + "dq": 0.000603183325778411, + "dkv": 0.0005147282440500237, + "d_index_q": 0.00028285470187259566, + "d_weights": 0.00021272988166387048, + "d_pool_kv": 0.00029001033928657117, + "d_pool_score": 0.00028677249205629184, + "d_ape": 6.258893265663241e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 238.40419006347656, + 239.01718139648438, + 238.46278381347656, + 239.06573486328125, + 239.3005828857422, + 239.5536346435547, + 239.0471649169922 + ], + "critical_prepare_samples_ms": [ + 1.8191479612141848, + 1.7172510270029306, + 1.7308369278907776, + 1.682960893958807, + 1.7325289081782103, + 1.730848103761673, + 1.6999311046674848, + 1.6765620093792677 + ] + } + } + }, + "1024k-near-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "ce3857fde76c10e837d4359215986ae8706ddba02140df49bb2b6014276e94fb", + "policies": { + "contiguous": { + "step_ms": 239.05783081054688, + "prepare_ms": 1.514466479420662, + "one_use_ms": 240.57229728996754, + "prepare_min_ms": 1.4685369096696377, + "prepare_max_ms": 1.6245651058852673, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0016036861711426922, + "dq": 0.0006002606451464885, + "dkv": 0.0005078364704675495, + "d_index_q": 0.00027393963055896855, + "d_weights": 0.00021436935672360165, + "d_pool_kv": 0.00029689000244164466, + "d_pool_score": 0.0002755855279061583, + "d_ape": 6.5191501926423906e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 238.77810668945312, + 239.35433959960938, + 239.67471313476562, + 238.6045684814453, + 239.44578552246094, + 238.9220428466797, + 239.05783081054688 + ], + "critical_prepare_samples_ms": [ + 1.6245651058852673, + 1.5482361195608974, + 1.4911219477653503, + 1.5080280136317015, + 1.4883260009810328, + 1.5209049452096224, + 1.4685369096696377, + 1.5511849196627736 + ] + }, + "zigzag": { + "step_ms": 244.37164306640625, + "prepare_ms": 4.788588033989072, + "one_use_ms": 249.16023110039532, + "prepare_min_ms": 4.718890995718539, + "prepare_max_ms": 6.958352983929217, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016149985147692735, + "dq": 0.0006012411018216009, + "dkv": 0.0004889619138367483, + "d_index_q": 0.00027384235416954054, + "d_weights": 0.00021508569090086002, + "d_pool_kv": 0.00028208163900576497, + "d_pool_score": 0.00029433482150776626, + "d_ape": 6.324001411904957e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 244.35714721679688, + 243.5417022705078, + 244.73663330078125, + 244.49600219726562, + 244.37164306640625, + 244.37135314941406, + 244.45872497558594 + ], + "critical_prepare_samples_ms": [ + 6.958352983929217, + 4.946545930579305, + 4.794639069586992, + 4.782536998391151, + 5.79125399235636, + 4.723230958916247, + 4.718890995718539, + 4.728949978016317 + ] + }, + "planner": { + "step_ms": 239.24879455566406, + "prepare_ms": 1.75143702654168, + "one_use_ms": 241.00023158220574, + "prepare_min_ms": 1.686341012828052, + "prepare_max_ms": 1.818434102460742, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015996148111491304, + "dq": 0.0005959255331279302, + "dkv": 0.00048740660403977806, + "d_index_q": 0.00025509571834305165, + "d_weights": 0.00019903247405294528, + "d_pool_kv": 0.00029018364601671756, + "d_pool_score": 0.0002817420290576312, + "d_ape": 5.9639498577661876e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 238.96421813964844, + 238.8380126953125, + 239.6174774169922, + 238.64874267578125, + 239.3194580078125, + 239.24879455566406, + 239.29000854492188 + ], + "critical_prepare_samples_ms": [ + 1.818434102460742, + 1.7426980193704367, + 1.7638478893786669, + 1.7180261202156544, + 1.7601760337129235, + 1.686341012828052, + 1.7822969239205122, + 1.6957151237875223 + ] + } + } + }, + "1024k-mixed-22": { + "documents": 22, + "tokens": 1048576, + "source_record_sha256": "94a3d9ad0055a320defc6d8a0f7147cda48f2bec810ce85de4e405b8c33cfeb9", + "policies": { + "contiguous": { + "step_ms": 576.2921752929688, + "prepare_ms": 2.234563056845218, + "one_use_ms": 578.526738349814, + "prepare_min_ms": 2.176858950406313, + "prepare_max_ms": 2.457708935253322, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011241890551148522, + "dq": 0.0003703792465013327, + "dkv": 0.0005637775340177526, + "d_index_q": 0.0004923585316969156, + "d_weights": 0.0004762473505531833, + "d_pool_kv": 0.0007050138286532109, + "d_pool_score": 0.000693366416930018, + "d_ape": 1.0040782845648817e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 575.5206909179688, + 579.4055786132812, + 576.2921752929688, + 575.3203125, + 579.8231811523438, + 575.8522338867188, + 578.5023803710938 + ], + "critical_prepare_samples_ms": [ + 2.457708935253322, + 2.2585249971598387, + 2.2657379740849137, + 2.2382939932867885, + 2.176858950406313, + 2.2308321204036474, + 2.1903219167143106, + 2.2301790304481983 + ] + }, + "zigzag": { + "step_ms": 493.968017578125, + "prepare_ms": 3.2312944531440735, + "one_use_ms": 497.1993120312691, + "prepare_min_ms": 3.0827909940853715, + "prepare_max_ms": 19.6960570756346, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011268685984526365, + "dq": 0.0003714092777960759, + "dkv": 0.0005664412351356137, + "d_index_q": 0.0005001608323414847, + "d_weights": 0.0005154558961486076, + "d_pool_kv": 0.0007053187475275309, + "d_pool_score": 0.0007033408865167627, + "d_ape": 1.1331546979456586e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 491.4543151855469, + 493.968017578125, + 493.0928649902344, + 494.013916015625, + 493.9161376953125, + 496.6776123046875, + 496.3877258300781 + ], + "critical_prepare_samples_ms": [ + 19.6960570756346, + 3.2806959934532642, + 3.1818929128348827, + 3.8822469068691134, + 3.1228240113705397, + 3.135847859084606, + 3.0827909940853715, + 3.6610440583899617 + ] + }, + "planner": { + "step_ms": 491.5184326171875, + "prepare_ms": 9.799090563319623, + "one_use_ms": 501.3175231805071, + "prepare_min_ms": 9.549538022838533, + "prepare_max_ms": 12.66833208501339, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0011264548823997533, + "dq": 0.00037080288591740675, + "dkv": 0.0005661102261241025, + "d_index_q": 0.0004907002290738318, + "d_weights": 0.00047702545312988654, + "d_pool_kv": 0.0007029215298665386, + "d_pool_score": 0.0006920376268098218, + "d_ape": 1.0082613861991237e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 491.75732421875, + 491.4028625488281, + 492.5019836425781, + 491.5184326171875, + 492.1903381347656, + 491.46417236328125, + 491.2967529296875 + ], + "critical_prepare_samples_ms": [ + 11.265398934483528, + 9.871467016637325, + 10.41808701120317, + 12.66833208501339, + 9.726714110001922, + 9.549538022838533, + 9.598031989298761, + 9.689398924820125 + ] + } + } + }, + "1024k-uniform-1024": { + "documents": 1024, + "tokens": 1048576, + "source_record_sha256": "5dcf57b1ff0c2b430ff56323c1062b39a1a343d01dae0faef1de67b8fef47bf9", + "policies": { + "contiguous": { + "step_ms": 106.8575668334961, + "prepare_ms": 7.491494528949261, + "one_use_ms": 114.34906136244535, + "prepare_min_ms": 7.3957579443231225, + "prepare_max_ms": 7.639074116013944, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.340375730780978e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.8596359553728106e-05, + "d_pool_score": 5.8019952559618404e-05, + "d_ape": 8.688634497722616e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.89324951171875, + 106.8575668334961, + 106.78739166259766, + 107.19465637207031, + 107.24988555908203, + 106.72332763671875, + 106.78819274902344 + ], + "critical_prepare_samples_ms": [ + 7.517205900512636, + 7.3957579443231225, + 7.419119938276708, + 7.494135992601514, + 7.56581814493984, + 7.639074116013944, + 7.488853065297008, + 7.402385119348764 + ] + }, + "zigzag": { + "step_ms": 121.1388168334961, + "prepare_ms": 48.3965675230138, + "one_use_ms": 169.5353843565099, + "prepare_min_ms": 47.7493159705773, + "prepare_max_ms": 161.87846509274095, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.983033003732531e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.297525455088219e-05, + "d_pool_score": 6.027668702368681e-05, + "d_ape": 8.260924534593652e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 121.27545928955078, + 120.9210205078125, + 121.1388168334961, + 120.83197021484375, + 120.97875213623047, + 121.54752349853516, + 121.18406677246094 + ], + "critical_prepare_samples_ms": [ + 51.41136993188411, + 47.85480094142258, + 47.7493159705773, + 161.87846509274095, + 48.38546901009977, + 48.245941987261176, + 48.40766603592783, + 50.705861882306635 + ] + }, + "planner": { + "step_ms": 106.80134582519531, + "prepare_ms": 9.633392037358135, + "one_use_ms": 116.43473786255345, + "prepare_min_ms": 9.522294043563306, + "prepare_max_ms": 10.079156025312841, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.190915790570461e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.686079011901337e-05, + "d_pool_score": 6.588902419850418e-05, + "d_ape": 9.060561985811648e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.7529296875, + 106.80134582519531, + 106.69001770019531, + 106.8199691772461, + 106.92153930664062, + 107.30044555664062, + 106.72911834716797 + ], + "critical_prepare_samples_ms": [ + 9.983930038288236, + 9.658558061346412, + 9.608226013369858, + 10.079156025312841, + 9.522294043563306, + 9.544037049636245, + 9.525719098746777, + 9.793340926989913 + ] + } + } + }, + "1024k-near-uniform-512": { + "documents": 512, + "tokens": 1048576, + "source_record_sha256": "0b8a7ab3c025afba1b9ff31062ef317a231a85fed72a4f9399d77f0a11a5d450", + "policies": { + "contiguous": { + "step_ms": 153.78297424316406, + "prepare_ms": 4.550960555206984, + "one_use_ms": 158.33393479837105, + "prepare_min_ms": 4.407087923027575, + "prepare_max_ms": 4.7475280007347465, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.635166627093273e-05, + "dq": 8.583753797678552e-06, + "dkv": 3.908999526342516e-05, + "d_index_q": 2.4805252190121222e-06, + "d_weights": 2.5308917739233984e-06, + "d_pool_kv": 6.176247649990505e-05, + "d_pool_score": 5.8405774399699e-05, + "d_ape": 8.241450646605944e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.78297424316406, + 154.11363220214844, + 154.24351501464844, + 153.45152282714844, + 153.8989715576172, + 153.48739624023438, + 153.42739868164062 + ], + "critical_prepare_samples_ms": [ + 4.606066038832068, + 4.407087923027575, + 4.484036937355995, + 4.617327940650284, + 4.496491979807615, + 4.602779052220285, + 4.499142058193684, + 4.7475280007347465 + ] + }, + "zigzag": { + "step_ms": 162.9409942626953, + "prepare_ms": 25.861825968604535, + "one_use_ms": 188.80282023129985, + "prepare_min_ms": 25.590608012862504, + "prepare_max_ms": 141.61201403476298, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 1.690072397931956e-05, + "dq": 8.902184563800953e-06, + "dkv": 3.583246096075656e-05, + "d_index_q": 2.6051406625142775e-06, + "d_weights": 2.4387942932217848e-06, + "d_pool_kv": 6.690809516440746e-05, + "d_pool_score": 5.7566133313703044e-05, + "d_ape": 9.085070529934042e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 163.11216735839844, + 162.67152404785156, + 162.5786895751953, + 163.08656311035156, + 163.2576904296875, + 162.770751953125, + 162.9409942626953 + ], + "critical_prepare_samples_ms": [ + 29.362841974943876, + 28.138961060903966, + 25.614228914491832, + 25.82601795438677, + 25.8976339828223, + 25.744552025571465, + 25.590608012862504, + 141.61201403476298 + ] + }, + "planner": { + "step_ms": 153.50277709960938, + "prepare_ms": 5.739656975492835, + "one_use_ms": 159.2424340751022, + "prepare_min_ms": 5.382872885093093, + "prepare_max_ms": 5.910926032811403, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.66493234380544e-05, + "dq": 8.69517004748584e-06, + "dkv": 3.7684565826764085e-05, + "d_index_q": 2.546078095564981e-06, + "d_weights": 2.625010459222263e-06, + "d_pool_kv": 6.008571022238068e-05, + "d_pool_score": 5.6580447419918704e-05, + "d_ape": 8.854442371773076e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.19667053222656, + 153.4423065185547, + 153.50277709960938, + 153.30099487304688, + 153.84597778320312, + 153.7281951904297, + 153.51461791992188 + ], + "critical_prepare_samples_ms": [ + 5.910926032811403, + 5.768615053966641, + 5.845936946570873, + 5.667127901688218, + 5.738166975788772, + 5.730206961743534, + 5.382872885093093, + 5.741146975196898 + ] + } + } + }, + "1024k-mixed-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "dddf71dc1a5edeecb47eec7b85b24e6c842fb9e8736fd55552388a3f37a1dc77", + "policies": { + "contiguous": { + "step_ms": 566.1741333007812, + "prepare_ms": 10.622373956721276, + "one_use_ms": 576.7965072575025, + "prepare_min_ms": 10.451745009049773, + "prepare_max_ms": 10.986343026161194, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0007698426967776738, + "dq": 0.000278721133254185, + "dkv": 0.0005366458202454897, + "d_index_q": 0.0004893065256747059, + "d_weights": 0.000496051600621093, + "d_pool_kv": 0.0006594956893625123, + "d_pool_score": 0.0006637931035948877, + "d_ape": 9.298445474597294e-06 + }, + "max_loss_abs_diff": 1.4551915228366852e-10, + "critical_step_samples_ms": [ + 568.0925903320312, + 565.6495971679688, + 565.1004638671875, + 567.221435546875, + 566.1741333007812, + 567.1893920898438, + 563.3007202148438 + ], + "critical_prepare_samples_ms": [ + 10.948296985588968, + 10.565283009782434, + 10.503393015824258, + 10.87179605383426, + 10.451745009049773, + 10.473948088474572, + 10.986343026161194, + 10.679464903660119 + ] + }, + "zigzag": { + "step_ms": 549.7163696289062, + "prepare_ms": 48.262091528158635, + "one_use_ms": 597.9784611570649, + "prepare_min_ms": 46.8996309209615, + "prepare_max_ms": 165.2717370307073, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0007832338017054126, + "dq": 0.0002632348658705566, + "dkv": 0.0004852835254326828, + "d_index_q": 0.0005267448260146545, + "d_weights": 0.0005010625147896707, + "d_pool_kv": 0.0006835671852074876, + "d_pool_score": 0.0006593969952528296, + "d_ape": 9.171136322244436e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 548.9651489257812, + 549.9647216796875, + 552.8906860351562, + 549.7163696289062, + 548.3233642578125, + 550.1369018554688, + 548.8441772460938 + ], + "critical_prepare_samples_ms": [ + 50.29113800264895, + 47.78582998551428, + 46.8996309209615, + 165.2717370307073, + 48.17056504543871, + 48.35361801087856, + 47.71811398677528, + 48.362344969064 + ] + }, + "planner": { + "step_ms": 493.4424743652344, + "prepare_ms": 34.91155954543501, + "one_use_ms": 528.3540339106694, + "prepare_min_ms": 34.27027305588126, + "prepare_max_ms": 119.49189391452819, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0008045749900604678, + "dq": 0.00028106644892919336, + "dkv": 0.0005445073846626518, + "d_index_q": 0.0005790968752337036, + "d_weights": 0.0005290486376168959, + "d_pool_kv": 0.000667951163237759, + "d_pool_score": 0.0006640420442457095, + "d_ape": 9.158098557262821e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 493.3392028808594, + 494.3460693359375, + 492.5264892578125, + 493.4424743652344, + 493.0041809082031, + 495.9510192871094, + 494.9887390136719 + ], + "critical_prepare_samples_ms": [ + 34.88306805957109, + 35.257180919870734, + 34.5360190840438, + 35.578013863414526, + 34.29650503676385, + 34.940051031298935, + 34.27027305588126, + 119.49189391452819 + ] + } + } + }, + "1024k-long-tail-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "e0714dbf812ab0b51ad0e854f8e5cc6fc0fd8a8549ffb279e54cb753bb104aa1", + "policies": { + "contiguous": { + "step_ms": 801.3711547851562, + "prepare_ms": 14.17607138864696, + "one_use_ms": 815.5472261738032, + "prepare_min_ms": 13.991917949169874, + "prepare_max_ms": 14.449334004893899, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009458803536275202, + "dq": 0.00032713645135391036, + "dkv": 0.0006961451986581857, + "d_index_q": 0.0007918691377933198, + "d_weights": 0.0006913715310103977, + "d_pool_kv": 0.000853195284882402, + "d_pool_score": 0.0008060075488969843, + "d_ape": 8.446302463058031e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 799.466796875, + 804.5145263671875, + 800.3975219726562, + 805.9153442382812, + 802.1598510742188, + 801.3711547851562, + 798.016845703125 + ], + "critical_prepare_samples_ms": [ + 14.449334004893899, + 14.040772104635835, + 14.401104883290827, + 14.088836847804487, + 14.436830999329686, + 14.040581998415291, + 13.991917949169874, + 14.263305929489434 + ] + }, + "zigzag": { + "step_ms": 715.4530029296875, + "prepare_ms": 151.8010766012594, + "one_use_ms": 867.2540795309469, + "prepare_min_ms": 51.4363938709721, + "prepare_max_ms": 163.23676297906786, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0008850672295580158, + "dq": 0.0002783013933514211, + "dkv": 0.0006988498625940763, + "d_index_q": 0.0006129151100521033, + "d_weights": 0.0005560157421175833, + "d_pool_kv": 0.0008523786742666889, + "d_pool_score": 0.0008077892330878795, + "d_ape": 7.12458281847456e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 717.642578125, + 714.7274169921875, + 715.4530029296875, + 714.2869873046875, + 714.8427124023438, + 717.7915649414062, + 717.44677734375 + ], + "critical_prepare_samples_ms": [ + 163.23676297906786, + 51.87820305582136, + 156.81109705474228, + 154.88058305345476, + 158.93646096810699, + 51.80381902027875, + 51.4363938709721, + 148.72157014906406 + ] + }, + "planner": { + "step_ms": 591.85302734375, + "prepare_ms": 38.67735603125766, + "one_use_ms": 630.5303833750077, + "prepare_min_ms": 38.26623398344964, + "prepare_max_ms": 126.19063805323094, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.000942381079150994, + "dq": 0.0003192592988327247, + "dkv": 0.0005112119580584089, + "d_index_q": 0.0005326490446759798, + "d_weights": 0.0005266672889364208, + "d_pool_kv": 0.0006746754160394707, + "d_pool_score": 0.0006688692091720076, + "d_ape": 7.366243955007939e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 588.470458984375, + 591.85302734375, + 589.7974243164062, + 595.7284545898438, + 592.322021484375, + 593.126953125, + 587.995849609375 + ], + "critical_prepare_samples_ms": [ + 38.26623398344964, + 39.0528819989413, + 126.19063805323094, + 38.612914038822055, + 38.74179802369326, + 38.45288895536214, + 38.46422384958714, + 39.24016701057553 + ] + } + } + } + }, + "cohorts": { + "windows-final": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "34c2d866f81c1127623ba482adce4ff365a7a1663f66bfdbf717a39cbb7c3beb", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + }, + "dozens": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": [ + "uniform-64", + "near-uniform-64", + "mixed-22" + ], + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + }, + "one-m-single": { + "arguments": { + "tokens": [ + 1048576 + ], + "cases": [ + "single" + ], + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + }, + "one-m-remaining": { + "arguments": { + "tokens": [ + 1048576 + ], + "cases": [ + "uniform-64", + "near-uniform-64", + "mixed-22", + "uniform-1024", + "near-uniform-512", + "mixed-1000", + "long-tail-1000" + ], + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + } + }, + "totals_by_context": { + "128": { + "contiguous": [ + 207.97830295562744, + 237.19847300602123 + ], + "zigzag": [ + 221.86822509765625, + 395.9265925376676 + ], + "planner": [ + 206.09743785858154, + 278.04668443091214 + ] + }, + "256": { + "contiguous": [ + 530.4107818603516, + 564.9268742501736 + ], + "zigzag": [ + 555.1017608642578, + 877.8458124990575 + ], + "planner": [ + 507.27488136291504, + 604.8603271567263 + ] + }, + "1024": { + "contiguous": [ + 3536.3107986450195, + 3580.6261210637167 + ], + "zigzag": [ + 3170.2919540405273, + 3459.5822826670483 + ], + "planner": [ + 2952.967788696289, + 3064.5250777811743 + ] + } + }, + "max_relative_rms": { + "output": 0.002136063325688249, + "dq": 0.0006708973533251172, + "dkv": 0.0010310600226582155, + "d_index_q": 0.0007967333850337871, + "d_weights": 0.0006913715310103977, + "d_pool_kv": 0.0013756446763457486, + "d_pool_score": 0.0013779578045436441, + "d_ape": 2.647273742900797e-05 + }, + "max_loss_abs_diff": 8.149072527885437e-10 +} diff --git a/tests/test_csa_cp_selection.py b/tests/test_csa_cp_selection.py index 7b0fec8e..17776068 100644 --- a/tests/test_csa_cp_selection.py +++ b/tests/test_csa_cp_selection.py @@ -236,3 +236,28 @@ def test_range_indices_preserve_duplicates_gaps_and_order(ranges): actual = _range_indices(ranges, torch.device("cpu")) expected = torch.tensor([i for a, z in ranges for i in range(a, z)], dtype=torch.int64) assert torch.equal(actual, expected) + + +@pytest.mark.parametrize("tokens", [128 << 10, 256 << 10, 1 << 20]) +@pytest.mark.parametrize("case,documents", [("uniform-64", 64), ("near-uniform-64", 64), ("mixed-22", 22)]) +def test_dozens_of_document_cases_cover_each_context(tokens, case, documents): + from benchmarks.bench_csa_cp_windows import cases + + lengths = cases(tokens)[case] + assert len(lengths) == documents and sum(lengths) == tokens + assert all(n > 0 and n % 4 == 0 for n in lengths) + cu = (0, *accumulate(lengths)) + for policy in ("contiguous", "zigzag", "planner"): + plan = CsaCPSelectionPlan.from_cu_seqlens( + cu, + rank=0, + world_size=8, + top_k=8, + policy=policy, + cost_model=costs(8), + config=THDPlannerConfig(min_chunk_tokens=1024), + ) + assert sum(r.local_tokens for r in plan.ranks) == tokens + intervals = sorted((s.token_start, s.token_end) for r in plan.ranks for s in r.spans) + assert intervals[0][0] == 0 and intervals[-1][1] == tokens + assert all(a[1] == b[0] for a, b in zip(intervals, intervals[1:])) From 489779ba35e2c5f51eccce7c7ff95ed4cdd640d0 Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Mon, 16 Nov 2026 21:53:51 +0800 Subject: [PATCH 8/9] perf(csa): prune unproductive searches and reduce preparation work Signed-off-by: shuyan.ycf --- cula/distributed/thd_planner.py | 110 ++++++++++++++++++++++---------- cula/hybrid/dsa.py | 24 ++++--- docs/THD_LAYOUT_PLANNER.md | 25 ++++++-- tests/test_hybrid_dsa_plan.py | 30 +++++++++ tests/test_thd_planner.py | 47 ++++++++++++++ 5 files changed, 190 insertions(+), 46 deletions(-) diff --git a/cula/distributed/thd_planner.py b/cula/distributed/thd_planner.py index 1d3247f2..5b8890a1 100644 --- a/cula/distributed/thd_planner.py +++ b/cula/distributed/thd_planner.py @@ -214,13 +214,23 @@ def estimate_ms(self, ranks: tuple[THDRankLayout, ...], *, block_size: int) -> f cursor = offsets[rank] last_source = -1 for span in layout.spans: + start, end = span.token_start, span.token_end + tokens = end - start compute[rank] += ( - self.candidate_ns * _candidate_work(span.causal_offset, span.tokens, block_size) / 1e6 - + self.query_ns * span.tokens / 1e6 + self.candidate_ns * _candidate_work(start - span.sequence_start, tokens, block_size) / 1e6 + + self.query_ns * tokens / 1e6 + self.span_us / 1e3 ) - identity &= span.token_start == cursor - cursor += span.tokens + identity &= start == cursor + cursor += tokens + # Pinned source spans are common in mixed packs. Their exact + # one-source intersection needs neither a bisect nor a walk. + if offsets[rank] <= start and end <= offsets[rank + 1]: + send_ordered[rank] &= start == source_cursor[rank] + source_cursor[rank] += tokens + receive_ordered[rank] &= rank >= last_source + last_source = rank + continue for source, rows in _source_intersections(span, offsets): send_ordered[source] &= max(offsets[source], span.token_start) == source_cursor[source] source_cursor[source] += rows @@ -419,9 +429,8 @@ def _adaptive_items( target_blocks = max(1, ceil(total_blocks / requested_grid)) min_split_blocks = config.min_split_tokens // block_size result = [] - for sequence, (sequence_start, sequence_end) in enumerate(zip(boundaries, boundaries[1:])): - if sequences is not None and sequence not in sequences: - continue + for sequence in range(len(boundaries) - 1) if sequences is None else sorted(sequences): + sequence_start, sequence_end = boundaries[sequence : sequence + 2] length_blocks = (sequence_end - sequence_start) // block_size pieces = min(requested_grid, max(1, ceil(length_blocks / target_blocks))) # A sequence shorter than two useful spans stays whole. This is what @@ -964,7 +973,7 @@ def plan_thd_layout( ) -def _selection_assign(items, pinned, *, world, capacity, offsets, model, block_size): +def _selection_assign(items, *, pinned_loads, pinned_compute, world, capacity, offsets, model, block_size): """Keep cheap source spans local; place long-span work under a row cap. Unlike a shared KDA/CSA layout, selection ownership need not have equal @@ -972,7 +981,8 @@ def _selection_assign(items, pinned, *, world, capacity, offsets, model, block_s scores ordered compute/transport phases using the same calibrated units as final candidate selection, with stable load/locality tie breaks. """ - loads, compute = [0] * world, [0.0] * world + # Copy the small per-rank state; all grids share the same pinned spans. + loads, compute = list(pinned_loads), list(pinned_compute) incoming, outgoing = [0.0] * world, [0.0] * world intra_in, intra_out, inter_in, inter_out = ([0.0] * world for _ in range(4)) peer_rows = [[0] * world for _ in range(world)] @@ -986,9 +996,6 @@ def _selection_assign(items, pinned, *, world, capacity, offsets, model, block_s def work(item): return model.candidate_ns * item.causal_work / 1e6 + model.query_ns * item.tokens / 1e6 + model.span_us / 1e3 - for item, owner in pinned: - loads[owner] += item.tokens - compute[owner] += work(item) for original in sorted(items, key=lambda item: (-work(item), -item.tokens, item.sequence, item.token_start)): item = original while item is not None: @@ -1220,6 +1227,29 @@ def overlap(left, right): return compute + transfer +def _zigzag_meets_floor(boundaries, world, block_size, floor, source_fragments): + """Check grain feasibility before pricing or materializing full zigzag. + + Whole documents with sufficiently large pieces need no piece walk. For + finer pieces, retain exactly the same original-source-fragment exception + used by the other candidates. This rejects no otherwise legal layout. + """ + if not floor: + return True + for sequence, (start, end) in enumerate(zip(boundaries, boundaries[1:])): + blocks = (end - start) // block_size + base, remainder = divmod(blocks, 2 * world) + if max(1, base) * block_size >= floor: + continue + left = start + for piece in range(min(blocks, 2 * world)): + right = left + (base + (piece < remainder)) * block_size + if right - left < floor and (sequence, left, right) not in source_fragments: + return False + left = right + return True + + def _source_selection_candidate(source, world, model): """Price known identity ownership without reconstructing its transport. @@ -1330,29 +1360,42 @@ def add(assignments, policy): ) / world_size lower_bound = _transfer_lower_bound(base, world_size, config.block_size, cost_model, lower_bound, baseline_ms) if movable and lower_bound < cost_model._migration_threshold_ms(baseline_ms): - pinned_spans = _coalesce(pinned) - add( - _paired_selection_items(boundaries, movable, world_size, config.block_size, offsets, cost_model), - "selection-paired-local", - ) - seen = set() - for grid in range(1, config.max_chunks_per_rank + 1): - items = tuple(_adaptive_items(boundaries, world_size, config, chunks_per_rank=grid, sequences=movable)) - if items in seen: - continue - seen.add(items) + pinned_loads, pinned_compute = [0] * world_size, [0.0] * world_size + for item, owner in pinned: + pinned_loads[owner] += item.tokens + pinned_compute[owner] += ( + cost_model.candidate_ns * item.causal_work / 1e6 + + cost_model.query_ns * item.tokens / 1e6 + + cost_model.span_us / 1e3 + ) + # These source-local spans remain in every long-document candidate. + # Nonnegative work and transport cannot reduce their critical compute + # time. This bound applies only to the pinned family, not full zigzag. + if max(pinned_compute) < cost_model._migration_threshold_ms(baseline_ms): + pinned_spans = _coalesce(pinned) add( - _selection_assign( - items, - pinned, - world=world_size, - capacity=max_tokens_per_rank, - offsets=offsets, - model=cost_model, - block_size=config.block_size, - ), - f"selection-local-{grid}x", + _paired_selection_items(boundaries, movable, world_size, config.block_size, offsets, cost_model), + "selection-paired-local", ) + seen = set() + for grid in range(1, config.max_chunks_per_rank + 1): + items = tuple(_adaptive_items(boundaries, world_size, config, chunks_per_rank=grid, sequences=movable)) + if items in seen: + continue + seen.add(items) + add( + _selection_assign( + items, + pinned_loads=pinned_loads, + pinned_compute=pinned_compute, + world=world_size, + capacity=max_tokens_per_rank, + offsets=offsets, + model=cost_model, + block_size=config.block_size, + ), + f"selection-local-{grid}x", + ) # Score the cheap local candidates first. They tighten the incumbent, # allowing an admissible bound to reject a fragmented full zigzag without # allocating O(documents * CP) span objects. No candidate-specific heuristic @@ -1360,6 +1403,7 @@ def add(assignments, policy): threshold = min(cost_model._migration_threshold_ms(baseline_ms), min(c[2] for c in candidates)) if ( lower_bound < threshold + and _zigzag_meets_floor(boundaries, world_size, config.block_size, config.min_chunk_tokens, source_fragments) and _zigzag_lower_bound(boundaries, base, world_size, config.block_size, cost_model) < threshold ): zigzag = zigzag_thd_layout(boundaries, world_size=world_size, block_size=config.block_size) diff --git a/cula/hybrid/dsa.py b/cula/hybrid/dsa.py index 4761bf91..def025e5 100644 --- a/cula/hybrid/dsa.py +++ b/cula/hybrid/dsa.py @@ -246,16 +246,22 @@ def from_cu_seqlens( row = 0 while row < query_length: - low, high = row + 1, query_length - while low < high: - middle = (low + high + 1) // 2 - needed = _candidate_floats(position + row, middle - row, block_size) - if needed <= budget_floats: - low = middle - else: - high = middle - 1 - row_end = low + # Most short documents fit whole. Check the exact remaining + # work once before searching a split; retain the same maximal + # prefix when a long document exceeds the scratch budget. + row_end = query_length candidates = _candidate_floats(position + row, row_end - row, block_size) + if candidates > budget_floats: + low, high = row + 1, query_length - 1 + while low < high: + middle = (low + high + 1) // 2 + needed = _candidate_floats(position + row, middle - row, block_size) + if needed <= budget_floats: + low = middle + else: + high = middle - 1 + row_end = low + candidates = _candidate_floats(position + row, row_end - row, block_size) atoms.append((sequence, row, row_end, key_length, position + row, candidates, query_length)) row = row_end diff --git a/docs/THD_LAYOUT_PLANNER.md b/docs/THD_LAYOUT_PLANNER.md index 00df1066..fbf2fd35 100644 --- a/docs/THD_LAYOUT_PLANNER.md +++ b/docs/THD_LAYOUT_PLANNER.md @@ -128,7 +128,7 @@ calibration and held-out validation. ## Admissible pruning and preparation complexity -Two lower bounds avoid building candidates that cannot meet the margin: +Three lower bounds avoid building candidates that cannot meet the margin: - Every partition has the same token/candidate sums and at least one span per document. Their mean is a compute lower bound. To reduce source rank i's @@ -142,18 +142,35 @@ Two lower bounds avoid building candidates that cannot meet the margin: source spans with those runs and the owner's two pieces avoids constructing O(documents * CP) objects. This bound omits incoming and peer bottlenecks and permutation copies. Local candidates run first to tighten its incumbent. +- The long-document candidate family retains pinned source spans. Its critical + compute time cannot fall below the largest pinned per-rank compute load; + added nonnegative work and transport cannot lower it. If that bound already + misses the migration margin, skip the entire long-document search. Full + zigzag is checked independently because it can move the short documents too. Bounds are tested against exhaustive small assignments and concrete zigzag layouts on contiguous, interleaved and irregular topologies. They cannot exclude a strictly better modeled zigzag; they make no statement about model error. Repeated planning grids are deduplicated. Pinned spans are coalesced once; -owner groups are built in one pass, and diagnostics are computed only for the +their initial per-rank loads are computed once and copied for each grid. Adaptive +grids visit only movable documents. Owner groups are built in one pass, and +diagnostics are computed only for the selected selection plan. Balanced-token scheduling uses a rank heap for identical-cost pieces and indexes capacity repair by donor and piece shape. -Source intersections use binary search. DSA window construction maintains an -integer candidate total instead of rescanning all preceding short documents; +Source-local spans have a direct intersection path; other source intersections +use binary search. A geometry-only grain check rejects infeasible full zigzag +before transport pricing, preserving the original-source-fragment exception. +These changes preserve the scores of evaluated candidates, choices and stable +tie breaks. Diagnostics can omit candidates proven unable to meet the margin. + +DSA window construction checks whether all remaining rows fit before searching +for a split. It uses exact integer work, including unaligned causal offsets; +long documents still receive the same maximal budget-fitting prefixes. An +integer candidate total avoids rescanning all preceding short documents; window boundaries, scratch requirements and numerical execution are unchanged. +All reuse is local to one preparation call; there is no retained layout cache +or new device-to-host metadata read. ## Calibrating and measuring diff --git a/tests/test_hybrid_dsa_plan.py b/tests/test_hybrid_dsa_plan.py index 15022473..234fccb9 100644 --- a/tests/test_hybrid_dsa_plan.py +++ b/tests/test_hybrid_dsa_plan.py @@ -66,6 +66,36 @@ def test_selection_plan_splits_one_sequence_to_bound_peak_scratch(): assert plan.windows[1].q_causal_offsets == (plan.windows[1].query_start,) +@pytest.mark.parametrize("block", [1, 4, 16]) +@pytest.mark.parametrize("length", [1, 3, 16, 33]) +@pytest.mark.parametrize("offset", [0, 5, 64]) +def test_selection_windows_use_exact_maximal_prefix_at_budget_boundary(block, length, offset): + # Independent row enumeration covers unaligned offsets, zero-work rows, + # whole-document fits and the final short remainder of a split document. + rows = [(offset + i + 1) // block for i in range(length)] + whole = sum(rows) + for budget in {max(1, max(rows)), max(1, max(rows), whole - 1), max(1, whole), whole + 1}: + plan = DsaSelectionPlan.from_cu_seqlens( + [0, length], + [0, max(1, (offset + length) // block)], + top_k=1, + block_size=block, + q_causal_offsets=[offset], + candidate_budget_bytes=budget * 4, + ) + cursor = 0 + for window in plan.windows: + assert window.query_start == cursor + cursor = window.query_end + used = sum(rows[window.query_start : cursor]) + assert window.candidate_offsets == (0, used) + assert window.q_causal_offsets == (offset + window.query_start,) + assert used <= budget + if cursor < length: + assert used + rows[cursor] > budget + assert cursor == length + + def test_selection_plan_keeps_chunked_prefill_position_offset(): plan = DsaSelectionPlan.from_cu_seqlens( [0, 8], diff --git a/tests/test_thd_planner.py b/tests/test_thd_planner.py index 04587a09..1b6b52d7 100644 --- a/tests/test_thd_planner.py +++ b/tests/test_thd_planner.py @@ -504,6 +504,53 @@ def test_transport_prices_only_executed_permutations(): ) == pytest.approx(0.128) +def test_source_local_spans_still_price_local_reordering(): + from benchmarks.cp_layouts import query_layouts + + layouts, _ = query_layouts(_boundaries([16] * 8), 4, "contiguous") + model = _cost_model(candidate_ns=0, query_ns=0, span_us=0, packing_ns_per_query=1000, exchange_us=10) + assert model.estimate_ms(layouts, block_size=4) == 0 + reordered = tuple(THDRankLayout(r.rank, r.spans[::-1]) for r in layouts) + # No rows cross ranks, but source-side packing and its inverse still + # execute, as does the group-wide eager exchange for a nonidentity map. + assert model.estimate_ms(reordered, block_size=4) == pytest.approx(0.032 + 0.010) + + +@pytest.mark.parametrize("margin", [0, 0.05]) +def test_pinned_critical_path_avoids_useless_long_document_search(monkeypatch, margin): + cu = _boundaries([64] * 998 + [33600] * 2) + model = _cost_model(candidate_ns=0.001, span_us=10, min_gain_fraction=margin) + + def forbidden(*args, **kwargs): + pytest.fail("moving long documents cannot improve the pinned critical path") + + monkeypatch.setattr("cula.distributed.thd_planner._selection_assign", forbidden) + monkeypatch.setattr("cula.distributed.thd_planner._paired_selection_items", forbidden) + plan = plan_thd_selection( + cu, world_size=4, max_tokens_per_rank=65536, cost_model=model, config=THDPlannerConfig(min_chunk_tokens=1024) + ) + assert plan.policy == "contiguous" + assert plan.metrics.score == model.estimate_ms(plan.ranks, block_size=4) + _assert_exact_cover(plan, cu) + + +@pytest.mark.parametrize("world", [1, 2, 8, 64]) +@pytest.mark.parametrize("floor", [0, 4, 16, 1024]) +def test_zigzag_grain_check_equals_materialized_candidate(world, floor): + from cula.distributed.thd_planner import _zigzag_meets_floor, zigzag_thd_layout + + rng = random.Random(812 + world) + for lengths in ([4] * world, [4 * world], [4, 1024, 65536], [4 * rng.randint(1, 1024) for _ in range(17)]): + cu = _boundaries(lengths) + source = _contiguous_items(cu, world, 4) + fragments = {(s.sequence, s.token_start, s.token_end) for s, _ in source} + zigzag = zigzag_thd_layout(cu, world_size=world) + expected = all( + s.tokens >= floor or (s.sequence, s.token_start, s.token_end) in fragments for r in zigzag for s in r.spans + ) + assert _zigzag_meets_floor(cu, world, 4, floor, fragments) == expected + + def test_independent_link_model_and_its_outgoing_bound(): from dataclasses import replace From aafd594dc209ca57a48fad84ac448bee9cef0bbb Mon Sep 17 00:00:00 2001 From: "shuyan.ycf" Date: Mon, 16 Nov 2026 22:43:14 +0800 Subject: [PATCH 9/9] docs(csa): publish optimized planner results across context lengths Signed-off-by: shuyan.ycf --- docs/CSA_CP_THREE_WAY_RESULTS.md | 301 +- docs/CSA_CP_WINDOW_RESULTS.md | 5 +- docs/results/csa_cp_preparation_20261116.json | 6770 +++++++++++++++++ 3 files changed, 6960 insertions(+), 116 deletions(-) create mode 100644 docs/results/csa_cp_preparation_20261116.json diff --git a/docs/CSA_CP_THREE_WAY_RESULTS.md b/docs/CSA_CP_THREE_WAY_RESULTS.md index 13448cb2..75a50087 100644 --- a/docs/CSA_CP_THREE_WAY_RESULTS.md +++ b/docs/CSA_CP_THREE_WAY_RESULTS.md @@ -1,10 +1,11 @@ # CSA CP: three policies across 128K, 256K and 1M -On B300 CP8, adaptive selection routing stays within **0.576%** of the faster -fixed contiguous/zigzag policy in steady execution across these 24 cases. The -largest gains occur at 1M with unequal long documents. Uniform packs usually -select contiguous; a single long document performs similarly to zigzag. -This is measured coverage, not a guarantee over arbitrary THD inputs. +This is the complete rerun after reducing host preparation work. On B300 CP8, +adaptive selection routing stays within **0.339%** of the faster fixed +contiguous/zigzag policy in steady execution across these 24 cases. All selected +layouts match the previous revision. The largest gains occur with unequal long +documents at 1M; uniform packs generally select contiguous. This is measured +coverage, not a guarantee over arbitrary THD inputs. The workload is **one complete CSA operator forward/backward**: ratio-4 pooling, pooled-key and attention-KV gathers, routed selection and inverse exchange, @@ -56,40 +57,40 @@ should be treated cautiously, rather than as strong evidence of a speedup. | Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | |---|---:|---:|---:|---:|---:| -| 1 long document | 47.60 | 46.23 | 45.82 | -3.74% | -0.88% | -| 64 uniform | 20.11 | 20.66 | 20.04 | -0.37% | -3.02% | -| 64 near-uniform | 20.08 | 20.79 | 20.06 | -0.13% | -3.53% | -| 20 short + 2 long | 35.44 | 35.23 | 35.44 | -0.00% | +0.58% | -| 1,024 uniform | 9.29 | 11.10 | 9.30 | +0.09% | -16.28% | -| 512 near-uniform | 10.01 | 11.21 | 10.03 | +0.14% | -10.52% | -| 998 short + 2 long | 33.07 | 38.75 | 33.07 | -0.00% | -14.64% | -| 1,000 long-tail | 32.37 | 37.90 | 32.35 | -0.06% | -14.64% | +| 1 long document | 47.69 | 46.27 | 45.83 | -3.91% | -0.97% | +| 64 uniform | 20.00 | 20.67 | 19.99 | -0.04% | -3.29% | +| 64 near-uniform | 20.03 | 20.76 | 20.02 | -0.03% | -3.53% | +| 20 short + 2 long | 35.41 | 35.36 | 35.48 | +0.19% | +0.34% | +| 1,024 uniform | 9.32 | 11.10 | 9.28 | -0.37% | -16.34% | +| 512 near-uniform | 10.00 | 11.20 | 10.01 | +0.12% | -10.60% | +| 998 short + 2 long | 33.10 | 38.75 | 33.09 | -0.05% | -14.63% | +| 1,000 long-tail | 32.33 | 37.88 | 32.33 | +0.01% | -14.66% | ## 256K steady execution | Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | |---|---:|---:|---:|---:|---:| -| 1 long document | 115.29 | 105.58 | 104.15 | -9.67% | -1.36% | -| 64 uniform | 50.34 | 51.73 | 50.33 | -0.00% | -2.70% | -| 64 near-uniform | 50.34 | 51.77 | 50.35 | +0.01% | -2.75% | -| 20 short + 2 long | 90.69 | 87.95 | 87.39 | -3.64% | -0.64% | -| 1,024 uniform | 18.96 | 22.56 | 18.94 | -0.10% | -16.02% | -| 512 near-uniform | 21.79 | 24.21 | 21.76 | -0.12% | -10.12% | -| 998 short + 2 long | 83.64 | 96.05 | 82.15 | -1.78% | -14.47% | -| 1,000 long-tail | 99.36 | 115.25 | 92.20 | -7.20% | -20.00% | +| 1 long document | 115.74 | 105.47 | 104.23 | -9.94% | -1.17% | +| 64 uniform | 50.32 | 51.76 | 50.36 | +0.08% | -2.70% | +| 64 near-uniform | 50.35 | 51.71 | 50.30 | -0.09% | -2.72% | +| 20 short + 2 long | 90.67 | 88.23 | 87.59 | -3.40% | -0.72% | +| 1,024 uniform | 18.96 | 22.55 | 18.94 | -0.15% | -16.01% | +| 512 near-uniform | 21.74 | 24.21 | 21.77 | +0.14% | -10.06% | +| 998 short + 2 long | 83.71 | 95.86 | 82.07 | -1.96% | -14.39% | +| 1,000 long-tail | 98.91 | 115.31 | 92.09 | -6.89% | -20.14% | ## 1M steady execution | Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | |---|---:|---:|---:|---:|---:| -| 1 long document | 853.46 | 638.50 | 637.55 | -25.30% | -0.15% | -| 64 uniform | 239.31 | 244.20 | 239.05 | -0.11% | -2.11% | -| 64 near-uniform | 239.06 | 244.37 | 239.25 | +0.08% | -2.10% | -| 20 short + 2 long | 576.29 | 493.97 | 491.52 | -14.71% | -0.50% | -| 1,024 uniform | 106.86 | 121.14 | 106.80 | -0.05% | -11.84% | -| 512 near-uniform | 153.78 | 162.94 | 153.50 | -0.18% | -5.79% | -| 998 short + 2 long | 566.17 | 549.72 | 493.44 | -12.85% | -10.24% | -| 1,000 long-tail | 801.37 | 715.45 | 591.85 | -26.14% | -17.28% | +| 1 long document | 852.26 | 637.36 | 637.78 | -25.17% | +0.07% | +| 64 uniform | 239.24 | 244.40 | 239.44 | +0.09% | -2.03% | +| 64 near-uniform | 239.18 | 244.56 | 239.23 | +0.02% | -2.18% | +| 20 short + 2 long | 578.44 | 492.49 | 490.87 | -15.14% | -0.33% | +| 1,024 uniform | 106.93 | 120.95 | 106.89 | -0.04% | -11.63% | +| 512 near-uniform | 153.33 | 162.94 | 153.63 | +0.19% | -5.71% | +| 998 short + 2 long | 567.96 | 550.56 | 494.71 | -12.90% | -10.14% | +| 1,000 long-tail | 800.96 | 715.78 | 591.82 | -26.11% | -17.32% | ## Preparation plus one use @@ -102,114 +103,186 @@ not replaced by a cache lookup. First compilation is still excluded. | Distribution | Contiguous | Zigzag | Planner | |---|---:|---:|---:| -| 1 long document | 48.33 | 47.14 | 53.37 | -| 64 uniform | 21.07 | 24.84 | 21.28 | -| 64 near-uniform | 21.06 | 24.95 | 21.24 | -| 20 short + 2 long | 36.25 | 37.16 | 43.04 | -| 1,024 uniform | 16.07 | 58.33 | 18.27 | -| 512 near-uniform | 13.76 | 35.29 | 14.98 | -| 998 short + 2 long | 40.72 | 84.47 | 61.39 | -| 1,000 long-tail | 39.95 | 83.74 | 44.48 | +| 1 long document | 48.41 | 47.17 | 53.27 | +| 64 uniform | 20.90 | 24.40 | 21.10 | +| 64 near-uniform | 21.07 | 25.31 | 21.31 | +| 20 short + 2 long | 36.17 | 37.22 | 43.18 | +| 1,024 uniform | 15.78 | 54.64 | 17.80 | +| 512 near-uniform | 13.56 | 33.89 | 14.72 | +| 998 short + 2 long | 40.23 | 82.95 | 42.34 | +| 1,000 long-tail | 39.66 | 83.09 | 41.45 | ### 256K | Distribution | Contiguous | Zigzag | Planner | |---|---:|---:|---:| -| 1 long document | 116.41 | 106.72 | 111.94 | -| 64 uniform | 51.40 | 55.87 | 51.61 | -| 64 near-uniform | 51.40 | 56.02 | 51.62 | -| 20 short + 2 long | 91.84 | 90.29 | 96.16 | -| 1,024 uniform | 25.87 | 69.42 | 27.91 | -| 512 near-uniform | 25.73 | 48.89 | 26.78 | -| 998 short + 2 long | 93.22 | 188.33 | 114.82 | -| 1,000 long-tail | 109.06 | 262.29 | 124.02 | +| 1 long document | 116.81 | 106.57 | 111.94 | +| 64 uniform | 51.31 | 55.48 | 51.57 | +| 64 near-uniform | 51.34 | 55.48 | 51.51 | +| 20 short + 2 long | 91.79 | 90.53 | 96.11 | +| 1,024 uniform | 26.19 | 67.67 | 28.22 | +| 512 near-uniform | 25.42 | 47.63 | 26.58 | +| 998 short + 2 long | 92.58 | 191.44 | 106.15 | +| 1,000 long-tail | 107.78 | 263.45 | 115.51 | ### 1M | Distribution | Contiguous | Zigzag | Planner | |---|---:|---:|---:| -| 1 long document | 855.70 | 640.79 | 646.87 | -| 64 uniform | 240.80 | 248.86 | 240.77 | -| 64 near-uniform | 240.57 | 249.16 | 241.00 | -| 20 short + 2 long | 578.53 | 497.20 | 501.32 | -| 1,024 uniform | 114.35 | 169.54 | 116.43 | -| 512 near-uniform | 158.33 | 188.80 | 159.24 | -| 998 short + 2 long | 576.80 | 597.98 | 528.35 | -| 1,000 long-tail | 815.55 | 867.25 | 630.53 | +| 1 long document | 854.50 | 639.80 | 647.29 | +| 64 uniform | 240.69 | 248.73 | 241.09 | +| 64 near-uniform | 240.69 | 248.94 | 240.94 | +| 20 short + 2 long | 580.61 | 495.68 | 500.99 | +| 1,024 uniform | 114.18 | 168.40 | 116.17 | +| 512 near-uniform | 158.24 | 187.40 | 159.63 | +| 998 short + 2 long | 577.91 | 599.07 | 521.10 | +| 1,000 long-tail | 813.63 | 817.01 | 620.98 | ## What the comparison shows The aggregate below sums one execution of each of the eight distributions at -each context length. It assumes equal case frequency, not equal runtime weights -or a measured production mix. Values are percentage time reductions relative -to the specified fixed policy; a negative reduction means a slowdown. +each context length. It assumes equal case frequency, not a production workload +mix. Values are percentage time reductions relative to each fixed policy; +negative reductions mean a slowdown. | Context | Steady vs contiguous | Steady vs zigzag | Prepare + one use vs contiguous | Prepare + one use vs zigzag | |---|---:|---:|---:|---:| -| 128K | 0.90% | 7.11% | -17.22% | 29.77% | -| 256K | 4.36% | 8.62% | -7.07% | 31.10% | -| 1M | 16.50% | 6.85% | 14.41% | 11.42% | - -At 1M, the two-long/998-short mixture improves steady time by **12.85% versus -contiguous and 10.24% versus zigzag**. The 1,000-document long tail improves by -**26.14% and 17.28%**, respectively. These two cases choose locality-preserving -six-piece-per-rank candidates. Balancing causal selection work while avoiding -unnecessary movement of short documents beats applying full zigzag everywhere. - -The dozens-of-documents cases add a useful distinction: 64 uniform/near-uniform -documents choose contiguous at all three lengths, avoiding the full-zigzag cost -for already-balanced work. The 20-short/two-long pack at 1M improves **14.71%** -over contiguous and is close to zigzag. At 128K, however, the same document-count -pattern is **0.576% slower than zigzag**. Layout choice is not perfectly predicted; -the larger aggregate gain at 1M does not establish monotonic improvement per case. - -Fresh preparation still matters. At 1M, including it leaves planner slightly -slower than zigzag for the single document and the 22-document pack, despite -similar steady execution. At 128K/256K the aggregate one-use result remains -slower than contiguous. The worst one-use gap to the faster fixed policy across -all 24 cases is **50.77%** for the 128K two-long/998-short mixture. Thus this study -does not establish universally optimal one-use selection or a <3% preparation -budget. A reused layout and a layout used only once need different preparation -budgets. Some preparation samples have large host-time spikes; the raw minimum, -maximum and all critical samples are retained in the linked JSON. +| 128K | 0.89% | 7.19% | -8.23% | 34.35% | +| 256K | 4.35% | 8.60% | -4.33% | 33.10% | +| 1M | 16.50% | 6.77% | 14.87% | 10.48% | + +The same planner and coefficients handle all three lengths. Single long documents +benefit from balancing causal work; many equal documents are already balanced +under contiguous ownership. Mixed packs benefit from redistributing expensive +long-document work while keeping cheap short documents local. At 1M, the +998-short/two-long and 1,000-document long-tail cases reduce steady time by +**10.14% and 17.32% versus full zigzag**, respectively. The 64-document cases +choose contiguous; the 22-document 1M case performs close to zigzag. + +Preparation remains a limitation for one-use layouts. At 128K/256K, aggregate +preparation-plus-one-step is still slower than contiguous. At 1M, the single +long document and 22-document pack remain slightly slower than zigzag under +one-use accounting. The largest one-use gap to the faster fixed policy is +**19.40%** (128k-mixed-22), compared with 50.77% in the previous run. +These are separately measured cohorts; the controlled host A/B below isolates +the preparation change. Neither cohort establishes universally optimal one-use +selection or a <3% preparation guarantee. Use a calibrated preparation budget +and account for expected layout reuse. + +## Equivalent preparation optimization + +The update removes redundant metadata work and prunes candidates with a proved +model-level bound; it introduces no retained cache: + +1. Compute pinned per-rank loads once per invocation and copy the small arrays + into each search grid. Visit only movable documents in adaptive grids. +2. Skip long-document candidates when immutable pinned work alone exceeds the + migration threshold. Full zigzag remains independently eligible. +3. Price a source-local span directly, preserving source/receive permutation + detection and floating-point accumulation order. +4. Check full-zigzag grain feasibility before transport estimation, retaining + the original-source-fragment exception. No feasible candidate is discarded. +5. Check exact whole-document/remainder work before the DSA binary split search. + Oversized intervals still use the same maximal budget-fitting prefix. + +Candidate definitions, coefficients, tie breaks, selected layouts, native window +boundaries, scratch sizes and GPU execution are unchanged. The diagnostic list +of evaluated scores can omit candidates rejected by the new lower bound. +All extra state is local to preparation; no mutable global cache, collective, +cast or D2H read is introduced. + +The following host-only A/B uses rank 7's complete `CsaCPSelectionPlan` constructor, +20 samples per variant, alternating before/after order after two warmups. It +excludes GPU workspace preparation. The baseline imports source from `ea71df3`; +the otherwise unchanged caller binds that revision's planner and DSA builder. +The raw report covers all 24 planner shapes plus fixed-policy controls. + +| Host-only case | Before (ms) | After (ms) | Time reduction | +|---|---:|---:|---:| +| 128k-single-planner | 6.177 | 6.052 | +2.0% | +| 256k-single-planner | 6.190 | 6.082 | +1.8% | +| 1024k-uniform-64-planner | 0.397 | 0.361 | +9.1% | +| 256k-mixed-22-planner | 6.795 | 6.507 | +4.2% | +| 128k-uniform-1024-planner | 4.494 | 4.157 | +7.5% | +| 128k-uniform-1024-zigzag | 38.705 | 37.026 | +4.3% | +| 128k-mixed-1000-planner | 21.876 | 3.862 | +82.3% | +| 1024k-long-tail-1000-planner | 24.956 | 16.904 | +32.3% | + +Full fresh preparation, including GPU metadata/workspace work, is shown next. +These before/after medians come from separate complete-CSA runs. Small values +have visible run-to-run variation; the table does not imply every case improved. +Host-time spikes remain in the raw samples. For example, the new 1M long-tail planner preparation ranges from 28.45 to 150.10 ms. + +| Context | Distribution | Before preparation | After preparation | Time reduction | +|---|---|---:|---:|---:| +| 128K | 1 long document | 7.55 | 7.45 | +1.4% | +| 128K | 64 uniform | 1.24 | 1.11 | +10.8% | +| 128K | 20 short + 2 long | 7.61 | 7.70 | -1.3% | +| 128K | 1,024 uniform | 8.97 | 8.52 | +5.1% | +| 128K | 998 short + 2 long | 28.31 | 9.26 | +67.3% | +| 128K | 1,000 long-tail | 12.13 | 9.12 | +24.8% | +| 256K | 1 long document | 7.79 | 7.71 | +1.0% | +| 256K | 64 uniform | 1.28 | 1.21 | +5.0% | +| 256K | 20 short + 2 long | 8.77 | 8.52 | +2.9% | +| 256K | 1,024 uniform | 8.97 | 9.28 | -3.5% | +| 256K | 998 short + 2 long | 32.67 | 24.09 | +26.3% | +| 256K | 1,000 long-tail | 31.81 | 23.42 | +26.4% | +| 1M | 1 long document | 9.32 | 9.51 | -2.0% | +| 1M | 64 uniform | 1.72 | 1.65 | +4.3% | +| 1M | 20 short + 2 long | 9.80 | 10.12 | -3.3% | +| 1M | 1,024 uniform | 9.63 | 9.29 | +3.6% | +| 1M | 998 short + 2 long | 34.91 | 26.39 | +24.4% | +| 1M | 1,000 long-tail | 38.68 | 29.16 | +24.6% | ## Precision, review and provenance -All **24 cases x four paths x eight ranks** pass row-identity/inverse checks and -full-tensor output/seven-gradient comparisons against legacy contiguous. Maximum -relative RMS across all public paths/ranks/cases is shown below; it is not a -bitwise-equivalence claim for nondeterministic native kernels. +- **518 focused CPU tests pass**, including exact budget-boundary checks with + unaligned causal offsets, local-reorder pricing and grain-check equivalence. +- **660 planner comparisons** preserve the selected layout, metrics and score + exactly versus `ea71df3`. Every retained candidate score is identical; omitted + candidates are verified to miss the migration margin: 55 shapes + (31 historical + 24 window cases) x CP8/16/64 x zero/original headroom x + zero/1,024-token floors. CP64 is host metadata validation, not a GPU benchmark. +- **300 randomized DSA plans and 69,281 windows** match the baseline exactly. +- **128 native subgroup checks pass** for BF16/FP32 weights, unequal sources, + empty destinations and retained outputs; canonical IDs and scores match + unrouted selection exactly in these fixtures. +- All **24 shapes x four paths x eight ranks** pass full-tensor output and + seven-gradient comparisons against legacy contiguous, plus exact row-marker + forward/inverse/autograd checks. Three small fixtures under all four paths + pass independent FP32 attention/teacher/KL/pooling checks. Those FP32 checks + use the selected key set and do not independently establish discrete ranking. + +Maximum relative RMS over all public paths/ranks/cases: | Tensor | Maximum relative RMS | |---|---:| -| Output | 0.21361% | -| dQ | 0.06709% | -| dKV | 0.10311% | -| Indexer dQ | 0.07967% | -| Indexer dWeights | 0.06914% | -| Pool dKV | 0.13756% | -| Pool dScore | 0.13780% | -| Pool dAPE | 0.00265% | - -Maximum auxiliary loss absolute difference is **8.15e-10**. No runtime math, -dtype, kernel or precision option changed for this coverage extension. -**444 focused CPU tests pass**, including nine new document-count/context -coverage tests. The unchanged runtime had also passed 128 native subgroup -selection checks and three small independent FP32 oracle fixtures under all -four paths, as detailed in the [original study](CSA_CP_WINDOW_RESULTS.md). - -The original ten 128K/256K records are reused from that study; fourteen additional -records cover six dozens-of-documents cases at 128K/256K and all eight 1M -distributions. Runtime source is `d2cf4fd`; only benchmark fixture -construction/help changed between timing cohorts. All 125 source hashes match -for the new cohorts. The only differing source hash in the original cohort is -that benchmark file. No algorithm or profile was tuned between these cohorts. - -The report preserves per-case timing/error envelopes, selected policies, all -critical timing/preparation samples, source-record hashes, and cohort source -manifests in [machine-readable results](results/csa_cp_three_way_20261116.json). -Diagnostic allocator peaks in the underlying harness include four coexisting -fixtures; they must not be interpreted as isolated per-policy memory usage. +| Output | 0.21341% | +| dQ | 0.06697% | +| dKV | 0.10307% | +| Indexer dQ | 0.09174% | +| Indexer dWeights | 0.06641% | +| Pool dKV | 0.13371% | +| Pool dScore | 0.13834% | +| Pool dAPE | 0.00254% | + +Maximum auxiliary loss absolute difference is **1.16e-09**. No math, +dtype or native precision setting changed. These error envelopes include native +nondeterminism; small differences from the previous run are not a new precision +improvement or a bitwise-equivalence claim. + +The tested source is `489779ba35e2c5f51eccce7c7ff95ed4cdd640d0`. All 125 source +files in the final timing manifest match the reviewed checkout. +Every selected policy matches the previous run, and every measured full-zigzag +baseline fits the planner's row cap. The new measurements supersede the primary +tables in this document. The [previous 24-case evidence](results/csa_cp_three_way_20261116.json) +and [original 128K/256K study](CSA_CP_WINDOW_RESULTS.md) remain available. + +[Current machine-readable results](results/csa_cp_preparation_20261116.json) +retain critical timing/preparation samples, selected policies, precision envelopes, +source-record hashes, cohort source hashes and the counterbalanced host A/B. +The allocator diagnostics in the underlying harness include four coexisting +fixtures; they are not isolated per-policy memory claims. ## Reproduction diff --git a/docs/CSA_CP_WINDOW_RESULTS.md b/docs/CSA_CP_WINDOW_RESULTS.md index ef596669..c7b3ceee 100644 --- a/docs/CSA_CP_WINDOW_RESULTS.md +++ b/docs/CSA_CP_WINDOW_RESULTS.md @@ -1,7 +1,8 @@ # CSA CP selection at 128K and 256K -The [expanded three-way comparison](CSA_CP_THREE_WAY_RESULTS.md) retains these -ten measurements and adds 1M contexts and 22-/64-document packs, for 24 cases. +The [expanded three-way comparison](CSA_CP_THREE_WAY_RESULTS.md) covers 1M +contexts and 22-/64-document packs, for 24 cases, and has been rerun after the +equivalent host-preparation optimization. The historical ten-case statistics below remain scoped to this original study. This tests the public `cudnn_csa_select_cp` API on eight B300 GPUs. The workload diff --git a/docs/results/csa_cp_preparation_20261116.json b/docs/results/csa_cp_preparation_20261116.json new file mode 100644 index 00000000..bccabb52 --- /dev/null +++ b/docs/results/csa_cp_preparation_20261116.json @@ -0,0 +1,6770 @@ +{ + "scope": "Complete CSA operator forward/backward, including layouts and communication; no KDA/projections/MoE", + "platform": "B300 CP8", + "baseline_commit": "ea71df3b3bfad69141d1ee3ddd884ac9ccdb27ce", + "native_source_commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "960a403b1e804b087653fd5c8032781fd2d1249f37a6a7b0f2b80c3fca1896f6", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "853f7045ead4d230d4bd5b0a211d60861c840c6190c6502bfdcde310cf866bc2", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0", + "cudnn_backend": "9.20.0.48", + "cudnn_frontend_version_scope": "Wheel metadata; imported source identified by native_source_commit" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "arguments": { + "tokens": [ + 131072, + 262144, + 1048576 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "cases": { + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "7ca4febcee1eaf89e99486a3e54fddeeb01614dd16c887ac63a35df7e1fd86e5", + "policies": { + "contiguous": { + "step_ms": 47.692832946777344, + "prepare_ms": 0.7185455178841949, + "one_use_ms": 48.41137846466154, + "prepare_min_ms": 0.699331983923912, + "prepare_max_ms": 0.7794059347361326, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015779392555523083, + "dq": 0.000579412002925132, + "dkv": 0.0006017076308924175, + "d_index_q": 0.000795245110121862, + "d_weights": 0.0006388088783563453, + "d_pool_kv": 0.0007155220754445791, + "d_pool_score": 0.0007320226024776074, + "d_ape": 2.5403258387015062e-05 + }, + "max_loss_abs_diff": 9.89530235528946e-10, + "critical_step_samples_ms": [ + 47.38896179199219, + 47.49897766113281, + 47.80780792236328, + 47.66294479370117, + 47.76643371582031, + 47.85734558105469, + 47.692832946777344 + ], + "critical_prepare_samples_ms": [ + 0.7794059347361326, + 0.7474300218746066, + 0.7116730557754636, + 0.7364189950749278, + 0.7166480645537376, + 0.7204429712146521, + 0.6996119627729058, + 0.699331983923912 + ], + "baseline_prepare_ms": 0.7295145187526941 + }, + "zigzag": { + "step_ms": 46.274879455566406, + "prepare_ms": 0.8905025315470994, + "one_use_ms": 47.165381987113506, + "prepare_min_ms": 0.8781669894233346, + "prepare_max_ms": 2.297741943039, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015800177020671097, + "dq": 0.0005803295677451422, + "dkv": 0.000603814033831282, + "d_index_q": 0.0005765121709522783, + "d_weights": 0.0005104610024849347, + "d_pool_kv": 0.0007192810999619293, + "d_pool_score": 0.0007225948121424813, + "d_ape": 2.3842562812573465e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-10, + "critical_step_samples_ms": [ + 45.98963165283203, + 46.42329788208008, + 46.58995056152344, + 46.043582916259766, + 46.320030212402344, + 46.1190071105957, + 46.274879455566406 + ], + "critical_prepare_samples_ms": [ + 2.297741943039, + 0.9367750026285648, + 0.8781669894233346, + 0.8949730545282364, + 0.8860320085659623, + 0.9126270888373256, + 0.8803660748526454, + 0.8839479414746165 + ], + "baseline_prepare_ms": 0.9122525225393474 + }, + "planner": { + "step_ms": 45.82748794555664, + "prepare_ms": 7.445039984304458, + "one_use_ms": 53.2725279298611, + "prepare_min_ms": 7.354012923315167, + "prepare_max_ms": 7.553054951131344, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.001576013197553258, + "dq": 0.000578531419351737, + "dkv": 0.0005801127675590207, + "d_index_q": 0.0004534482180659951, + "d_weights": 0.00043037340000837636, + "d_pool_kv": 0.0007108593320852681, + "d_pool_score": 0.0007300664640860723, + "d_ape": 2.353131665386829e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-09, + "critical_step_samples_ms": [ + 45.80588912963867, + 46.03481674194336, + 45.82748794555664, + 45.93056106567383, + 45.71779251098633, + 46.30985641479492, + 45.72528076171875 + ], + "critical_prepare_samples_ms": [ + 7.553054951131344, + 7.538643083535135, + 7.417501998133957, + 7.354012923315167, + 7.4830540688708425, + 7.437848136760294, + 7.372890948317945, + 7.452231831848621 + ], + "baseline_prepare_ms": 7.54898099694401 + } + } + }, + "128k-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "23706ee0a647a3cb27010c2ed87cc4d8b06a80125c3b8c22123ae1a312a4a571", + "policies": { + "contiguous": { + "step_ms": 19.99852752685547, + "prepare_ms": 0.8981814607977867, + "one_use_ms": 20.896708987653255, + "prepare_min_ms": 0.8844570256769657, + "prepare_max_ms": 0.9815751109272242, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.5358557388634415e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.850226281255351e-05, + "d_pool_score": 6.240650825233437e-05, + "d_ape": 1.9435079133677194e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 19.997407913208008, + 19.980575561523438, + 20.005632400512695, + 20.014080047607422, + 19.99852752685547, + 19.980127334594727, + 20.010080337524414 + ], + "critical_prepare_samples_ms": [ + 0.9815751109272242, + 0.9244229877367616, + 0.8844570256769657, + 0.9146069642156363, + 0.884861103259027, + 0.8991919457912445, + 0.8856870699673891, + 0.8971709758043289 + ], + "baseline_prepare_ms": 0.9577220189385116 + }, + "zigzag": { + "step_ms": 20.669599533081055, + "prepare_ms": 3.7295810761861503, + "one_use_ms": 24.399180609267205, + "prepare_min_ms": 3.5765160573646426, + "prepare_max_ms": 4.66969795525074, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.199779774104606e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.823004119093153e-05, + "d_pool_score": 6.126991867025295e-05, + "d_ape": 1.8107487625478647e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.700927734375, + 20.724992752075195, + 20.669599533081055, + 20.676511764526367, + 20.666271209716797, + 20.65430450439453, + 20.660608291625977 + ], + "critical_prepare_samples_ms": [ + 3.811003058217466, + 4.470668034628034, + 3.6063099978491664, + 3.6481590941548347, + 4.66969795525074, + 3.6183729534968734, + 4.324503941461444, + 3.5765160573646426 + ], + "baseline_prepare_ms": 4.178662959020585 + }, + "planner": { + "step_ms": 19.990047454833984, + "prepare_ms": 1.1078330571763217, + "one_use_ms": 21.097880512010306, + "prepare_min_ms": 1.0634340578690171, + "prepare_max_ms": 1.1886698193848133, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.403263577176253e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 8.073828813471261e-05, + "d_pool_score": 7.995610208711404e-05, + "d_ape": 2.2451904402386835e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 19.990047454833984, + 20.148128509521484, + 19.977664947509766, + 20.00320053100586, + 19.976255416870117, + 20.004608154296875, + 19.986560821533203 + ], + "critical_prepare_samples_ms": [ + 1.1886698193848133, + 1.1112691136077046, + 1.1124929878860712, + 1.0784630430862308, + 1.130640972405672, + 1.1040159733965993, + 1.1043970007449389, + 1.0634340578690171 + ], + "baseline_prepare_ms": 1.2417834950610995 + } + } + }, + "128k-near-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "7a9bb46a066badd62c28275ab7d42939639e7bca37e919dc022bff73cf1a20fe", + "policies": { + "contiguous": { + "step_ms": 20.031328201293945, + "prepare_ms": 1.0363015462644398, + "one_use_ms": 21.067629747558385, + "prepare_min_ms": 0.9917590068653226, + "prepare_max_ms": 1.106114941649139, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016380789447513027, + "dq": 8.525440840760677e-05, + "dkv": 0.00013011699218693176, + "d_index_q": 2.42101830699352e-05, + "d_weights": 3.241630010282288e-05, + "d_pool_kv": 7.695981858658009e-05, + "d_pool_score": 9.499932585619792e-05, + "d_ape": 3.1762099590003316e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.028352737426758, + 19.99331283569336, + 20.05708885192871, + 20.031328201293945, + 20.022687911987305, + 20.038463592529297, + 20.053375244140625 + ], + "critical_prepare_samples_ms": [ + 1.106114941649139, + 1.0442269267514348, + 1.0163340484723449, + 1.0369500378146768, + 1.0006509255617857, + 1.0416280711069703, + 0.9917590068653226, + 1.0356530547142029 + ], + "baseline_prepare_ms": 0.9788154857233167 + }, + "zigzag": { + "step_ms": 20.757631301879883, + "prepare_ms": 4.550810961518437, + "one_use_ms": 25.30844226339832, + "prepare_min_ms": 4.380041034892201, + "prepare_max_ms": 6.088564987294376, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.00016408046524629192, + "dq": 8.506906905685405e-05, + "dkv": 0.00013016442209025538, + "d_index_q": 2.4266660285821442e-05, + "d_weights": 3.5896669106976304e-05, + "d_pool_kv": 9.384115935688863e-05, + "d_pool_score": 0.00012792608827345438, + "d_ape": 3.490957941984386e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.83123207092285, + 20.77631950378418, + 20.74355125427246, + 20.726688385009766, + 20.844192504882812, + 20.74188804626465, + 20.757631301879883 + ], + "critical_prepare_samples_ms": [ + 4.700917052105069, + 4.677826887927949, + 4.382802057079971, + 4.414985072799027, + 4.380041034892201, + 4.423795035108924, + 5.361750023439527, + 6.088564987294376 + ], + "baseline_prepare_ms": 4.162916971836239 + }, + "planner": { + "step_ms": 20.02444839477539, + "prepare_ms": 1.2869200436398387, + "one_use_ms": 21.31136843841523, + "prepare_min_ms": 1.2415660312399268, + "prepare_max_ms": 1.3624459970742464, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016313288531341212, + "dq": 8.540299959803365e-05, + "dkv": 0.00012140824119413083, + "d_index_q": 2.4035200300153297e-05, + "d_weights": 2.2592142747911436e-05, + "d_pool_kv": 7.511559012165556e-05, + "d_pool_score": 7.687588823617633e-05, + "d_ape": 2.910722067354833e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.129823684692383, + 20.039775848388672, + 20.014432907104492, + 20.02444839477539, + 20.013120651245117, + 20.06505584716797, + 20.01033592224121 + ], + "critical_prepare_samples_ms": [ + 1.3624459970742464, + 1.2910119257867336, + 1.301727956160903, + 1.2415660312399268, + 1.2881830334663391, + 1.2434010859578848, + 1.2856570538133383, + 1.2555728899315 + ], + "baseline_prepare_ms": 1.181907020509243 + } + } + }, + "128k-mixed-22": { + "documents": 22, + "tokens": 131072, + "source_record_sha256": "6ecd138552df3ddc716d8f2b5546c243019202c335e763cb31803e5a23d38a46", + "policies": { + "contiguous": { + "step_ms": 35.410911560058594, + "prepare_ms": 0.754816981498152, + "one_use_ms": 36.165728541556746, + "prepare_min_ms": 0.7329720538109541, + "prepare_max_ms": 1.6807179199531674, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0019090964634188628, + "dq": 0.0005760018331729727, + "dkv": 0.0006459942222257381, + "d_index_q": 0.0009173576570437036, + "d_weights": 0.0005682475663426924, + "d_pool_kv": 0.000817398720386681, + "d_pool_score": 0.0008227400563089687, + "d_ape": 1.4598269933704832e-05 + }, + "max_loss_abs_diff": 5.238689482212067e-10, + "critical_step_samples_ms": [ + 35.033023834228516, + 34.84620666503906, + 35.41068649291992, + 35.410911560058594, + 35.445377349853516, + 35.47119903564453, + 35.475006103515625 + ], + "critical_prepare_samples_ms": [ + 0.823180889710784, + 0.756045919843018, + 1.6807179199531674, + 0.7380660390481353, + 0.8030008757486939, + 0.753588043153286, + 0.7329720538109541, + 0.7516909390687943 + ], + "baseline_prepare_ms": 0.8113369694910944 + }, + "zigzag": { + "step_ms": 35.35753631591797, + "prepare_ms": 1.8669585115276277, + "one_use_ms": 37.2244948274456, + "prepare_min_ms": 1.8487819470465183, + "prepare_max_ms": 3.108761040493846, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0019284075136349816, + "dq": 0.0005817180180578517, + "dkv": 0.00046197630358230487, + "d_index_q": 0.0003621120560024531, + "d_weights": 0.00034954515220108934, + "d_pool_kv": 0.0006609639282456203, + "d_pool_score": 0.0006636759480983085, + "d_ape": 1.4089509206796339e-05 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 35.02534484863281, + 35.17510223388672, + 36.00688171386719, + 35.21503829956055, + 35.84960174560547, + 35.35753631591797, + 36.07881546020508 + ], + "critical_prepare_samples_ms": [ + 3.108761040493846, + 1.942998031154275, + 1.861648983322084, + 1.8494660034775734, + 1.8487819470465183, + 1.8964800983667374, + 1.8524520564824343, + 1.8722680397331715 + ], + "baseline_prepare_ms": 1.9275369704701006 + }, + "planner": { + "step_ms": 35.47750473022461, + "prepare_ms": 7.7040985925123096, + "one_use_ms": 43.18160332273692, + "prepare_min_ms": 7.597577874548733, + "prepare_max_ms": 9.156922926194966, + "selected_policy": "selection-local-3x", + "max_relative_rms": { + "output": 0.0019138774083845046, + "dq": 0.000577212651210394, + "dkv": 0.00046300417813060113, + "d_index_q": 0.00036281654146900024, + "d_weights": 0.00034608194650246384, + "d_pool_kv": 0.0006540298845914364, + "d_pool_score": 0.0006691398806773541, + "d_ape": 1.3458997363557325e-05 + }, + "max_loss_abs_diff": 4.0745362639427185e-10, + "critical_step_samples_ms": [ + 34.82156753540039, + 35.76873779296875, + 35.48476791381836, + 35.47750473022461, + 35.731231689453125, + 35.40195083618164, + 35.03801727294922 + ], + "critical_prepare_samples_ms": [ + 9.156922926194966, + 7.684374926611781, + 7.653841050341725, + 7.862776052206755, + 7.597577874548733, + 7.689073099754751, + 7.847677916288376, + 7.719124085269868 + ], + "baseline_prepare_ms": 7.6061730505898595 + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "e42faf3fe40f13f60990140dc3fa5dac55345f029f7783599374a9ac2e46649d", + "policies": { + "contiguous": { + "step_ms": 9.31715202331543, + "prepare_ms": 6.460049480665475, + "one_use_ms": 15.777201503980905, + "prepare_min_ms": 6.399138015694916, + "prepare_max_ms": 6.61184499040246, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8425583640526028e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.07692854412947e-05, + "d_pool_score": 4.358570392543705e-05, + "d_ape": 2.1843910536361686e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.288512229919434, + 9.269824028015137, + 9.289088249206543, + 9.31715202331543, + 9.563712120056152, + 9.406111717224121, + 9.327136039733887 + ], + "critical_prepare_samples_ms": [ + 6.61184499040246, + 6.471202941611409, + 6.399138015694916, + 6.513276020996273, + 6.425823085010052, + 6.459023919887841, + 6.449210923165083, + 6.4610750414431095 + ], + "baseline_prepare_ms": 6.7794660571962595 + }, + "zigzag": { + "step_ms": 11.095487594604492, + "prepare_ms": 43.545027961954474, + "one_use_ms": 54.64051555655897, + "prepare_min_ms": 43.15978987142444, + "prepare_max_ms": 159.60040991194546, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.823962386686128e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.014173056955901e-05, + "d_pool_score": 3.808940089626602e-05, + "d_ape": 2.3177467841882244e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.1212797164917, + 11.13651180267334, + 11.091487884521484, + 11.085920333862305, + 11.095487594604492, + 11.101375579833984, + 11.079903602600098 + ], + "critical_prepare_samples_ms": [ + 46.472262940369546, + 43.510618968866765, + 43.37981704156846, + 159.60040991194546, + 44.112925068475306, + 43.441104935482144, + 43.15978987142444, + 43.57943695504218 + ], + "baseline_prepare_ms": 47.22392506664619 + }, + "planner": { + "step_ms": 9.282464027404785, + "prepare_ms": 8.517480513546616, + "one_use_ms": 17.7999445409514, + "prepare_min_ms": 8.404429885558784, + "prepare_max_ms": 8.715566829778254, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8184966417286295e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.250769702550404e-05, + "d_pool_score": 3.6555913008951535e-05, + "d_ape": 2.173760194321718e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.282464027404785, + 9.294624328613281, + 9.262335777282715, + 9.278528213500977, + 9.307040214538574, + 9.284832000732422, + 9.274815559387207 + ], + "critical_prepare_samples_ms": [ + 8.67331400513649, + 8.40490497648716, + 8.536944980733097, + 8.404429885558784, + 8.61146894749254, + 8.442464983090758, + 8.498016046360135, + 8.715566829778254 + ], + "baseline_prepare_ms": 8.97316396003589 + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "eba0eefb74fef716f6c396b10698ddd865c6ba04c6b69c37170224765a3ca7af", + "policies": { + "contiguous": { + "step_ms": 9.996671676635742, + "prepare_ms": 3.5602174466475844, + "one_use_ms": 13.556889123283327, + "prepare_min_ms": 3.4920680336654186, + "prepare_max_ms": 3.646206925623119, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2234539065804016e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.212013067353867e-05, + "d_pool_score": 6.974185018420703e-05, + "d_ape": 2.2844736073180494e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.996671676635742, + 9.997088432312012, + 9.978528022766113, + 10.020832061767578, + 9.99020767211914, + 9.993856430053711, + 10.014687538146973 + ], + "critical_prepare_samples_ms": [ + 3.5972590558230877, + 3.5885649267584085, + 3.5170429619029164, + 3.4920680336654186, + 3.4954349976032972, + 3.593871952034533, + 3.5318699665367603, + 3.646206925623119 + ], + "baseline_prepare_ms": 3.743164998013526 + }, + "zigzag": { + "step_ms": 11.19603157043457, + "prepare_ms": 22.693975479342043, + "one_use_ms": 33.890007049776614, + "prepare_min_ms": 22.371524944901466, + "prepare_max_ms": 136.65311911609024, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.0477249966524713e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.326994000792746e-05, + "d_pool_score": 6.248366916465234e-05, + "d_ape": 2.302149020581522e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.188480377197266, + 11.215295791625977, + 11.511136054992676, + 11.195136070251465, + 11.212479591369629, + 11.19603157043457, + 11.186783790588379 + ], + "critical_prepare_samples_ms": [ + 24.338410003110766, + 33.537326962687075, + 22.391692851670086, + 22.684970987029374, + 22.371524944901466, + 22.702979971654713, + 22.49394287355244, + 136.65311911609024 + ], + "baseline_prepare_ms": 24.085226003080606 + }, + "planner": { + "step_ms": 10.009087562561035, + "prepare_ms": 4.715179966297001, + "one_use_ms": 14.724267528858036, + "prepare_min_ms": 4.600052139721811, + "prepare_max_ms": 4.814712912775576, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.962048572951886e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.034841683489502e-05, + "d_pool_score": 6.365301246893881e-05, + "d_ape": 2.1817777204410688e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.988896369934082, + 10.01091194152832, + 9.999456405639648, + 10.009087562561035, + 10.284640312194824, + 10.013312339782715, + 9.988832473754883 + ], + "critical_prepare_samples_ms": [ + 4.814712912775576, + 4.696596995927393, + 4.743945901282132, + 4.629171919077635, + 4.722174955531955, + 4.600052139721811, + 4.7081849770620465, + 4.768294980749488 + ], + "baseline_prepare_ms": 4.955698561388999 + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "dbc1324660bfdc95476a1f0d82d936cf16edd8a9cc6990afa6a37e17401f5dc6", + "policies": { + "contiguous": { + "step_ms": 33.102272033691406, + "prepare_ms": 7.131611928343773, + "one_use_ms": 40.23388396203518, + "prepare_min_ms": 7.029787986539304, + "prepare_max_ms": 7.579102064482868, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0021210770683415432, + "dq": 0.0006697244454984808, + "dkv": 0.0006752772580638095, + "d_index_q": 0.00041775240877173306, + "d_weights": 0.0003214792609200598, + "d_pool_kv": 0.000576733611039875, + "d_pool_score": 0.0005744006365142611, + "d_ape": 6.907255374051929e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 33.08230209350586, + 33.068511962890625, + 33.102272033691406, + 33.116512298583984, + 33.155582427978516, + 33.54044723510742, + 33.07088088989258 + ], + "critical_prepare_samples_ms": [ + 7.579102064482868, + 7.029787986539304, + 7.084714015945792, + 7.242373889312148, + 7.1294138906523585, + 7.133809966035187, + 7.39094999153167, + 7.094503031112254 + ], + "baseline_prepare_ms": 7.641692471224815 + }, + "zigzag": { + "step_ms": 38.75347137451172, + "prepare_ms": 44.199544994626194, + "one_use_ms": 82.95301636913791, + "prepare_min_ms": 43.52790501434356, + "prepare_max_ms": 159.79488100856543, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0021340651638746743, + "dq": 0.0006454264679630054, + "dkv": 0.0005374193486070684, + "d_index_q": 0.000355993913482489, + "d_weights": 0.00032167200796056337, + "d_pool_kv": 0.0005637152047024604, + "d_pool_score": 0.0005797494053691072, + "d_ape": 6.916447167492779e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.73811340332031, + 38.75721740722656, + 38.72956848144531, + 38.75347137451172, + 38.723487854003906, + 38.79964828491211, + 38.764190673828125 + ], + "critical_prepare_samples_ms": [ + 45.98132590763271, + 43.77324297092855, + 43.52790501434356, + 159.79488100856543, + 44.65641709975898, + 44.314848026260734, + 43.64072694443166, + 44.084241962991655 + ], + "baseline_prepare_ms": 45.72551843011752 + }, + "planner": { + "step_ms": 33.08563232421875, + "prepare_ms": 9.258774924091995, + "one_use_ms": 42.344407248310745, + "prepare_min_ms": 9.213590994477272, + "prepare_max_ms": 9.48783487547189, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002089306735283265, + "dq": 0.0006319712339703342, + "dkv": 0.0005306332292867902, + "d_index_q": 0.0003543899517389669, + "d_weights": 0.00031634301834457875, + "d_pool_kv": 0.0005684620661380382, + "d_pool_score": 0.0005770038318201088, + "d_ape": 6.819908090487931e-06 + }, + "max_loss_abs_diff": 4.0745362639427185e-10, + "critical_step_samples_ms": [ + 33.08563232421875, + 33.09142303466797, + 33.06816101074219, + 33.11497497558594, + 33.05699157714844, + 33.14908981323242, + 33.06867218017578 + ], + "critical_prepare_samples_ms": [ + 9.332454996183515, + 9.246370987966657, + 9.213590994477272, + 9.24178201239556, + 9.244293090887368, + 9.430562960915267, + 9.271178860217333, + 9.48783487547189 + ], + "baseline_prepare_ms": 28.312876995187253 + } + } + }, + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "64a03500c3f1fe411b7cadd8009104ede8ca9b139cd0732acb13828bd86add34", + "policies": { + "contiguous": { + "step_ms": 32.327518463134766, + "prepare_ms": 7.331589062232524, + "one_use_ms": 39.65910752536729, + "prepare_min_ms": 7.055153022520244, + "prepare_max_ms": 9.122443967498839, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015056029919507346, + "dq": 0.00045665164417140727, + "dkv": 0.0004702181088295793, + "d_index_q": 0.0003533783738426617, + "d_weights": 0.000315067727554275, + "d_pool_kv": 0.0005386844431895517, + "d_pool_score": 0.0005634558855019262, + "d_ape": 3.905854431531449e-06 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 32.32707214355469, + 32.327518463134766, + 32.334590911865234, + 32.341793060302734, + 32.315391540527344, + 32.321632385253906, + 32.36006546020508 + ], + "critical_prepare_samples_ms": [ + 7.307856110855937, + 9.122443967498839, + 8.694347110576928, + 7.367579033598304, + 7.055153022520244, + 7.122335839085281, + 7.355322013609111, + 7.146980031393468 + ], + "baseline_prepare_ms": 7.578457531053573 + }, + "zigzag": { + "step_ms": 37.883487701416016, + "prepare_ms": 45.2110730111599, + "one_use_ms": 83.09456071257591, + "prepare_min_ms": 43.57520886696875, + "prepare_max_ms": 169.3743329960853, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016271139134036983, + "dq": 0.0004914893001159871, + "dkv": 0.0004848398943906415, + "d_index_q": 0.0003562240851567133, + "d_weights": 0.0003187078936891727, + "d_pool_kv": 0.0005284217741949873, + "d_pool_score": 0.000541357985562331, + "d_ape": 3.691615037628746e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 37.89449691772461, + 37.91689682006836, + 37.87980651855469, + 37.855712890625, + 37.8691520690918, + 37.9150390625, + 37.883487701416016 + ], + "critical_prepare_samples_ms": [ + 45.877177035436034, + 45.723085990175605, + 50.25763192679733, + 169.3743329960853, + 44.69906003214419, + 44.112405041232705, + 43.57520886696875, + 44.34900498017669 + ], + "baseline_prepare_ms": 45.84232851630077 + }, + "planner": { + "step_ms": 32.330718994140625, + "prepare_ms": 9.115683031268418, + "one_use_ms": 41.44640202540904, + "prepare_min_ms": 9.055349975824356, + "prepare_max_ms": 9.575794916599989, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015061092586627954, + "dq": 0.00045634085695053377, + "dkv": 0.00046791157809618843, + "d_index_q": 0.000356679693690852, + "d_weights": 0.0003092211654689727, + "d_pool_kv": 0.0005367827264826421, + "d_pool_score": 0.0005700652608558374, + "d_ape": 3.810351760870522e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.33180618286133, + 32.330718994140625, + 32.31001663208008, + 32.39107131958008, + 32.3276481628418, + 32.335872650146484, + 32.297183990478516 + ], + "critical_prepare_samples_ms": [ + 9.234660072252154, + 9.0619339607656, + 9.575794916599989, + 9.059371892362833, + 9.055349975824356, + 9.288421017117798, + 9.15169098880142, + 9.079675073735416 + ], + "baseline_prepare_ms": 12.12866249261424 + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "68b017a16ef4cf9fea1c3c9c0cc1a272786364ef4943afcf9a72f5eb3c9f90b7", + "policies": { + "contiguous": { + "step_ms": 115.74121856689453, + "prepare_ms": 1.0700724669732153, + "one_use_ms": 116.81129103386775, + "prepare_min_ms": 1.0450789704918861, + "prepare_max_ms": 1.2837169924750924, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.001482133304629509, + "dq": 0.0005028985179263687, + "dkv": 0.0010282109682620694, + "d_index_q": 0.0007703566859007169, + "d_weights": 0.0006286309166410099, + "d_pool_kv": 0.0013354595252499009, + "d_pool_score": 0.0013699629151741153, + "d_ape": 2.1869040480447828e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 114.95225524902344, + 115.74121856689453, + 116.17721557617188, + 114.34761810302734, + 116.25465393066406, + 114.593505859375, + 116.58633422851562 + ], + "critical_prepare_samples_ms": [ + 1.2837169924750924, + 1.1237040162086487, + 1.0450789704918861, + 1.0568589204922318, + 1.0688218753784895, + 1.0853169951587915, + 1.058198045939207, + 1.0713230585679412 + ], + "baseline_prepare_ms": 1.1161149595864117 + }, + "zigzag": { + "step_ms": 105.46825408935547, + "prepare_ms": 1.10343296546489, + "one_use_ms": 106.57168705482036, + "prepare_min_ms": 1.090729027055204, + "prepare_max_ms": 2.6004710234701633, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014829370691384724, + "dq": 0.000502969985804172, + "dkv": 0.0010306683280579473, + "d_index_q": 0.0007713429886426259, + "d_weights": 0.0006304765275187952, + "d_pool_kv": 0.0013371215867479295, + "d_pool_score": 0.00138344136125153, + "d_ape": 2.0698491606366296e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 105.06502532958984, + 105.46825408935547, + 104.91980743408203, + 105.63510131835938, + 105.70121765136719, + 105.19264221191406, + 105.93225860595703 + ], + "critical_prepare_samples_ms": [ + 2.6004710234701633, + 1.137568149715662, + 1.1008289875462651, + 1.1032898910343647, + 1.1035760398954153, + 1.1218799045309424, + 1.090729027055204, + 1.0995609918609262 + ], + "baseline_prepare_ms": 1.1357140028849244 + }, + "planner": { + "step_ms": 104.23197174072266, + "prepare_ms": 7.709605968557298, + "one_use_ms": 111.94157770927995, + "prepare_min_ms": 7.618593983352184, + "prepare_max_ms": 8.737536030821502, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014859596400049872, + "dq": 0.0005041720349634445, + "dkv": 0.001030506433322175, + "d_index_q": 0.0006257742363324554, + "d_weights": 0.0005596577742386472, + "d_pool_kv": 0.001329566057037352, + "d_pool_score": 0.00137685351760298, + "d_ape": 2.1145177390575036e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 104.40544128417969, + 104.23197174072266, + 104.6523208618164, + 104.536865234375, + 104.16582489013672, + 104.04156494140625, + 104.11017608642578 + ], + "critical_prepare_samples_ms": [ + 7.696714950725436, + 8.737536030821502, + 7.659266120754182, + 7.647061021998525, + 8.64289712626487, + 7.72249698638916, + 7.618593983352184, + 8.357836049981415 + ], + "baseline_prepare_ms": 7.791060954332352 + } + } + }, + "256k-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "6f1d4a0da49698ef15f82a432e981a02d9d084b70fe81e2b56d8f189b043cd6a", + "policies": { + "contiguous": { + "step_ms": 50.320064544677734, + "prepare_ms": 0.9923935285769403, + "one_use_ms": 51.312458073254675, + "prepare_min_ms": 0.9678329806774855, + "prepare_max_ms": 1.1316180462017655, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008982351679445784, + "dq": 0.0004175412751329343, + "dkv": 0.0002881395432601503, + "d_index_q": 0.0001082730122096085, + "d_weights": 9.702894207094345e-05, + "d_pool_kv": 0.0001972150021053803, + "d_pool_score": 0.00019493588963177602, + "d_ape": 6.910623983780962e-06 + }, + "max_loss_abs_diff": 3.4924596548080444e-10, + "critical_step_samples_ms": [ + 50.30031967163086, + 50.302433013916016, + 50.362239837646484, + 50.36908721923828, + 50.320064544677734, + 50.36316680908203, + 50.31760025024414 + ], + "critical_prepare_samples_ms": [ + 1.1316180462017655, + 1.0181260295212269, + 0.9678329806774855, + 0.9887899504974484, + 0.9702970273792744, + 0.9985698852688074, + 0.9720890084281564, + 0.9959971066564322 + ], + "baseline_prepare_ms": 1.0602579568512738 + }, + "zigzag": { + "step_ms": 51.75715255737305, + "prepare_ms": 3.72084544505924, + "one_use_ms": 55.47799800243229, + "prepare_min_ms": 3.68827604688704, + "prepare_max_ms": 4.462504060938954, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009079115543848496, + "dq": 0.0004210909515919965, + "dkv": 0.0002940172927688538, + "d_index_q": 0.00010857831208366282, + "d_weights": 9.552466137385803e-05, + "d_pool_kv": 0.00020868563840840786, + "d_pool_score": 0.0002198801712107116, + "d_ape": 7.540515901910232e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 51.67875289916992, + 51.771583557128906, + 51.7336311340332, + 51.81840133666992, + 51.714080810546875, + 51.75715255737305, + 51.81683349609375 + ], + "critical_prepare_samples_ms": [ + 4.01491392403841, + 3.7799880374222994, + 3.68827604688704, + 3.7137470208108425, + 3.7259499076753855, + 3.7157409824430943, + 4.462504060938954, + 3.702106885612011 + ], + "baseline_prepare_ms": 4.144389531575143 + }, + "planner": { + "step_ms": 50.36211013793945, + "prepare_ms": 1.2112134136259556, + "one_use_ms": 51.57332355156541, + "prepare_min_ms": 1.1628499487414956, + "prepare_max_ms": 1.3246370945125818, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008988738864725319, + "dq": 0.00041697524787721483, + "dkv": 0.0002882229310364065, + "d_index_q": 0.00010873610457156427, + "d_weights": 9.523323253939226e-05, + "d_pool_kv": 0.00021093823450393177, + "d_pool_score": 0.00021293712669284863, + "d_ape": 6.762282048261405e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 50.2674560546875, + 50.269630432128906, + 50.581024169921875, + 50.36211013793945, + 50.84463882446289, + 50.260257720947266, + 50.700191497802734 + ], + "critical_prepare_samples_ms": [ + 1.3246370945125818, + 1.218253979459405, + 1.2112829135730863, + 1.1899759992957115, + 1.211143913678825, + 1.1628499487414956, + 1.2666999828070402, + 1.1778250336647034 + ], + "baseline_prepare_ms": 1.275484450161457 + } + } + }, + "256k-near-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "921d5ba93c6562588668d8e4fe34833a7208e52c5c1a0b8b6c5b728539491a9d", + "policies": { + "contiguous": { + "step_ms": 50.34755325317383, + "prepare_ms": 0.9968500817194581, + "one_use_ms": 51.344403334893286, + "prepare_min_ms": 0.9588320972397923, + "prepare_max_ms": 1.0964140528813004, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009002720083858091, + "dq": 0.00041658039734501545, + "dkv": 0.00029032119524166356, + "d_index_q": 0.0001083702031927515, + "d_weights": 9.270341749291116e-05, + "d_pool_kv": 0.00021243719360570197, + "d_pool_score": 0.00021657144439094105, + "d_ape": 7.183122931513331e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.26278305053711, + 50.286529541015625, + 50.332576751708984, + 50.415008544921875, + 50.4021110534668, + 50.41408157348633, + 50.34755325317383 + ], + "critical_prepare_samples_ms": [ + 1.0964140528813004, + 1.005454920232296, + 0.968775013461709, + 1.0026779491454363, + 0.9609600529074669, + 0.9930110536515713, + 0.9588320972397923, + 1.000689109787345 + ], + "baseline_prepare_ms": 1.0578265646472573 + }, + "zigzag": { + "step_ms": 51.7072639465332, + "prepare_ms": 3.7724130088463426, + "one_use_ms": 55.479676955379546, + "prepare_min_ms": 3.713970072567463, + "prepare_max_ms": 4.540979978628457, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009040166709284812, + "dq": 0.0004216716611809067, + "dkv": 0.00028856434640319855, + "d_index_q": 0.00010881543305361698, + "d_weights": 9.496546039412519e-05, + "d_pool_kv": 0.0002074227586177013, + "d_pool_score": 0.00020241674597203904, + "d_ape": 6.808701999575195e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 51.7072639465332, + 51.767616271972656, + 51.7031364440918, + 51.71673583984375, + 51.699039459228516, + 51.76015853881836, + 51.689247131347656 + ], + "critical_prepare_samples_ms": [ + 4.038910963572562, + 3.892948036082089, + 3.727796138264239, + 3.766917041502893, + 3.743300912901759, + 3.713970072567463, + 4.540979978628457, + 3.777908976189792 + ], + "baseline_prepare_ms": 4.253186983987689 + }, + "planner": { + "step_ms": 50.30198287963867, + "prepare_ms": 1.2103734770789742, + "one_use_ms": 51.512356356717646, + "prepare_min_ms": 1.165413879789412, + "prepare_max_ms": 1.283300924114883, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009011720652930045, + "dq": 0.00041697590209048004, + "dkv": 0.0002890001975034279, + "d_index_q": 0.00010860096710106817, + "d_weights": 9.628555678061784e-05, + "d_pool_kv": 0.0002099978400113182, + "d_pool_score": 0.00021443311522890485, + "d_ape": 6.98920313885421e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.25263977050781, + 50.36908721923828, + 50.4403190612793, + 50.30198287963867, + 50.267486572265625, + 50.27724838256836, + 50.336639404296875 + ], + "critical_prepare_samples_ms": [ + 1.283300924114883, + 1.2185070663690567, + 1.224570907652378, + 1.1921209516003728, + 1.2027559569105506, + 1.165413879789412, + 1.217990997247398, + 1.1812089942395687 + ], + "baseline_prepare_ms": 1.2725520064122975 + } + } + }, + "256k-mixed-22": { + "documents": 22, + "tokens": 262144, + "source_record_sha256": "fe159c639833d743955457884a4611ca25e984548b1f3208042b4a6bb301638c", + "policies": { + "contiguous": { + "step_ms": 90.67167663574219, + "prepare_ms": 1.1164660099893808, + "one_use_ms": 91.78814264573157, + "prepare_min_ms": 1.0949659626930952, + "prepare_max_ms": 1.2457289267331362, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00143689993907933, + "dq": 0.0004899801690773162, + "dkv": 0.0007826892603974143, + "d_index_q": 0.000796723694054483, + "d_weights": 0.0006164320797316369, + "d_pool_kv": 0.0012264280539640705, + "d_pool_score": 0.0012747699641356362, + "d_ape": 1.3222093515720844e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 90.87088012695312, + 90.7339859008789, + 90.64134216308594, + 89.80009460449219, + 90.7357406616211, + 90.02569580078125, + 90.67167663574219 + ], + "critical_prepare_samples_ms": [ + 1.2457289267331362, + 1.1465109419077635, + 1.0949659626930952, + 1.1196030536666512, + 1.100623863749206, + 1.1375189060345292, + 1.0985910193994641, + 1.1133289663121104 + ], + "baseline_prepare_ms": 1.150823023635894 + }, + "zigzag": { + "step_ms": 88.22876739501953, + "prepare_ms": 2.2970764548517764, + "one_use_ms": 90.52584384987131, + "prepare_min_ms": 2.2732780780643225, + "prepare_max_ms": 15.867101028561592, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.00144284104523375, + "dq": 0.0004920673948700904, + "dkv": 0.0006368864218746345, + "d_index_q": 0.0005277277670036296, + "d_weights": 0.00044285628176593295, + "d_pool_kv": 0.0006697211734598175, + "d_pool_score": 0.0006819690109157976, + "d_ape": 1.273986864151461e-05 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 88.02159881591797, + 88.35001373291016, + 88.56687927246094, + 87.62576293945312, + 88.37532806396484, + 88.22876739501953, + 87.9927978515625 + ], + "critical_prepare_samples_ms": [ + 10.660310974344611, + 15.867101028561592, + 2.304245950654149, + 2.2795250406488776, + 2.2899069590494037, + 2.288761897943914, + 2.2732780780643225, + 2.6616440154612064 + ], + "baseline_prepare_ms": 2.339682076126337 + }, + "planner": { + "step_ms": 87.58985900878906, + "prepare_ms": 8.515741501469165, + "one_use_ms": 96.10560051025823, + "prepare_min_ms": 8.127353037707508, + "prepare_max_ms": 9.741527959704399, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0014447715952959578, + "dq": 0.0004925536881693634, + "dkv": 0.0007865823928776758, + "d_index_q": 0.0008311647195845252, + "d_weights": 0.0006640931260018509, + "d_pool_kv": 0.001227268093921778, + "d_pool_score": 0.0012784182647735045, + "d_ape": 1.3639331200697894e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 87.6021728515625, + 87.31673431396484, + 87.58985900878906, + 87.67081451416016, + 87.4035873413086, + 87.91382598876953, + 87.31298828125 + ], + "critical_prepare_samples_ms": [ + 8.778966031968594, + 8.252516970969737, + 9.164304006844759, + 8.212398970499635, + 9.741527959704399, + 8.127353037707508, + 9.139055968262255, + 8.186427876353264 + ], + "baseline_prepare_ms": 8.772762026637793 + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "c3e7815a901d92f31141e0604e18d7bc895fd5f085746cc903da1efe1d6ff7d1", + "policies": { + "contiguous": { + "step_ms": 18.964704513549805, + "prepare_ms": 7.2258434956893325, + "one_use_ms": 26.190548009239137, + "prepare_min_ms": 6.673496915027499, + "prepare_max_ms": 10.859394096769392, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.126639010558396e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.658018032939693e-05, + "d_pool_score": 4.361095036650771e-05, + "d_ape": 1.4700628800457994e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 18.96460723876953, + 18.953855514526367, + 18.956928253173828, + 18.976831436157227, + 18.967615127563477, + 18.964704513549805, + 19.011167526245117 + ], + "critical_prepare_samples_ms": [ + 10.859394096769392, + 7.232020958326757, + 7.296605035662651, + 6.673496915027499, + 7.219666033051908, + 7.159358123317361, + 7.325196056626737, + 7.035364978946745 + ], + "baseline_prepare_ms": 6.907838920596987 + }, + "zigzag": { + "step_ms": 22.546016693115234, + "prepare_ms": 45.124090975150466, + "one_use_ms": 67.6701076682657, + "prepare_min_ms": 43.87128900270909, + "prepare_max_ms": 163.06352394167334, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3271739597648486e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.455468819835076e-05, + "d_pool_score": 5.371778108792677e-05, + "d_ape": 1.5063328225066925e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 22.546239852905273, + 22.5980167388916, + 22.535808563232422, + 22.534208297729492, + 22.546144485473633, + 22.53615951538086, + 22.546016693115234 + ], + "critical_prepare_samples_ms": [ + 45.488978968933225, + 44.75626104976982, + 43.87128900270909, + 163.06352394167334, + 44.75920298136771, + 45.767895062454045, + 44.65588403400034, + 45.569678070023656 + ], + "baseline_prepare_ms": 46.86611460056156 + }, + "planner": { + "step_ms": 18.937055587768555, + "prepare_ms": 9.278230951167643, + "one_use_ms": 28.215286538936198, + "prepare_min_ms": 9.084169985726476, + "prepare_max_ms": 9.426652104593813, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.496402962841108e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.0054426196458054e-05, + "d_pool_score": 4.135736654968628e-05, + "d_ape": 1.2068528167060991e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 18.950944900512695, + 18.939456939697266, + 18.937055587768555, + 18.931711196899414, + 18.945024490356445, + 18.936767578125, + 18.926591873168945 + ], + "critical_prepare_samples_ms": [ + 9.400052018463612, + 9.135371074080467, + 9.426652104593813, + 9.084169985726476, + 9.326501982286572, + 9.229959920048714, + 9.1595669509843, + 9.419546113349497 + ], + "baseline_prepare_ms": 8.967155939899385 + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "7a6daf1992407b30c4ae176c4adb7995e06aa0300d429547621f5fe178c8a4b1", + "policies": { + "contiguous": { + "step_ms": 21.74198341369629, + "prepare_ms": 3.678025444969535, + "one_use_ms": 25.420008858665824, + "prepare_min_ms": 3.6021980922669172, + "prepare_max_ms": 3.9428939344361424, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.417560025712104e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 7.005921384441596e-05, + "d_pool_score": 5.8958733777925906e-05, + "d_ape": 1.631341031376204e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.741151809692383, + 21.74300765991211, + 21.733728408813477, + 21.758752822875977, + 21.73583984375, + 21.74198341369629, + 21.77382469177246 + ], + "critical_prepare_samples_ms": [ + 3.671723883599043, + 3.7506880471482873, + 3.645248943939805, + 3.6927630426362157, + 3.6021980922669172, + 3.684327006340027, + 3.6574859404936433, + 3.9428939344361424 + ], + "baseline_prepare_ms": 3.9414139464497566 + }, + "zigzag": { + "step_ms": 24.20911979675293, + "prepare_ms": 23.415951000060886, + "one_use_ms": 47.625070796813816, + "prepare_min_ms": 22.827126900665462, + "prepare_max_ms": 139.44511592853814, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.379937858947025e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.0763077851366874e-05, + "d_pool_score": 5.891486019054288e-05, + "d_ape": 1.6771080713138341e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 24.205888748168945, + 24.241024017333984, + 24.198047637939453, + 24.33344078063965, + 24.214048385620117, + 24.20911979675293, + 24.200191497802734 + ], + "critical_prepare_samples_ms": [ + 24.720999062992632, + 23.846879019401968, + 22.83283998258412, + 22.985022980719805, + 24.500835104845464, + 22.827126900665462, + 22.859422024339437, + 139.44511592853814 + ], + "baseline_prepare_ms": 24.67734646052122 + }, + "planner": { + "step_ms": 21.773439407348633, + "prepare_ms": 4.805412027053535, + "one_use_ms": 26.578851434402168, + "prepare_min_ms": 4.7372711123898625, + "prepare_max_ms": 4.933310090564191, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.865546995903407e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.477011134407135e-05, + "d_pool_score": 4.405915600329243e-05, + "d_ape": 1.4306253170553605e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.733535766601562, + 21.773439407348633, + 21.731647491455078, + 21.738143920898438, + 21.783647537231445, + 22.051231384277344, + 22.021024703979492 + ], + "critical_prepare_samples_ms": [ + 4.933310090564191, + 4.814392072148621, + 4.882890032604337, + 4.7437300672754645, + 4.796431981958449, + 4.772201064042747, + 4.873599042184651, + 4.7372711123898625 + ], + "baseline_prepare_ms": 5.023348960094154 + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "7446722df043d987c1f521ffc1939cab03422244a16d5f3d3765383e4fe0713d", + "policies": { + "contiguous": { + "step_ms": 83.70726776123047, + "prepare_ms": 8.869351528119296, + "one_use_ms": 92.57661928934976, + "prepare_min_ms": 8.712492999620736, + "prepare_max_ms": 10.470369947142899, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017232232291779785, + "dq": 0.0005228538978110929, + "dkv": 0.000691125497722852, + "d_index_q": 0.00036843245190847605, + "d_weights": 0.0003596773286925188, + "d_pool_kv": 0.0013194996004303847, + "d_pool_score": 0.0013134479113316487, + "d_ape": 8.920837294635517e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 84.19203186035156, + 83.22354888916016, + 83.92806243896484, + 83.70726776123047, + 84.4559326171875, + 83.31343841552734, + 83.37763214111328 + ], + "critical_prepare_samples_ms": [ + 10.470369947142899, + 9.009449044242501, + 8.902203058823943, + 8.816724992357194, + 8.712492999620736, + 8.713754126802087, + 9.297300945036113, + 8.836499997414649 + ], + "baseline_prepare_ms": 9.585648542270064 + }, + "zigzag": { + "step_ms": 95.86224365234375, + "prepare_ms": 95.58228251989931, + "one_use_ms": 191.44452617224306, + "prepare_min_ms": 46.32592911366373, + "prepare_max_ms": 165.51054792944342, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017361334114495666, + "dq": 0.0005265492561785378, + "dkv": 0.0004599309949688459, + "d_index_q": 0.0003699972474757121, + "d_weights": 0.00036300246633119064, + "d_pool_kv": 0.0006498637378581222, + "d_pool_score": 0.0006531249898699503, + "d_ape": 7.658629854388498e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 95.64492797851562, + 95.81046295166016, + 95.86739349365234, + 96.00326538085938, + 95.86224365234375, + 96.04275512695312, + 95.80735778808594 + ], + "critical_prepare_samples_ms": [ + 48.216517083346844, + 165.51054792944342, + 147.26835896726698, + 47.15038393624127, + 46.32592911366373, + 46.40162189025432, + 142.94804795645177, + 146.32742700632662 + ], + "baseline_prepare_ms": 92.28663396788761 + }, + "planner": { + "step_ms": 82.06880187988281, + "prepare_ms": 24.085616052616388, + "one_use_ms": 106.1544179324992, + "prepare_min_ms": 23.79455603659153, + "prepare_max_ms": 108.25915494933724, + "selected_policy": "selection-local-5x", + "max_relative_rms": { + "output": 0.001736570688522341, + "dq": 0.0005270035493433086, + "dkv": 0.0005767161784870217, + "d_index_q": 0.00039398350109750283, + "d_weights": 0.00036772447259371074, + "d_pool_kv": 0.0006438267350921343, + "d_pool_score": 0.0006413864917895826, + "d_ape": 8.028800212264691e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 82.06880187988281, + 81.95337677001953, + 81.74278259277344, + 82.35673522949219, + 81.7702407836914, + 82.2399673461914, + 82.25609588623047 + ], + "critical_prepare_samples_ms": [ + 23.79455603659153, + 24.032195913605392, + 24.03438393957913, + 23.953115101903677, + 24.136848165653646, + 24.561780970543623, + 108.25915494933724, + 24.656706023961306 + ], + "baseline_prepare_ms": 32.668465457390994 + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "dabbfa12c68f3a6cd8dbe7612e5f54ad194e004ecc568eeeeffa174212da0761", + "policies": { + "contiguous": { + "step_ms": 98.90694427490234, + "prepare_ms": 8.876458974555135, + "one_use_ms": 107.78340324945748, + "prepare_min_ms": 8.691001101396978, + "prepare_max_ms": 9.725504904054105, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011522479235681476, + "dq": 0.00042835242305097246, + "dkv": 0.0004943333278686983, + "d_index_q": 0.000657896894736068, + "d_weights": 0.0005964786262765446, + "d_pool_kv": 0.0006674154267850759, + "d_pool_score": 0.0006874466443201366, + "d_ape": 6.1667568045954095e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 98.90694427490234, + 98.74559783935547, + 100.46985626220703, + 99.39778900146484, + 98.54434967041016, + 98.55680084228516, + 99.67833709716797 + ], + "critical_prepare_samples_ms": [ + 9.50452999677509, + 8.7447389960289, + 8.844938012771308, + 8.691001101396978, + 8.871630998328328, + 8.881286950781941, + 9.725504904054105, + 9.013960952870548 + ], + "baseline_prepare_ms": 9.696168475784361 + }, + "zigzag": { + "step_ms": 115.310302734375, + "prepare_ms": 148.14059244235978, + "one_use_ms": 263.4508951767348, + "prepare_min_ms": 47.415707958862185, + "prepare_max_ms": 176.64630699437112, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0010866654170538991, + "dq": 0.0003696364920229251, + "dkv": 0.0003946430012396241, + "d_index_q": 0.0003810622253103829, + "d_weights": 0.0003747132595040899, + "d_pool_kv": 0.0006645926398509092, + "d_pool_score": 0.0006783532515969929, + "d_ape": 5.437880086557659e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 115.310302734375, + 115.21078491210938, + 115.49664306640625, + 115.5719985961914, + 115.37644958496094, + 115.2808609008789, + 115.2738265991211 + ], + "critical_prepare_samples_ms": [ + 140.08078002370894, + 156.66547801811248, + 176.64630699437112, + 47.415707958862185, + 48.025651019997895, + 144.5655939169228, + 159.11502100061625, + 151.71559096779674 + ], + "baseline_prepare_ms": 147.0409840112552 + }, + "planner": { + "step_ms": 92.09024047851562, + "prepare_ms": 23.416623007506132, + "one_use_ms": 115.50686348602176, + "prepare_min_ms": 22.514423937536776, + "prepare_max_ms": 127.23819306120276, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0010924404756515048, + "dq": 0.000392027321823514, + "dkv": 0.0004789729076346326, + "d_index_q": 0.000595231909155876, + "d_weights": 0.0005075082975035412, + "d_pool_kv": 0.0007611845879591085, + "d_pool_score": 0.0007714277449577657, + "d_ape": 6.021118465344542e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 91.97811126708984, + 92.4843521118164, + 91.98320007324219, + 92.22860717773438, + 91.90172576904297, + 92.09024047851562, + 92.3569564819336 + ], + "critical_prepare_samples_ms": [ + 22.948727943003178, + 109.54794602002949, + 22.841048892587423, + 23.339509032666683, + 22.514423937536776, + 24.088384001515806, + 127.23819306120276, + 23.49373698234558 + ], + "baseline_prepare_ms": 31.8146159988828 + } + } + }, + "1024k-single": { + "documents": 1, + "tokens": 1048576, + "source_record_sha256": "9119bf217085106d62d4a79a61271150ec5fd5efc62dc415a0070fc2ecfc741e", + "policies": { + "contiguous": { + "step_ms": 852.262451171875, + "prepare_ms": 2.238098415546119, + "one_use_ms": 854.5005495874211, + "prepare_min_ms": 2.167516970075667, + "prepare_max_ms": 2.5991599541157484, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011273030335427169, + "dq": 0.0003575866171757266, + "dkv": 0.00041255472186497554, + "d_index_q": 0.000522703724234038, + "d_weights": 0.000503204507333504, + "d_pool_kv": 0.0007366385730565614, + "d_pool_score": 0.000730156729966756, + "d_ape": 1.2261158117605297e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 855.7032470703125, + 852.262451171875, + 851.0695190429688, + 850.6251220703125, + 851.63916015625, + 852.8311157226562, + 854.872314453125 + ], + "critical_prepare_samples_ms": [ + 2.5991599541157484, + 2.248013042844832, + 2.268044976517558, + 2.2184940753504634, + 2.167516970075667, + 2.2399878362193704, + 2.188600949011743, + 2.236208994872868 + ], + "baseline_prepare_ms": 2.235381514765322 + }, + "zigzag": { + "step_ms": 637.3577880859375, + "prepare_ms": 2.4432449718005955, + "one_use_ms": 639.8010330577381, + "prepare_min_ms": 2.4204020155593753, + "prepare_max_ms": 5.492284893989563, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011418886057533793, + "dq": 0.00036775419972180114, + "dkv": 0.0004790727674961075, + "d_index_q": 0.0004997064155363637, + "d_weights": 0.0004932188618751324, + "d_pool_kv": 0.0007411197792606418, + "d_pool_score": 0.0007282443063629072, + "d_ape": 1.238206578062022e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 637.708740234375, + 642.3746948242188, + 637.3577880859375, + 637.919921875, + 635.5283813476562, + 637.1450805664062, + 633.81103515625 + ], + "critical_prepare_samples_ms": [ + 5.492284893989563, + 2.5590929435566068, + 2.472839900292456, + 2.4355159839615226, + 2.4204020155593753, + 2.4509739596396685, + 2.4335948983207345, + 2.4213059805333614 + ], + "baseline_prepare_ms": 2.2861745092086494 + }, + "planner": { + "step_ms": 637.7809448242188, + "prepare_ms": 9.511192969512194, + "one_use_ms": 647.292137793731, + "prepare_min_ms": 9.069906896911561, + "prepare_max_ms": 10.374767007306218, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.001161625101370349, + "dq": 0.0003849271614974025, + "dkv": 0.0005282062594106454, + "d_index_q": 0.0005244622397098521, + "d_weights": 0.0005299670263664599, + "d_pool_kv": 0.0007377781316978346, + "d_pool_score": 0.0007313667519857782, + "d_ape": 1.2469585835438161e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 638.3487548828125, + 637.7809448242188, + 636.5733642578125, + 637.4932250976562, + 637.3499755859375, + 639.65185546875, + 640.6060791015625 + ], + "critical_prepare_samples_ms": [ + 10.374767007306218, + 9.756346000358462, + 9.435725980438292, + 9.219342959113419, + 9.78031090926379, + 9.120933944359422, + 9.586659958586097, + 9.069906896911561 + ], + "baseline_prepare_ms": 9.320752928033471 + } + } + }, + "1024k-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "477bd6d1a7cf100b540a3a743b44248720e6f9ab40bee8715349b4361dad4122", + "policies": { + "contiguous": { + "step_ms": 239.2353973388672, + "prepare_ms": 1.450755458790809, + "one_use_ms": 240.686152797658, + "prepare_min_ms": 1.4129939954727888, + "prepare_max_ms": 1.5769170131534338, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015949253947342458, + "dq": 0.0005970787439556988, + "dkv": 0.0004837391776387549, + "d_index_q": 0.00028350067830152883, + "d_weights": 0.00020850715397855953, + "d_pool_kv": 0.0002889538103473619, + "d_pool_score": 0.0002844647514544151, + "d_ape": 5.9795103431945e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 239.2353973388672, + 238.79302978515625, + 238.77218627929688, + 239.280029296875, + 238.88275146484375, + 239.3966064453125, + 239.5908203125 + ], + "critical_prepare_samples_ms": [ + 1.5769170131534338, + 1.4974380610510707, + 1.4129939954727888, + 1.4513909118250012, + 1.456466969102621, + 1.4202899765223265, + 1.4501200057566166, + 1.43467599991709 + ], + "baseline_prepare_ms": 1.490010938141495 + }, + "zigzag": { + "step_ms": 244.40403747558594, + "prepare_ms": 4.328943963628262, + "one_use_ms": 248.7329814392142, + "prepare_min_ms": 4.188165999948978, + "prepare_max_ms": 6.3333159778267145, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016162390583326469, + "dq": 0.0006094600566733209, + "dkv": 0.0005030974253165897, + "d_index_q": 0.0002799287027435939, + "d_weights": 0.00021150429278782401, + "d_pool_kv": 0.0002866744762082726, + "d_pool_score": 0.000295534879448145, + "d_ape": 6.645333132807697e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 244.40403747558594, + 243.9402313232422, + 244.39878845214844, + 244.6631622314453, + 245.0597686767578, + 245.14906311035156, + 244.3597412109375 + ], + "critical_prepare_samples_ms": [ + 6.3333159778267145, + 4.481954965740442, + 4.322783905081451, + 4.29266900755465, + 4.335104022175074, + 4.287454066798091, + 5.151383113116026, + 4.188165999948978 + ], + "baseline_prepare_ms": 4.6627100091427565 + }, + "planner": { + "step_ms": 239.4423370361328, + "prepare_ms": 1.6502354992553592, + "one_use_ms": 241.09257253538817, + "prepare_min_ms": 1.596967107616365, + "prepare_max_ms": 1.7511360347270966, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015923902784638652, + "dq": 0.0005966125090240904, + "dkv": 0.00048673665217852476, + "d_index_q": 0.0002642586924764183, + "d_weights": 0.0002012227499861691, + "d_pool_kv": 0.00029146264458877296, + "d_pool_score": 0.000316266196867594, + "d_ape": 6.500418666259421e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 239.1868438720703, + 238.84658813476562, + 239.47996520996094, + 239.3558349609375, + 239.5425567626953, + 240.12112426757812, + 239.4423370361328 + ], + "critical_prepare_samples_ms": [ + 1.7511360347270966, + 1.6750609502196312, + 1.6717149410396814, + 1.6503350343555212, + 1.6501359641551971, + 1.6250868793576956, + 1.6428639646619558, + 1.596967107616365 + ], + "baseline_prepare_ms": 1.7240439774468541 + } + } + }, + "1024k-near-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "b51c859098119635daa6f893eb71b0c37c1fef528ca4601e030756c553e3543b", + "policies": { + "contiguous": { + "step_ms": 239.17958068847656, + "prepare_ms": 1.5109099331311882, + "one_use_ms": 240.69049062160775, + "prepare_min_ms": 1.4560830313712358, + "prepare_max_ms": 1.6104099340736866, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0016016710758490706, + "dq": 0.0005993639977494133, + "dkv": 0.0005071280037082532, + "d_index_q": 0.00024417279990879927, + "d_weights": 0.00019413726329423075, + "d_pool_kv": 0.00028954282003200476, + "d_pool_score": 0.0002955359390565756, + "d_ape": 6.6829402506119585e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 239.7038116455078, + 239.43247985839844, + 238.82850646972656, + 239.17958068847656, + 239.11888122558594, + 239.16099548339844, + 239.1847686767578 + ], + "critical_prepare_samples_ms": [ + 1.6104099340736866, + 1.5317690558731556, + 1.4775969320908189, + 1.5244908863678575, + 1.4560830313712358, + 1.5005789464339614, + 1.4756300952285528, + 1.521240919828415 + ], + "baseline_prepare_ms": 1.514466479420662 + }, + "zigzag": { + "step_ms": 244.55760192871094, + "prepare_ms": 4.382276034448296, + "one_use_ms": 248.93987796315923, + "prepare_min_ms": 4.281172063201666, + "prepare_max_ms": 6.378103978931904, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016200773797571998, + "dq": 0.0006071238591595291, + "dkv": 0.0005038878658893274, + "d_index_q": 0.000277629521087908, + "d_weights": 0.00021663570090953182, + "d_pool_kv": 0.0002824822719976606, + "d_pool_score": 0.00027970202182346534, + "d_ape": 5.951399279862816e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 244.45948791503906, + 244.55760192871094, + 245.24301147460938, + 244.60943603515625, + 244.06588745117188, + 244.22259521484375, + 244.9438018798828 + ], + "critical_prepare_samples_ms": [ + 6.378103978931904, + 4.513074993155897, + 4.392945091240108, + 4.371606977656484, + 5.423646070994437, + 4.291923018172383, + 4.289834876544774, + 4.281172063201666 + ], + "baseline_prepare_ms": 4.788588033989072 + }, + "planner": { + "step_ms": 239.22537231445312, + "prepare_ms": 1.7141494899988174, + "one_use_ms": 240.93952180445194, + "prepare_min_ms": 1.6789290821179748, + "prepare_max_ms": 1.81203696411103, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015931991971287927, + "dq": 0.000594438919428117, + "dkv": 0.00048420164599018655, + "d_index_q": 0.0002598202280658747, + "d_weights": 0.00019847531748186766, + "d_pool_kv": 0.0002823099923752181, + "d_pool_score": 0.0002799009540318669, + "d_ape": 6.337974334637259e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 238.89100646972656, + 239.2686767578125, + 238.71510314941406, + 239.22537231445312, + 238.6892547607422, + 239.8118133544922, + 239.24957275390625 + ], + "critical_prepare_samples_ms": [ + 1.81203696411103, + 1.7418409697711468, + 1.7184868920594454, + 1.7098120879381895, + 1.7221469897776842, + 1.6911160200834274, + 1.7061970429494977, + 1.6789290821179748 + ], + "baseline_prepare_ms": 1.75143702654168 + } + } + }, + "1024k-mixed-22": { + "documents": 22, + "tokens": 1048576, + "source_record_sha256": "9a79bc56f979185b724f8b5b55bd4292ce17d924f7ad524db1046fb348026159", + "policies": { + "contiguous": { + "step_ms": 578.4398193359375, + "prepare_ms": 2.1686649997718632, + "one_use_ms": 580.6084843357094, + "prepare_min_ms": 2.13688297662884, + "prepare_max_ms": 2.4644359946250916, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011041011872111484, + "dq": 0.00035084277748766797, + "dkv": 0.00048077283639952237, + "d_index_q": 0.0004962393008161014, + "d_weights": 0.0004618130449118246, + "d_pool_kv": 0.0006457713057874247, + "d_pool_score": 0.0006349061560406983, + "d_ape": 1.0350294386060346e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 580.9935913085938, + 577.6489868164062, + 578.0813598632812, + 580.2940673828125, + 578.3426513671875, + 578.4398193359375, + 579.4547729492188 + ], + "critical_prepare_samples_ms": [ + 2.4644359946250916, + 2.202583011239767, + 2.1521480521187186, + 2.176910056732595, + 2.152706030756235, + 2.13688297662884, + 2.1651480346918106, + 2.172181964851916 + ], + "baseline_prepare_ms": 2.234563056845218 + }, + "zigzag": { + "step_ms": 492.48590087890625, + "prepare_ms": 3.1932704732753336, + "one_use_ms": 495.6791713521816, + "prepare_min_ms": 3.0714080203324556, + "prepare_max_ms": 4.142447025515139, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011285694800997107, + "dq": 0.0003719019185829731, + "dkv": 0.0005662235654619531, + "d_index_q": 0.0005086592423664251, + "d_weights": 0.0004577104856603165, + "d_pool_kv": 0.0007116715193057737, + "d_pool_score": 0.0007034021456370472, + "d_ape": 1.0718926168143519e-05 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 492.2532653808594, + 493.8275146484375, + 492.48590087890625, + 494.0948181152344, + 491.0968933105469, + 491.9676513671875, + 496.3102722167969 + ], + "critical_prepare_samples_ms": [ + 4.142447025515139, + 3.217385965399444, + 3.1294310465455055, + 3.0714080203324556, + 3.084978088736534, + 3.169154981151223, + 3.448216011747718, + 3.476505051366985 + ], + "baseline_prepare_ms": 3.2312944531440735 + }, + "planner": { + "step_ms": 490.8729248046875, + "prepare_ms": 10.11992251733318, + "one_use_ms": 500.9928473220207, + "prepare_min_ms": 9.30530100595206, + "prepare_max_ms": 12.625832110643387, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0011161842289795045, + "dq": 0.0003611815947831294, + "dkv": 0.0005663782910108529, + "d_index_q": 0.0005064407713656789, + "d_weights": 0.0004754969744536212, + "d_pool_kv": 0.0007073881102403647, + "d_pool_score": 0.0006989918887090544, + "d_ape": 1.049051513112601e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 490.52728271484375, + 488.947998046875, + 494.4607849121094, + 491.1177062988281, + 493.6095886230469, + 490.8729248046875, + 490.8112487792969 + ], + "critical_prepare_samples_ms": [ + 10.942328022792935, + 9.30530100595206, + 10.56371396407485, + 10.191290057264268, + 9.348670137114823, + 12.625832110643387, + 10.048554977402091, + 9.677355061285198 + ], + "baseline_prepare_ms": 9.799090563319623 + } + } + }, + "1024k-uniform-1024": { + "documents": 1024, + "tokens": 1048576, + "source_record_sha256": "b444906588101601ed60331d4bddb587f91cd677f6cfb7b815248b53a1451470", + "policies": { + "contiguous": { + "step_ms": 106.93244934082031, + "prepare_ms": 7.248607522342354, + "one_use_ms": 114.18105686316267, + "prepare_min_ms": 6.926600937731564, + "prepare_max_ms": 7.728449068963528, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3262557940735125e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.029982612822922e-05, + "d_pool_score": 5.971250741145832e-05, + "d_ape": 8.469250666388928e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.99427032470703, + 106.90831756591797, + 107.24447631835938, + 106.77391815185547, + 107.3912353515625, + 106.73372650146484, + 106.93244934082031 + ], + "critical_prepare_samples_ms": [ + 7.304837927222252, + 7.447938900440931, + 7.5955630745738745, + 7.06713495310396, + 7.192377117462456, + 7.016728050075471, + 6.926600937731564, + 7.728449068963528 + ], + "baseline_prepare_ms": 7.491494528949261 + }, + "zigzag": { + "step_ms": 120.94866943359375, + "prepare_ms": 47.448766068555415, + "one_use_ms": 168.39743550214916, + "prepare_min_ms": 44.58731797058135, + "prepare_max_ms": 171.35898303240538, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.375492679918749e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.900628434802078e-05, + "d_pool_score": 5.1757344668054784e-05, + "d_ape": 7.775777513380578e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 120.98294067382812, + 120.94866943359375, + 121.13782501220703, + 120.91907501220703, + 121.1909408569336, + 120.86470031738281, + 120.89344024658203 + ], + "critical_prepare_samples_ms": [ + 50.39555404800922, + 48.49247704260051, + 48.66090486757457, + 171.35898303240538, + 44.880184112116694, + 45.30161793809384, + 44.58731797058135, + 46.40505509451032 + ], + "baseline_prepare_ms": 48.3965675230138 + }, + "planner": { + "step_ms": 106.88835144042969, + "prepare_ms": 9.285612439271063, + "one_use_ms": 116.17396387970075, + "prepare_min_ms": 9.05843498185277, + "prepare_max_ms": 10.61727199703455, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.44561194645775e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.3430278698572834e-05, + "d_pool_score": 5.700039307126877e-05, + "d_ape": 8.3915886081418e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.72726440429688, + 106.88835144042969, + 106.67664337158203, + 107.06781005859375, + 106.97030639648438, + 106.76934051513672, + 106.93551635742188 + ], + "critical_prepare_samples_ms": [ + 9.35503700748086, + 9.251538896933198, + 10.61727199703455, + 9.05843498185277, + 9.093354921787977, + 9.319685981608927, + 9.11523203831166, + 10.08200692012906 + ], + "baseline_prepare_ms": 9.633392037358135 + } + } + }, + "1024k-near-uniform-512": { + "documents": 512, + "tokens": 1048576, + "source_record_sha256": "5ff06f93b96c53292e5c777c40e19bc4f75b59888b574f4084c6304e27d06705", + "policies": { + "contiguous": { + "step_ms": 153.3333740234375, + "prepare_ms": 4.90699551301077, + "one_use_ms": 158.24036953644827, + "prepare_min_ms": 4.817495006136596, + "prepare_max_ms": 5.127965938299894, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.6945818513152077e-05, + "dq": 8.852569918672518e-06, + "dkv": 4.778136519064779e-05, + "d_index_q": 2.61152275055734e-06, + "d_weights": 2.609757780546301e-06, + "d_pool_kv": 6.295818459824281e-05, + "d_pool_score": 6.4597007352727e-05, + "d_ape": 8.241673159031186e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.1051788330078, + 153.3333740234375, + 153.77328491210938, + 153.28533935546875, + 153.67117309570312, + 153.2650909423828, + 153.51904296875 + ], + "critical_prepare_samples_ms": [ + 4.958503064699471, + 4.855487961322069, + 5.127965938299894, + 4.836354055441916, + 4.9949249951168895, + 4.826268996112049, + 4.817495006136596, + 5.11400296818465 + ], + "baseline_prepare_ms": 4.550960555206984 + }, + "zigzag": { + "step_ms": 162.93936157226562, + "prepare_ms": 24.463195994030684, + "one_use_ms": 187.4025575662963, + "prepare_min_ms": 24.322091951034963, + "prepare_max_ms": 27.672135969623923, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 1.6907351105767428e-05, + "dq": 8.810149949287884e-06, + "dkv": 4.1814702326799884e-05, + "d_index_q": 2.701785992294692e-06, + "d_weights": 2.6607871764855918e-06, + "d_pool_kv": 5.689533114065568e-05, + "d_pool_score": 5.6776242766562597e-05, + "d_ape": 7.905820789837788e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 162.92745971679688, + 163.33734130859375, + 162.88768005371094, + 162.93936157226562, + 162.84515380859375, + 163.23513793945312, + 162.99880981445312 + ], + "critical_prepare_samples_ms": [ + 27.672135969623923, + 25.097765144892037, + 24.366947007365525, + 24.44107295013964, + 24.474958889186382, + 24.664230993948877, + 24.322091951034963, + 24.451433098874986 + ], + "baseline_prepare_ms": 25.861825968604535 + }, + "planner": { + "step_ms": 153.63226318359375, + "prepare_ms": 6.0009079170413315, + "one_use_ms": 159.63317110063508, + "prepare_min_ms": 5.884354002773762, + "prepare_max_ms": 6.281962036155164, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.6380870010580877e-05, + "dq": 8.576173759861243e-06, + "dkv": 4.329798528645843e-05, + "d_index_q": 2.859165424391761e-06, + "d_weights": 2.390300293499087e-06, + "d_pool_kv": 6.976847445393797e-05, + "d_pool_score": 8.047768811080957e-05, + "d_ape": 8.925679845335159e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.46054077148438, + 154.20037841796875, + 153.07359313964844, + 153.19833374023438, + 153.63571166992188, + 154.10687255859375, + 153.63226318359375 + ], + "critical_prepare_samples_ms": [ + 6.0230528470128775, + 5.923283053562045, + 6.161181954666972, + 5.884354002773762, + 6.277190987020731, + 5.908905062824488, + 5.978762987069786, + 6.281962036155164 + ], + "baseline_prepare_ms": 5.739656975492835 + } + } + }, + "1024k-mixed-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "4c0764d76c3bfca709bc5cdc64a79988a30a203391319ceab498e84528d43186", + "policies": { + "contiguous": { + "step_ms": 567.9577026367188, + "prepare_ms": 9.956162073649466, + "one_use_ms": 577.9138647103682, + "prepare_min_ms": 9.866954875178635, + "prepare_max_ms": 11.781121022067964, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008054851232509614, + "dq": 0.00030349153325609713, + "dkv": 0.0003996891751172817, + "d_index_q": 0.0006246139925808879, + "d_weights": 0.0005733311451828375, + "d_pool_kv": 0.0006205091711447106, + "d_pool_score": 0.0006195543072615393, + "d_ape": 8.589075672373344e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 565.80810546875, + 568.4535522460938, + 567.845947265625, + 569.2208862304688, + 563.9498901367188, + 567.9577026367188, + 568.2915649414062 + ], + "critical_prepare_samples_ms": [ + 11.781121022067964, + 9.866954875178635, + 9.920966112986207, + 10.023329989053309, + 9.899892029352486, + 9.991358034312725, + 10.618924046866596, + 9.895945899188519 + ], + "baseline_prepare_ms": 10.622373956721276 + }, + "zigzag": { + "step_ms": 550.558837890625, + "prepare_ms": 48.51576604414731, + "one_use_ms": 599.0746039347723, + "prepare_min_ms": 46.67189810425043, + "prepare_max_ms": 174.14286790881306, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0008195260459777163, + "dq": 0.000301416184948774, + "dkv": 0.0005447644196372983, + "d_index_q": 0.0006037227037525915, + "d_weights": 0.0005483621762803569, + "d_pool_kv": 0.0007216194622467821, + "d_pool_score": 0.0007074085651829549, + "d_ape": 8.845595009769592e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 547.7075805664062, + 551.5230712890625, + 550.558837890625, + 550.6272583007812, + 551.8482666015625, + 549.5032348632812, + 549.90185546875 + ], + "critical_prepare_samples_ms": [ + 49.92610192857683, + 47.86653909832239, + 46.67189810425043, + 143.82607594598085, + 174.14286790881306, + 48.84456901345402, + 47.94406390283257, + 48.186963074840605 + ], + "baseline_prepare_ms": 48.262091528158635 + }, + "planner": { + "step_ms": 494.7050476074219, + "prepare_ms": 26.39079309301451, + "one_use_ms": 521.0958407004364, + "prepare_min_ms": 25.676422053948045, + "prepare_max_ms": 27.048783958889544, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0008365326911981586, + "dq": 0.000304561997298154, + "dkv": 0.0005395322403243995, + "d_index_q": 0.0006160848721497884, + "d_weights": 0.0005678273845894476, + "d_pool_kv": 0.000678002818444316, + "d_pool_score": 0.0006561034875025234, + "d_ape": 8.545834009840338e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 495.04931640625, + 495.3780822753906, + 492.96002197265625, + 494.4125671386719, + 495.3255615234375, + 493.7383117675781, + 494.7050476074219 + ], + "critical_prepare_samples_ms": [ + 26.55792306177318, + 27.048783958889544, + 26.075869100168347, + 26.223148917779326, + 26.27995307557285, + 26.50163311045617, + 25.676422053948045, + 26.891826069913805 + ], + "baseline_prepare_ms": 34.91155954543501 + } + } + }, + "1024k-long-tail-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "d3ad6415c7f6691fe0b88f453159fe81ee75cd82494a33142fdec3b521028dc9", + "policies": { + "contiguous": { + "step_ms": 800.95556640625, + "prepare_ms": 12.669915508013219, + "one_use_ms": 813.6254819142632, + "prepare_min_ms": 12.12127401959151, + "prepare_max_ms": 13.159677851945162, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009079882713568034, + "dq": 0.0002924982024565224, + "dkv": 0.0004493330361466962, + "d_index_q": 0.0005156894968942304, + "d_weights": 0.0005167765162935939, + "d_pool_kv": 0.0007712278582484425, + "d_pool_score": 0.0007494124280620555, + "d_ape": 7.171042317242159e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 800.335693359375, + 802.2348022460938, + 799.2125244140625, + 799.6749877929688, + 800.95556640625, + 801.217041015625, + 801.6029663085938 + ], + "critical_prepare_samples_ms": [ + 13.159677851945162, + 12.533110100775957, + 12.759627075865865, + 12.12127401959151, + 12.722588959150016, + 12.617242056876421, + 12.80664000660181, + 12.566132005304098 + ], + "baseline_prepare_ms": 14.17607138864696 + }, + "zigzag": { + "step_ms": 715.7775268554688, + "prepare_ms": 101.23607551213354, + "one_use_ms": 817.0136023676023, + "prepare_min_ms": 50.27641903143376, + "prepare_max_ms": 169.727465021424, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009004337940703815, + "dq": 0.0002908384586939322, + "dkv": 0.0008283107689368773, + "d_index_q": 0.0006025743415942871, + "d_weights": 0.0005952597316348166, + "d_pool_kv": 0.0009031995573276561, + "d_pool_score": 0.0008609984776183619, + "d_ape": 7.574199067023971e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 712.8131103515625, + 711.6310424804688, + 715.7775268554688, + 712.3639526367188, + 717.9108276367188, + 716.431396484375, + 716.0610961914062 + ], + "critical_prepare_samples_ms": [ + 169.727465021424, + 50.38151808548719, + 150.52157104946673, + 159.4792689429596, + 163.56680891476572, + 51.95057997480035, + 50.27641903143376, + 50.714182085357606 + ], + "baseline_prepare_ms": 151.8010766012594 + }, + "planner": { + "step_ms": 591.8178100585938, + "prepare_ms": 29.16059346171096, + "one_use_ms": 620.9784035203047, + "prepare_min_ms": 28.44624104909599, + "prepare_max_ms": 150.10115189943463, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0009046024533399656, + "dq": 0.00028358853532615296, + "dkv": 0.000828257132860518, + "d_index_q": 0.0007049305803991532, + "d_weights": 0.000616040566927685, + "d_pool_kv": 0.0009008018655028041, + "d_pool_score": 0.0008551253342927941, + "d_ape": 7.554278135195296e-06 + }, + "max_loss_abs_diff": 8.731149137020111e-11, + "critical_step_samples_ms": [ + 591.6988525390625, + 589.947509765625, + 591.8178100585938, + 596.8615112304688, + 592.572021484375, + 591.2091674804688, + 592.2348022460938 + ], + "critical_prepare_samples_ms": [ + 28.44624104909599, + 29.306308017112315, + 29.014878906309605, + 28.898976044729352, + 29.324857983738184, + 150.10115189943463, + 28.808553935959935, + 124.28496405482292 + ], + "baseline_prepare_ms": 38.67735603125766 + } + } + } + }, + "totals_by_context": { + "128": { + "contiguous": [ + 207.8772144317627, + 235.76852785609663 + ], + "zigzag": [ + 221.9881248474121, + 388.67559937527403 + ], + "planner": [ + 206.02739143371582, + 255.17840154655278 + ] + }, + "256": { + "contiguous": [ + 530.4014129638672, + 563.2268744944595 + ], + "zigzag": [ + 555.0891208648682, + 878.2458056765608 + ], + "planner": [ + 507.35546112060547, + 587.5882775196806 + ] + }, + "1024": { + "contiguous": [ + 3538.296340942383, + 3580.4464503666386 + ], + "zigzag": [ + 3169.0297241210938, + 3405.041263183113 + ], + "planner": [ + 2954.3650512695312, + 3048.1984586566687 + ] + } + }, + "max_relative_rms": { + "output": 0.0021340651638746743, + "dq": 0.0006697244454984808, + "dkv": 0.0010306683280579473, + "d_index_q": 0.0009173576570437036, + "d_weights": 0.0006640931260018509, + "d_pool_kv": 0.0013371215867479295, + "d_pool_score": 0.00138344136125153, + "d_ape": 2.5403258387015062e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-09, + "worst_steady_gap": [ + 0.3393008303370637, + "128k-mixed-22" + ], + "worst_one_use_gap": [ + 19.39923530952372, + "128k-mixed-22" + ], + "host_ab": { + "baseline_commit": "ea71df3b3bfad69141d1ee3ddd884ac9ccdb27ce", + "scope": "Host-only CsaCPSelectionPlan constructor on rank 7 of CP8; no GPU/workspace preparation", + "repeats": 20, + "cases": { + "128k-single-planner": { + "samples_ms": { + "before": [ + 6.241483031772077, + 6.129086017608643, + 6.2108299462124705, + 6.163439014926553, + 6.159020005725324, + 6.204652949236333, + 6.147739011794329, + 6.155765964649618, + 6.279730005189776, + 6.224854965694249, + 6.132257985882461, + 6.104929954744875, + 6.146484054625034, + 6.200143019668758, + 6.191351916640997, + 6.217735004611313, + 6.158133968710899, + 6.157501949928701, + 6.267914897762239, + 6.197582930326462 + ], + "after": [ + 6.05339405592531, + 6.09124603215605, + 6.0327439568936825, + 6.391837960109115, + 6.117061013355851, + 6.04431692045182, + 6.072498974390328, + 6.031388998962939, + 6.078919977881014, + 6.038562045432627, + 6.127536064013839, + 6.046790978871286, + 6.069870083592832, + 6.025816081091762, + 6.028156029060483, + 6.082112086005509, + 6.01962604559958, + 6.050291936844587, + 6.105632055550814, + 5.963249015621841 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.177395465783775, + "after": 6.0518429963849485 + }, + "metadata_exact": true + }, + "128k-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.40457199793308973, + 0.3915169509127736, + 0.3725989954546094, + 0.38956699427217245, + 0.37341401912271976, + 0.3859599819406867, + 0.3727299626916647, + 0.3891169326379895, + 0.3697650972753763, + 0.3940380411222577, + 0.37929299287498, + 0.390987959690392, + 0.37147896364331245, + 0.3826599568128586, + 0.37088198587298393, + 0.3915060078725219, + 0.372899929061532, + 0.3812409704551101, + 0.368123990483582, + 0.3834000090137124 + ], + "after": [ + 0.36852096673101187, + 0.3598059993237257, + 0.367147964425385, + 0.34407293424010277, + 0.3628099802881479, + 0.3502810141071677, + 0.3541019978001714, + 0.3426559269428253, + 0.358566059730947, + 0.3564689541235566, + 0.3626210382208228, + 0.34407898783683777, + 0.3595729358494282, + 0.35286403726786375, + 0.36064302548766136, + 0.343213090673089, + 0.35762006882578135, + 0.34747098106890917, + 0.3642140654847026, + 0.3391670761629939 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.3819504636339843, + "after": 0.357044511474669 + }, + "metadata_exact": true + }, + "128k-near-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.4026839742437005, + 0.39121892768889666, + 0.376117997802794, + 0.38380094338208437, + 0.3803960280492902, + 0.3904419718310237, + 0.37196208722889423, + 0.3914200933650136, + 0.3753060009330511, + 0.38710690569132566, + 0.3820260753855109, + 0.3933230182155967, + 0.3741809632629156, + 0.3947960212826729, + 0.3754019271582365, + 0.39576191920787096, + 0.3724439302459359, + 0.38169207982718945, + 0.37471100222319365, + 0.39148691575974226 + ], + "after": [ + 0.36272103898227215, + 0.3440839936956763, + 0.35585102159529924, + 0.3516109427437186, + 0.3656300250440836, + 0.34017907455563545, + 0.3507171059027314, + 0.35162700805813074, + 0.35890506114810705, + 0.3831470385193825, + 0.35917398054152727, + 0.34544302616268396, + 0.35676301922649145, + 0.3443220630288124, + 0.35430595744401217, + 0.34445198252797127, + 0.35806000232696533, + 0.34743198193609715, + 0.3568349638953805, + 0.33541000448167324 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.38291350938379765, + "after": 0.35296648275107145 + }, + "metadata_exact": true + }, + "128k-mixed-22-planner": { + "samples_ms": { + "before": [ + 6.297884043306112, + 6.32572709582746, + 6.304105976596475, + 6.388368899933994, + 6.2440200708806515, + 6.35086209513247, + 6.266923970542848, + 6.328370072878897, + 6.334865931421518, + 6.350334966555238, + 6.248642923310399, + 6.344994064420462, + 6.300061009824276, + 6.30893197376281, + 6.245687021873891, + 6.297999992966652, + 6.334190955385566, + 6.349519011564553, + 6.3354120356962085, + 6.280714995227754 + ], + "after": [ + 5.9602969558909535, + 5.947844008915126, + 5.957170040346682, + 5.975153995677829, + 5.950245074927807, + 5.9789830120280385, + 5.968288984149694, + 5.968034965917468, + 5.970570957288146, + 5.9893300058320165, + 5.974469007924199, + 5.955578992143273, + 5.940887960605323, + 6.066182977519929, + 5.963379982858896, + 5.992084043100476, + 5.993772996589541, + 6.073997006751597, + 5.978425033390522, + 6.0032380279153585 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.317329534795135, + "after": 5.972519982606173 + }, + "metadata_exact": true + }, + "128k-uniform-1024-planner": { + "samples_ms": { + "before": [ + 4.660016042180359, + 4.370620008558035, + 4.610156989656389, + 4.405395011417568, + 4.589467076584697, + 4.427237086929381, + 4.581127082929015, + 4.438054980710149, + 4.516609013080597, + 4.428007057867944, + 4.5784530229866505, + 4.393365001305938, + 4.563861060887575, + 4.433515015989542, + 4.587459960021079, + 4.470839980058372, + 4.528368008323014, + 4.389258101582527, + 4.762601922266185, + 4.427127074450254 + ], + "after": [ + 4.281781031750143, + 4.191130050458014, + 4.174580913968384, + 4.1822659550234675, + 4.152310895733535, + 4.233917919918895, + 4.1525529231876135, + 4.132509930059314, + 4.176480928435922, + 4.160788026638329, + 4.145585931837559, + 4.104895982891321, + 4.152894951403141, + 4.110871930606663, + 4.168654908426106, + 4.12381102796644, + 4.132851026952267, + 4.139319993555546, + 34.61243200581521, + 4.214663058519363 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 4.4937244965694845, + "after": 4.156841489020735 + }, + "metadata_exact": true + }, + "128k-near-uniform-512-planner": { + "samples_ms": { + "before": [ + 2.415246097370982, + 2.372286980971694, + 2.414268092252314, + 2.392580034211278, + 2.3668940411880612, + 2.36852397210896, + 2.380969002842903, + 2.3735250579193234, + 2.3787349928170443, + 2.379633951932192, + 2.394836046732962, + 2.3807089310139418, + 2.3655840195715427, + 2.373978029936552, + 2.379238954745233, + 2.3784040240570903, + 2.3559159599244595, + 2.3839951027184725, + 2.3599970154464245, + 2.3900530068203807 + ], + "after": [ + 2.221758011728525, + 2.173527958802879, + 2.2238719975575805, + 2.2164909169077873, + 2.238516928628087, + 2.2274290677160025, + 2.2253500064834952, + 2.1910270443186164, + 2.206207951530814, + 2.18932400457561, + 2.214886015281081, + 2.2102269576862454, + 2.2298539988696575, + 2.193198073655367, + 2.2093040170148015, + 2.2074709413573146, + 2.222349983640015, + 2.22071202006191, + 2.254899009130895, + 2.188967075198889 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.3789869737811387, + "after": 2.2156884660944343 + }, + "metadata_exact": true + }, + "128k-mixed-1000-planner": { + "samples_ms": { + "before": [ + 22.04404992517084, + 21.91894897259772, + 22.10096293129027, + 21.933616953901947, + 21.74217300489545, + 22.01524598058313, + 21.75239403732121, + 21.638283040374517, + 22.058080998249352, + 21.76110597793013, + 52.93793696910143, + 22.040894022211432, + 21.75105398055166, + 21.90667192917317, + 21.845451090484858, + 21.720419987104833, + 21.844093920663, + 21.968669956550002, + 21.688856999389827, + 21.834119921550155 + ], + "after": [ + 3.842544974759221, + 3.952922997996211, + 3.8634379161521792, + 3.8387340027838945, + 3.8389438996091485, + 3.857493051327765, + 3.85215796995908, + 3.9768930291756988, + 3.8245299365371466, + 3.8569149328395724, + 3.916134941391647, + 3.8611970376223326, + 3.885005018673837, + 3.850206034258008, + 3.961231908760965, + 3.874346031807363, + 3.897645976394415, + 3.849889966659248, + 3.866290906444192, + 3.8647529436275363 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 21.876061509829015, + "after": 3.862317476887256 + }, + "metadata_exact": true + }, + "128k-long-tail-1000-planner": { + "samples_ms": { + "before": [ + 6.098993006162345, + 6.06885296292603, + 6.041291984729469, + 6.102801999077201, + 6.055154954083264, + 6.118261022493243, + 6.063149077817798, + 6.077650003135204, + 6.003413000144064, + 6.109009962528944, + 6.0182519955560565, + 6.069358089007437, + 6.046032998710871, + 6.179652060382068, + 6.067587994039059, + 36.73038596753031, + 6.08207902405411, + 6.207393016666174, + 6.0486659640446305, + 6.28370197955519 + ], + "after": [ + 3.680306952446699, + 3.667220938950777, + 3.6552910460159183, + 3.65017494186759, + 3.651484032161534, + 3.630999941378832, + 3.6778110079467297, + 3.6716859322041273, + 3.616343019530177, + 3.675099927932024, + 3.6119039868935943, + 3.66475700866431, + 3.6578980507329106, + 3.6507350159808993, + 3.6120960721746087, + 3.7088380195200443, + 3.6899440456181765, + 3.7302899872884154, + 3.651808016002178, + 3.6856549559161067 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.073504046071321, + "after": 3.6613275296986103 + }, + "metadata_exact": true + }, + "256k-single-planner": { + "samples_ms": { + "before": [ + 6.273308070376515, + 6.1859110137447715, + 6.228829966858029, + 6.189352949149907, + 6.210490013472736, + 6.1686799162998796, + 6.207579048350453, + 6.210181047208607, + 6.191586027853191, + 6.232277024537325, + 6.19978504255414, + 6.172747001983225, + 6.196624017320573, + 6.1631620628759265, + 6.173041998408735, + 6.198281073011458, + 6.189342006109655, + 6.118671968579292, + 6.14934996701777, + 6.1761620454490185 + ], + "after": [ + 6.109548965469003, + 6.110334070399404, + 6.099766003899276, + 6.0219180304557085, + 6.071530049666762, + 6.033507990650833, + 6.164598977193236, + 6.1124550411477685, + 6.096701021306217, + 6.0586490435525775, + 6.0906229773536325, + 6.123605999164283, + 6.119196070358157, + 6.050423951819539, + 6.057007005438209, + 6.0730380937457085, + 6.070937961339951, + 6.0674239648506045, + 6.090157083235681, + 6.056089070625603 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.190469488501549, + "after": 6.081597588490695 + }, + "metadata_exact": true + }, + "256k-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.41015096940100193, + 0.3944459604099393, + 0.37739600520581007, + 0.3896639682352543, + 0.38664997555315495, + 0.4359870217740536, + 0.3803169820457697, + 0.3860710421577096, + 0.38487394340336323, + 0.3898339346051216, + 0.3733679186552763, + 0.38506602868437767, + 0.37351599894464016, + 0.38967595901340246, + 0.3705939743667841, + 0.3850290086120367, + 0.37040491588413715, + 0.38908596616238356, + 0.3693180624395609, + 0.39668893441557884 + ], + "after": [ + 0.3678260836750269, + 0.36171393003314734, + 0.3648300189524889, + 0.3432539524510503, + 0.35647302865982056, + 0.34370110370218754, + 0.35738199949264526, + 0.3357569221407175, + 0.3518399316817522, + 0.34252204932272434, + 0.35713601391762495, + 0.34019898157566786, + 0.35376299638301134, + 0.34015101846307516, + 0.35704299807548523, + 0.33987394999712706, + 0.3501370083540678, + 0.33766403794288635, + 0.35145890433341265, + 0.3514030249789357 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.38556853542104363, + "after": 0.3514309646561742 + }, + "metadata_exact": true + }, + "256k-near-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.4030209966003895, + 0.39380101952701807, + 0.37597701884806156, + 0.388526008464396, + 0.37151994183659554, + 0.379147008061409, + 0.37576595786958933, + 0.4110520239919424, + 0.37336302921175957, + 0.3853539237752557, + 0.3833520459011197, + 0.38638501428067684, + 0.37212600000202656, + 0.38512097671628, + 0.37283089477568865, + 0.395660987123847, + 0.3773580538108945, + 0.38487801793962717, + 0.3688790602609515, + 0.38768898230046034 + ], + "after": [ + 0.36612001713365316, + 0.3428839845582843, + 0.34924305509775877, + 0.33483700826764107, + 0.3550719702616334, + 0.3430369542911649, + 0.3512970870360732, + 0.3375509986653924, + 0.359527999535203, + 0.34393894020467997, + 0.35246205516159534, + 0.3352659987285733, + 0.35411003045737743, + 0.34323392901569605, + 0.36014197394251823, + 0.3383359871804714, + 0.3529870882630348, + 0.342561979778111, + 0.3608709666877985, + 0.3360230475664139 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.38411503192037344, + "after": 0.34659099765121937 + }, + "metadata_exact": true + }, + "256k-mixed-22-planner": { + "samples_ms": { + "before": [ + 6.795704015530646, + 6.791763938963413, + 6.846706965006888, + 6.778295035474002, + 6.843929062597454, + 6.815892062149942, + 6.78327598143369, + 6.800882983952761, + 6.778726004995406, + 6.860786932520568, + 6.743785925209522, + 6.825271062552929, + 6.850271951407194, + 6.783836986869574, + 6.734416005201638, + 6.789679056964815, + 6.794990971684456, + 6.805825047194958, + 6.787401041947305, + 6.9325019139796495 + ], + "after": [ + 6.480885902419686, + 6.506833015009761, + 6.519202026538551, + 6.67197594884783, + 6.540074944496155, + 6.439254037104547, + 6.543172057718039, + 6.608141935430467, + 6.494894973002374, + 6.468622013926506, + 6.479641073383391, + 6.490246974863112, + 6.506317062303424, + 6.468169973231852, + 6.521328934468329, + 6.469527957960963, + 6.47113798186183, + 6.5275689121335745, + 6.535366992466152, + 6.519639049656689 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.795347493607551, + "after": 6.506575038656592 + }, + "metadata_exact": true + }, + "256k-uniform-1024-planner": { + "samples_ms": { + "before": [ + 4.528131103143096, + 4.519441979937255, + 4.45167301222682, + 4.516285029239953, + 4.445967031642795, + 4.486287012696266, + 4.479257971979678, + 4.522491944953799, + 4.453120985999703, + 4.484906909056008, + 4.395911004394293, + 4.493816988542676, + 4.383869003504515, + 4.455235903151333, + 4.413711954839528, + 4.532746970653534, + 4.4780169846490026, + 4.572667996399105, + 4.481656011193991, + 4.5083899749442935 + ], + "after": [ + 4.2732099536806345, + 4.213123000226915, + 4.125150968320668, + 4.145705956034362, + 4.167816019617021, + 4.113686038181186, + 4.167040926404297, + 4.176416085101664, + 4.202626994810998, + 4.1443039663136005, + 4.177298047579825, + 4.142577992752194, + 4.125079023651779, + 4.1348160011693835, + 4.203732940368354, + 4.115461022593081, + 4.169943975284696, + 4.170452011749148, + 4.154626978561282, + 4.169474937953055 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 4.483281460124999, + "after": 4.167428473010659 + }, + "metadata_exact": true + }, + "256k-near-uniform-512-planner": { + "samples_ms": { + "before": [ + 2.4475599639117718, + 2.4344889679923654, + 2.395572024397552, + 2.387490007095039, + 2.3774970322847366, + 2.4405719013884664, + 2.4193800054490566, + 2.4099330184981227, + 2.4256249889731407, + 2.408756990917027, + 2.396582975052297, + 2.4163590278476477, + 2.3899219231680036, + 32.957385992631316, + 2.451278967782855, + 2.404685947112739, + 2.3890669690445065, + 2.4140229215845466, + 2.4009819608181715, + 2.457590075209737 + ], + "after": [ + 2.2638730006292462, + 2.2619450464844704, + 2.23950098734349, + 2.2017869632691145, + 2.215599990449846, + 2.204743097536266, + 2.232411061413586, + 2.207626006565988, + 2.206437988206744, + 2.177756978198886, + 2.250102930702269, + 2.2028969833627343, + 2.236993983387947, + 2.2443720372393727, + 2.2390360245481133, + 2.2466800874099135, + 2.2268480388447642, + 2.2540870122611523, + 2.21646495629102, + 2.2201479878276587 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.4119779700413346, + "after": 2.229629550129175 + }, + "metadata_exact": true + }, + "256k-mixed-1000-planner": { + "samples_ms": { + "before": [ + 24.54034995753318, + 24.531275033950806, + 24.561429978348315, + 24.81341699603945, + 54.413284990005195, + 24.231813033111393, + 24.326125043444335, + 24.51071992982179, + 24.568932014517486, + 24.544357089325786, + 24.218196980655193, + 24.278299999423325, + 54.848834057338536, + 24.51823907904327, + 24.61230696644634, + 24.68205103650689, + 24.32792796753347, + 24.375067907385528, + 24.580372031778097, + 24.808765039779246 + ], + "after": [ + 16.764546977356076, + 16.3035630248487, + 16.358520020730793, + 16.450993018224835, + 16.33723301347345, + 16.516595031134784, + 16.50680403690785, + 16.276226029731333, + 16.32769708521664, + 16.374025959521532, + 16.418756917119026, + 16.53418701607734, + 16.604391974397004, + 16.265636077150702, + 16.35824504774064, + 16.457217978313565, + 16.542282071895897, + 16.551429987885058, + 16.530111082829535, + 16.244163969531655 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 24.542353523429483, + "after": 16.43487496767193 + }, + "metadata_exact": true + }, + "256k-long-tail-1000-planner": { + "samples_ms": { + "before": [ + 23.28029298223555, + 22.953859996050596, + 23.113874020054936, + 23.350395960733294, + 22.959975991398096, + 22.866397979669273, + 53.57191897928715, + 23.445561062544584, + 22.97284291125834, + 23.276207968592644, + 23.004574002698064, + 23.091771989129484, + 23.446962935850024, + 23.451111977919936, + 23.194006993435323, + 53.4638250246644, + 23.02024490199983, + 23.022315930575132, + 23.23351800441742, + 23.345256922766566 + ], + "after": [ + 15.0655850302428, + 15.106525970622897, + 15.416737063787878, + 14.999473001807928, + 15.052370028570294, + 15.261941007338464, + 14.996233046986163, + 15.140222036279738, + 15.17251692712307, + 15.06192295346409, + 15.014206990599632, + 15.350549947470427, + 15.018568956293166, + 15.140537987463176, + 15.73672704398632, + 14.988586073741317, + 15.155900036916137, + 15.214379993267357, + 15.043093007989228, + 15.059495926834643 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 23.21376249892637, + "after": 15.086055500432849 + }, + "metadata_exact": true + }, + "1024k-single-planner": { + "samples_ms": { + "before": [ + 6.401306949555874, + 6.363680004142225, + 6.276860018260777, + 6.301490939222276, + 6.289783981628716, + 6.289764074608684, + 6.253407918848097, + 6.344108027406037, + 6.206598016433418, + 6.301910034380853, + 6.258120061829686, + 6.300506996922195, + 6.237262045033276, + 6.291158031672239, + 6.250014994293451, + 6.290553021244705, + 6.247104960493743, + 6.244288058951497, + 6.247733021155, + 6.355733028613031 + ], + "after": [ + 6.215190049260855, + 6.177706993184984, + 6.197663955390453, + 6.145993946120143, + 6.222699070349336, + 6.169933010824025, + 6.188363069668412, + 6.152393063530326, + 6.236356915906072, + 6.181873031891882, + 6.244497955776751, + 6.177121074870229, + 6.194881978444755, + 6.158321048133075, + 6.16318394895643, + 6.140960962511599, + 6.226410041563213, + 6.211370928213, + 6.178151001222432, + 6.177538889460266 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.2897740281187, + "after": 6.180012016557157 + }, + "metadata_exact": true + }, + "1024k-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.42080599814653397, + 0.4038750194013119, + 0.38857804611325264, + 0.4034269368276, + 0.38967293221503496, + 0.40273508056998253, + 0.3832620568573475, + 0.3959540044888854, + 0.3901791060343385, + 0.3977610031142831, + 0.43747795280069113, + 0.4107679706066847, + 0.3902378957718611, + 0.3971179248765111, + 0.38518698420375586, + 0.4006110830232501, + 0.3892940003424883, + 0.3975259605795145, + 0.3838939592242241, + 0.4060509381815791 + ], + "after": [ + 0.36720791831612587, + 0.3625159151852131, + 0.37771405186504126, + 0.35041000228375196, + 0.36165607161819935, + 0.349257024936378, + 0.36535400431603193, + 0.34397293347865343, + 0.36269298288971186, + 0.34938903991132975, + 0.3802860155701637, + 0.357507960870862, + 0.3609630512073636, + 0.34980091731995344, + 0.36741001531481743, + 0.3414710517972708, + 0.3660570364445448, + 0.3538549644872546, + 0.37089898250997066, + 0.35520095843821764 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.3973219427280128, + "after": 0.3613095614127815 + }, + "metadata_exact": true + }, + "1024k-near-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.411357032135129, + 0.4092190647497773, + 0.38928911089897156, + 0.39956194814294577, + 0.39079703856259584, + 0.40034507401287556, + 0.3874669782817364, + 0.40125101804733276, + 0.38783892523497343, + 0.4079029895365238, + 0.38890400901436806, + 0.3955049905925989, + 0.3836510004475713, + 0.39660301990807056, + 0.37577003240585327, + 0.40066102519631386, + 0.38612191565334797, + 0.40559505578130484, + 0.38500898517668247, + 0.3992560086771846 + ], + "after": [ + 0.36906194873154163, + 0.3460940206423402, + 0.359949073754251, + 0.34843606408685446, + 0.36219297908246517, + 0.34608994610607624, + 0.36883691791445017, + 0.34593301825225353, + 0.3651459701359272, + 0.35133492201566696, + 0.36212196573615074, + 0.3420230932533741, + 0.356883043423295, + 0.34345604944974184, + 0.3590090200304985, + 0.34881499595940113, + 0.3593889996409416, + 0.34409610088914633, + 0.3649709979072213, + 0.3488330403342843 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.39605400525033474, + "after": 0.354108982719481 + }, + "metadata_exact": true + }, + "1024k-mixed-22-planner": { + "samples_ms": { + "before": [ + 7.116922060959041, + 7.177904015406966, + 7.13980000000447, + 7.036815048195422, + 7.111265091225505, + 7.035356014966965, + 7.030882057733834, + 7.0656288880854845, + 7.051595952361822, + 7.055683992803097, + 7.0082469610497355, + 7.1635430213063955, + 7.037836010567844, + 7.078544935211539, + 7.08668795414269, + 7.108084042556584, + 7.043128018267453, + 7.059555035084486, + 7.088234066031873, + 7.057390990667045 + ], + "after": [ + 6.80316099897027, + 6.773620028980076, + 6.781495991162956, + 6.73370691947639, + 6.742869038134813, + 6.748192943632603, + 6.805401062592864, + 6.764878984540701, + 6.730035995133221, + 6.813118001446128, + 6.788165075704455, + 6.766583072021604, + 6.699447054415941, + 6.768840947188437, + 6.729939021170139, + 6.789920036680996, + 6.8319570273160934, + 6.730383029207587, + 6.752830930054188, + 6.794777000322938 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 7.062591961584985, + "after": 6.76771200960502 + }, + "metadata_exact": true + }, + "1024k-uniform-1024-planner": { + "samples_ms": { + "before": [ + 4.723243066109717, + 4.931011004373431, + 4.759495961479843, + 4.715371993370354, + 4.765891004353762, + 4.700714023783803, + 4.798600100912154, + 4.737420007586479, + 4.807673045434058, + 4.715082002803683, + 4.753511981107295, + 4.659031983464956, + 4.738165996968746, + 4.7197270905599, + 37.12526906747371, + 4.684593062847853, + 4.794273059815168, + 4.75063594058156, + 4.797957022674382, + 4.689068999141455 + ], + "after": [ + 4.392486996948719, + 4.454082925803959, + 4.302648943848908, + 4.259853973053396, + 4.329560091719031, + 4.278030013665557, + 4.359407001174986, + 4.328979062847793, + 4.3300940888002515, + 4.302379908040166, + 4.365602973848581, + 4.28319105412811, + 4.396082018502057, + 4.303489928133786, + 4.510325030423701, + 4.342301981523633, + 4.266428062692285, + 4.304714035242796, + 4.332002019509673, + 4.264610004611313 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 4.744400968775153, + "after": 4.329269577283412 + }, + "metadata_exact": true + }, + "1024k-near-uniform-512-planner": { + "samples_ms": { + "before": [ + 2.515185042284429, + 2.4836119264364243, + 2.4774610064923763, + 2.4514689575880766, + 2.4803068954497576, + 2.4744559777900577, + 2.441972028464079, + 2.450105967000127, + 2.4273539893329144, + 2.5212119799107313, + 2.449671970680356, + 2.4625950027257204, + 2.465908066369593, + 2.4435248924419284, + 2.432778012007475, + 2.447120030410588, + 2.4312910391017795, + 2.4819980608299375, + 2.413704991340637, + 2.454346977174282 + ], + "after": [ + 2.2778710117563605, + 2.224409021437168, + 2.2687099408358335, + 2.189205028116703, + 2.4770909221842885, + 2.2167189745232463, + 2.2215070202946663, + 2.2258959943428636, + 2.24177201744169, + 2.220332040451467, + 2.2220200626179576, + 2.1935610566288233, + 2.365433028899133, + 2.3080429527908564, + 2.229491015896201, + 2.2006690269336104, + 2.2265500156208873, + 2.22190807107836, + 2.215657033957541, + 2.203262993134558 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.4529079673811793, + "after": 2.223214542027563 + }, + "metadata_exact": true + }, + "1024k-mixed-1000-planner": { + "samples_ms": { + "before": [ + 25.652137934230268, + 25.46787296887487, + 25.711324997246265, + 25.766848935745656, + 25.748226093128324, + 25.68940701894462, + 25.380223989486694, + 25.50192188937217, + 25.78106301371008, + 57.11816402617842, + 25.759859010577202, + 25.784690980799496, + 25.438977987505496, + 25.33386100549251, + 25.694990996271372, + 25.588147924281657, + 25.80513700377196, + 25.829382007941604, + 56.03781796526164, + 25.28274105861783 + ], + "after": [ + 17.50727999024093, + 48.44116396270692, + 17.65985193196684, + 17.201192094944417, + 17.23041303921491, + 17.503901035524905, + 17.601707950234413, + 17.612167983315885, + 17.567483941093087, + 17.297217971645296, + 17.34165393281728, + 17.509303987026215, + 17.61009602341801, + 17.63615501113236, + 17.625470063649118, + 17.18583598267287, + 17.181201023049653, + 17.410952015779912, + 17.32693100348115, + 17.414303962141275 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 25.70315799675882, + "after": 17.505590512882918 + }, + "metadata_exact": true + }, + "1024k-long-tail-1000-planner": { + "samples_ms": { + "before": [ + 25.22357099223882, + 25.02503409050405, + 24.86469899304211, + 24.80024192482233, + 24.994525010697544, + 24.8831610661, + 25.079349987208843, + 25.113655952736735, + 24.65354697778821, + 24.713349994271994, + 24.960741982795298, + 24.926676996983588, + 24.976362008601427, + 24.950529914349318, + 24.81701597571373, + 25.46109596733004, + 24.933145963586867, + 24.92614893708378, + 24.99815507326275, + 25.036363978870213 + ], + "after": [ + 16.639605979435146, + 16.80207799654454, + 47.643300029449165, + 16.94847596809268, + 17.137111048214138, + 16.74931705929339, + 16.65344601497054, + 16.915512969717383, + 17.00916001573205, + 17.092264024540782, + 17.00384693685919, + 46.60715803038329, + 16.701380955055356, + 16.74321002792567, + 16.781526035629213, + 16.892694984562695, + 17.648648004978895, + 16.605001059360802, + 16.71510294545442, + 48.10157907195389 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 24.955635948572308, + "after": 16.90410397714004 + }, + "metadata_exact": true + }, + "128k-uniform-1024-zigzag": { + "samples_ms": { + "before": [ + 69.64113505091518, + 73.64157098345459, + 38.45433599781245, + 72.69224699120969, + 38.10993500519544, + 38.43369905371219, + 38.421318982727826, + 38.43365400098264, + 38.649844005703926, + 38.74922904651612, + 38.66042196750641, + 38.57172594871372, + 75.46621793881059, + 38.436721079051495, + 72.60013394989073, + 76.3658790383488, + 38.860194967128336, + 73.75223899725825, + 38.60297589562833, + 39.060689974576235 + ], + "after": [ + 37.776404060423374, + 36.784694995731115, + 36.321370978839695, + 36.35120403487235, + 36.09032602980733, + 74.06754896510392, + 36.53954900801182, + 71.28948497120291, + 74.33333899825811, + 37.268035928718746, + 72.1440309425816, + 36.91740706562996, + 37.01140102930367, + 36.919855047017336, + 36.93395503796637, + 36.86134691815823, + 37.082409020513296, + 37.04060195013881, + 37.2941589448601, + 74.28962702397257 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 38.704825507011265, + "after": 37.02600148972124 + }, + "metadata_exact": true + }, + "128k-uniform-1024-contiguous": { + "samples_ms": { + "before": [ + 3.268019063398242, + 2.554591977968812, + 2.479085000231862, + 2.5484160287305713, + 2.4840160040184855, + 2.4452970828861, + 2.4797720834612846, + 2.4469959316775203, + 2.4430269841104746, + 2.4849720066413283, + 2.456852002069354, + 2.4770579766482115, + 2.437687013298273, + 2.4830519687384367, + 2.3991750786080956, + 2.492676954716444, + 2.4221889907494187, + 2.460542949847877, + 2.421214012429118, + 2.4535140255466104 + ], + "after": [ + 2.305899979546666, + 2.254140912555158, + 2.2742359433323145, + 2.231505000963807, + 2.252420992590487, + 2.2559830686077476, + 2.2199279628694057, + 2.2693349746987224, + 2.2399929584935308, + 2.216992899775505, + 2.2454659920185804, + 2.216396969743073, + 2.266811905428767, + 2.1937619894742966, + 2.2288409527391195, + 2.2255079820752144, + 2.255016006529331, + 2.1914190147072077, + 2.2441610926762223, + 2.207132987678051 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.4688004632480443, + "after": 2.2420770255848765 + }, + "metadata_exact": true + } + }, + "source_sha256": { + "cula/distributed/thd_planner.py": "960a403b1e804b087653fd5c8032781fd2d1249f37a6a7b0f2b80c3fca1896f6", + "cula/hybrid/dsa.py": "853f7045ead4d230d4bd5b0a211d60861c840c6190c6502bfdcde310cf866bc2", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f" + }, + "metadata_comparison_scope": "All plan fields except evaluated_scores; omitted nonwinning candidates are checked in the separate planner parity suite", + "warmups_per_variant": 2, + "order": "Alternating before/after and after/before, 20 samples per variant" + }, + "schema_version": 1, + "tested_source_commit": "489779ba35e2c5f51eccce7c7ff95ed4cdd640d0", + "validation": { + "focused_cpu_tests_passed": 518, + "planner_comparisons": 660, + "planner_comparison_scope": "Selected layout, metrics and score exactly match baseline; retained candidate scores match; omitted candidates cannot meet migration margin", + "random_dsa_plans": 300, + "identical_dsa_windows": 69281, + "native_subgroup_checks": 128, + "native_subgroup_scope": "Unchanged native runtime: BF16 and FP32 weights, explicit subgroups, empty destinations, retained outputs", + "complete_csa_cases": 24, + "paths_per_case": 4, + "ranks_per_case": 8, + "compared_tensors_per_public_path": 8, + "comparison": "Full tensor output and seven gradients against legacy contiguous; native nondeterminism enabled", + "relative_rms_limit": 0.01, + "auxiliary_loss_abs_limit": 1e-05, + "small_fp32_oracle_cases": 3, + "small_fp32_oracle_scope": "All four paths; attention, teacher/KL and pooling with selected keys; no independent native ranking oracle", + "cp64_scope": "CPU metadata only" + } +}