diff --git a/README.md b/README.md index e1d4c545..2206afee 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,8 @@ kernel boundary and the overlap assessment against FlashInfer and cuDNN. The optional [cuDNN hybrid-attention runtime](docs/HYBRID_CUDNN_RUNTIME.md) documents packed index pooling, memory-bounded DSA selection, and CP layout ownership without precision downgrades. +The [CSA CP selection API](docs/CSA_CP_SELECTION.md) packages adaptive THD +planning, query routing and result restoration for downstream frameworks. ## Benchmarks diff --git a/benchmarks/bench_csa_cp_windows.py b/benchmarks/bench_csa_cp_windows.py new file mode 100644 index 00000000..e5c10a27 --- /dev/null +++ b/benchmarks/bench_csa_cp_windows.py @@ -0,0 +1,271 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Public CSA selection API: complete CSA F/B across packed context lengths. + +No projections, KDA, MoE, optimizer, or checkpoint recomputation are included. +All variants use the same BF16 inputs and FP32 communication/reduction contract. +""" + +import argparse +import datetime +import gc +import hashlib +import json +import math +import os +import statistics +import time +from dataclasses import replace +from pathlib import Path + +import torch +import torch.distributed as dist + +from benchmarks.bench_full_csa_cp import FullCSA, check_redistribution, check_reference, compare_cpu_reference, json_safe +from cula.distributed import THDPlannerConfig, THDSelectionCostModel + + +def cases(tokens): + """Block-aligned layouts with one, dozens, and hundreds of documents.""" + short = 64 + mixed = [short] * 998 + [(tokens - 998 * short) // 2] * 2 + # The two long documents have different positions in a reproducible pack. + mixed = mixed[:199] + [mixed[-1]] + mixed[199:-2] + [mixed[-2]] + tail = [64] * 800 + [256] * 199 + [tokens - 800 * 64 - 199 * 256] + return { + "single": [tokens], + "uniform-64": [tokens // 64] * 64, + "near-uniform-64": [tokens // 64 - tokens // 1024, tokens // 64 + tokens // 1024] * 32, + "mixed-22": [1024] * 20 + [(tokens - 20 * 1024) // 2] * 2, + "uniform-1024": [tokens // 1024] * 1024, + "near-uniform-512": [tokens // 512 - 4, tokens // 512 + 4] * 256, + "mixed-1000": mixed, + "long-tail-1000": tail, + } + + +def tensors(csa): + r, b = csa.last, csa.indexer + return dict( + output=r["source_output"], + dq=r["source_dq"], + dkv=r["local_dkv"], + d_index_q=r["source_dqi"], + d_weights=r["source_dw"], + d_pool_kv=b.local_kv.grad, + d_pool_score=b.pool_score.grad, + d_ape=b.ape.grad, + ) + + +def iteration(csa): + dist.barrier(device_ids=[torch.cuda.current_device()]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, end = (torch.cuda.Event(enable_timing=True) for _ in range(2)) + wall = time.perf_counter() + begin.record() + csa.forward() + csa.backward() + end.record() + end.synchronize() + return dict( + total_ms=begin.elapsed_time(end), + wall_ms=1000 * (time.perf_counter() - wall), + allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + reserved_gib=torch.cuda.max_memory_reserved() / 2**30, + **csa.timings(), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--tokens", type=int, nargs="+", default=[128 << 10, 256 << 10]) + parser.add_argument("--cases", nargs="+", help="Subset of the named distributions") + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--prepare-repeats", type=int, default=8) + parser.add_argument("--min-gain-ms", type=float, help="Explicit preparation budget override, recorded in the report") + parser.add_argument( + "--min-chunk-tokens", type=int, default=1024, help="Minimum newly created planner span; independent of context length" + ) + parser.add_argument("--max-chunks-per-rank", type=int, default=6) + parser.add_argument("--check-only", action="store_true", help="Small independent FP32 oracle fixtures") + args = parser.parse_args() + if min(args.warmups, args.repeats, args.prepare_repeats) < 1 or any(t < 128 << 10 or t % 4096 for t in args.tokens): + parser.error("warmups/repeats must be positive; contexts must be >= 128K and divisible by 4096") + torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0))) + device = torch.device("cuda", torch.cuda.current_device()) + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=10)) + rank, world = dist.get_rank(), dist.get_world_size() + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False + profiles = [json.loads(args.profile.read_text())["model"] if rank == 0 else None] + dist.broadcast_object_list(profiles, src=0) + profiles[0]["rank_node_ids"] = tuple(profiles[0]["rank_node_ids"]) + model = THDSelectionCostModel(**profiles[0]) + if args.min_gain_ms is not None: + model = replace(model, min_gain_ms=args.min_gain_ms) + if len(model.rank_node_ids) != world: + raise ValueError("profile topology does not match world size") + args.output_dir.mkdir(parents=True, exist_ok=True) + from dataclasses import asdict + + manifest = dict(arguments={k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()}, model=asdict(model)) + source = Path(__file__).resolve().parents[1] + import importlib.metadata + + try: + frontend_version = importlib.metadata.version("nvidia-cudnn-frontend") + except importlib.metadata.PackageNotFoundError: + frontend_version = None # Source-only frontend checkouts have no wheel metadata. + manifest["software"] = { + "torch": torch.__version__, + "cuda_runtime": torch.version.cuda, + "cutlass_dsl": importlib.metadata.version("nvidia-cutlass-dsl"), + "cudnn_frontend": frontend_version, + } + manifest["source_sha256"] = { + str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() + for directory in (source / "cula", source / "benchmarks") + for p in sorted(directory.rglob("*.py")) + } + manifests = [None] * world + dist.all_gather_object(manifests, manifest) + if any(m != manifests[0] for m in manifests): + raise RuntimeError("source/config differs across ranks") + if rank == 0: + (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + suite = [ + (f"{t // 1024}k-{name}", lengths) + for t in args.tokens + for name, lengths in cases(t).items() + if not args.cases or name in args.cases + ] + if args.check_only: + suite = [("check-single", [4096]), ("check-many", [4] * 1024), ("check-mixed", [4] * 64 + [1920] * 2)] + if not suite: + raise ValueError("no matching cases") + for name, lengths in suite: + variants, samples, checks, preparation = {}, {}, {}, {} + policies = ("legacy-contiguous", "contiguous", "zigzag", "planner") + for policy in policies: + torch.manual_seed(20261116 + rank) + csa = FullCSA( + lengths, + 64, + 512, + device, + 0, + layout="contiguous" if policy.startswith("legacy") else policy, + planner_cost_model=model, + planner_strategy="selection", + selection_api=not policy.startswith("legacy"), + planner_config=THDPlannerConfig( + min_chunk_tokens=args.min_chunk_tokens, max_chunks_per_rank=args.max_chunks_per_rank + ), + ) + variants[policy], samples[policy] = csa, [] + if not check_redistribution(csa): + raise RuntimeError("row identity/inverse failed") + if rank == 0: + print("CASE", name, "selected", variants["planner"].selected_policy, flush=True) + orders = [] + for i in range(args.warmups + args.repeats): + order = policies[i % len(policies) :] + policies[: i % len(policies)] + if i % 2: + order = order[::-1] + orders.append(order) + for policy in order: + sample = iteration(variants[policy]) + if i >= args.warmups: + samples[policy].append(sample) + if rank == 0: + print("ITER", name, i, {p: round(samples[p][-1]["total_ms"], 3) for p in policies if samples[p]}, flush=True) + # Fresh metadata is measured after kernels/allocators are warm, in + # rotated order. Retain cold measurements separately, without silently + # substituting cache lookup time for fresh plan construction. + cold_prepare = {p: dict(common_ms=c.common_prepare_ms, layout_ms=c.layout_prepare_ms) for p, c in variants.items()} + prepare_orders = [] + for j in range(args.prepare_repeats): + order = policies[j % len(policies) :] + policies[: j % len(policies)] + if j % 2: + order = order[::-1] + prepare_orders.append(order) + for p in order: + dist.barrier(device_ids=[torch.cuda.current_device()]) + preparation.setdefault(p, []).append(variants[p].prepare_metadata()) + reference = {k: v.detach().cpu() for k, v in tensors(variants["legacy-contiguous"]).items()} + # Loss is normalized by global tokens in FullCSA. Reduce its scalar + # separately; selection routing must not change the objective. + losses = torch.stack([variants[p].last["loss"].detach() for p in policies]) + dist.all_reduce(losses) + loss_values = losses.cpu().tolist() + okay = True + for policy, csa in variants.items(): + errors = {k: compare_cpu_reference(v, reference[k]) for k, v in tensors(csa).items()} + ref = check_reference(csa) if args.check_only else None + okay &= all(e["finite"] and e["relative_rms"] < 0.01 for e in errors.values()) + loss = loss_values[policies.index(policy)] + loss_delta = abs(loss - loss_values[0]) + okay &= ref is None or ref["pass_"] + okay &= math.isfinite(loss) and loss_delta < 1e-5 + checks[policy] = dict( + errors=errors, + global_loss=loss, + loss_abs_diff=loss_delta, + fp32=ref, + selected_policy=csa.selected_policy, + cold_prepare=cold_prepare[policy], + prepare_samples=preparation[policy], + source_rows=csa.source_n, + selection_rows=csa.selection_layout.local_tokens, + selection_spans=len(csa.selection_layout.spans), + workspace_bytes=0 + if csa.indexer.select_plan is None + else csa.indexer.select_plan.candidate_workspace_floats * 4, + ) + records = [None] * world + dist.all_gather_object(records, dict(rank=rank, samples=samples, checks=checks, okay=okay)) + if rank == 0: + summary = {} + for policy in policies: + step = statistics.median( + max(r["samples"][policy][i]["total_ms"] for r in records) for i in range(args.repeats) + ) + prepare_samples = [ + max(sum(r["checks"][policy]["prepare_samples"][j].values()) for r in records) + for j in range(args.prepare_repeats) + ] + prepare = statistics.median(prepare_samples) + summary[policy] = dict( + step_ms=step, + prepare_ms=prepare, + one_use_ms=step + prepare, + prepare_min_ms=min(prepare_samples), + prepare_max_ms=max(prepare_samples), + ) + report = dict( + case=name, + lengths=lengths, + world_size=world, + orders=orders, + prepare_orders=prepare_orders, + summary=summary, + ranks=records, + ) + (args.output_dir / f"{name}.json").write_text(json.dumps(json_safe(report), indent=2, allow_nan=False) + "\n") + print("RESULT", name, json.dumps(summary), flush=True) + if not all(r["okay"] for r in records): + raise RuntimeError("numerical qualification failed; result retained") + del variants, csa, reference, records + gc.collect() + torch.cuda.empty_cache() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_csa_layout_matrix.py b/benchmarks/bench_csa_layout_matrix.py new file mode 100644 index 00000000..18e97c65 --- /dev/null +++ b/benchmarks/bench_csa_layout_matrix.py @@ -0,0 +1,155 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Real-CP layout comparison with full output/gradient baselines in CPU RAM.""" + +import argparse +import gc +import sys +from pathlib import Path + +from benchmarks.thd_1m_cases import DEFAULT_SEED, counted_cases, heldout_cases, statistics_cases + + +def performance_cases(suite="baseline", seed=DEFAULT_SEED): + if suite == "validation": + if seed != DEFAULT_SEED: + raise ValueError("the validation suite fixes its seeds for reproducibility") + return [ + *performance_cases("baseline"), + *performance_cases("counted"), + *( + (f"{case.name}-seed{heldout_seed}", list(case.lengths), False) + for heldout_seed in (20261114, 20261115, 20261116) + for case in heldout_cases(heldout_seed) + ), + ] + cases = [] + if suite in ("baseline", "all"): + cases += [ + ("1m-single", [1048576], False), + ("1m-uniform", [16384] * 64, False), + ("1m-near-uniform", [15360, 17408] * 32, False), + ("1m-20s2l", [1024] * 20 + [514048] * 2, False), + ("1m-power-law", [1024, 4096, 16384, 65536, 131072, 307200, 523264], False), + ] + if suite in ("statistics", "all"): + cases += [(case.name, list(case.lengths), False) for case in statistics_cases(seed)] + if suite in ("counted", "all"): + cases += [(case.name, list(case.lengths), False) for case in counted_cases(seed)] + if suite in ("heldout", "all"): + cases += [(case.name, list(case.lengths), False) for case in heldout_cases(seed)] + if not cases: + raise ValueError(f"unknown suite: {suite}") + return cases + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--checks-only", action="store_true") + parser.add_argument( + "--suite", choices=("baseline", "statistics", "counted", "heldout", "all", "validation"), default="baseline" + ) + parser.add_argument("--cases", nargs="+", help="Exact performance case names from the selected suite") + parser.add_argument("--seed", type=int, default=DEFAULT_SEED, help="Synthetic THD generation seed") + parser.add_argument("--list-cases", action="store_true", help="List shapes without initializing CUDA/distributed") + parser.add_argument("--redistribution-scope", choices=("selection", "queries"), default="selection") + parser.add_argument("--skip-checks", action="store_true", help="Skip small FP32 checks after a separately validated run") + parser.add_argument( + "--layouts", nargs="+", choices=("contiguous", "zigzag", "planner"), default=["contiguous", "zigzag", "planner"] + ) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--heads", type=int, default=64, help="Attention heads; native SM100 D576 requires 64") + parser.add_argument("--planner-cost-model", help="JSON calibrated selection-routing coefficients") + parser.add_argument("--memory-stages", action="store_true") + parser.add_argument("--planner-strategy", choices=("balanced", "selection"), default="balanced") + args = parser.parse_args() + selected = performance_cases(args.suite, args.seed) + if args.cases: + unknown = set(args.cases) - {name for name, _, _ in selected} + if unknown: + parser.error(f"unknown cases for suite {args.suite}: {sorted(unknown)}") + selected = [case for case in selected if case[0] in args.cases] + if args.checks_only and args.cases: + parser.error("--cases cannot be combined with --checks-only") + if args.checks_only and args.skip_checks: + parser.error("--checks-only and --skip-checks are mutually exclusive") + if args.layouts[0] != "contiguous" or len(set(args.layouts)) != len(args.layouts): + parser.error("layouts must start with the contiguous numerical baseline and contain no duplicates") + if args.warmups < 0 or args.repeats < 1: + parser.error("warmups must be nonnegative and repeats positive") + if args.list_cases: + for name, lengths, _ in selected: + print(name, "documents", len(lengths), "tokens", sum(lengths)) + return + + import torch + import torch.distributed as dist + + from benchmarks.bench_full_csa_cp import main as run_case + + args.output_dir.mkdir(parents=True, exist_ok=True) + cases = [ + ("check-single", [32768], True), + ("check-mixed", [4] * 20 + [2008] * 2, True), + ("check-uniform", [256] * 16, True), + ("check-dense", [4] * 256 + [32] * 96, True), + ] + if args.skip_checks: + cases = [] + if not args.checks_only: + cases += selected + for case_index, (name, lengths, check) in enumerate(cases): + baseline = None + # Keep the numerical baseline first, alternate the remaining policies + # to avoid always giving one policy the same thermal/cache order. + layouts = args.layouts if case_index % 2 == 0 else [args.layouts[0], *reversed(args.layouts[1:])] + for layout in layouts: + print("CASE", name, layout, flush=True) + sys.argv = [ + "bench_full_csa_cp.py", + "--sequence-lengths", + *map(str, lengths), + "--heads", + str(args.heads), + "--top-k", + "512", + "--with-kda", + "--layout", + layout, + "--redistribution-scope", + args.redistribution_scope, + "--planner-strategy", + args.planner_strategy, + "--pool-backward-order", + "per-loss", + "--output", + str(args.output_dir / f"layout-{name}-{layout}.json"), + "--warmups", + "0" if check else str(args.warmups), + "--repeats", + "1" if check else str(args.repeats), + "--candidate-mib", + "64" if check else "0", + ] + if check: + sys.argv.append("--check") + if args.planner_cost_model: + sys.argv += ["--planner-cost-model", args.planner_cost_model] + if args.memory_stages: + sys.argv.append("--memory-stages") + saved = run_case(keep_group=True, baseline=baseline, save_baseline=layout == "contiguous") + if saved is not None: + baseline = saved + gc.collect() + torch.cuda.empty_cache() + dist.barrier(device_ids=[torch.cuda.current_device()]) + del baseline, saved + gc.collect() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_full_csa_cp.py b/benchmarks/bench_full_csa_cp.py new file mode 100644 index 00000000..c4f9cf68 --- /dev/null +++ b/benchmarks/bench_full_csa_cp.py @@ -0,0 +1,941 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Eager, real-CP KDA + full MLA-absorb CSA operator benchmark. + +This experiment includes pooling, selection, attention, the detached MaxPool +teacher/KL indexer backward, and all CP activation/gradient collectives. It is +not a model benchmark: projection, residual, MLP and optimizer are excluded. +It deliberately reuses allocating cuDNN convenience APIs; no CUDA Graph or +bounded compilation-cache guarantee is made for this experimental harness. +Document lengths and CP cuts must be multiples of four. Per-query causal +partial blocks (including the first three queries of every doc) are supported. +""" + +import argparse +import datetime +import json +import math +import os +import statistics +import time +from dataclasses import asdict +from itertools import accumulate +from types import SimpleNamespace + +import torch +import torch.distributed as dist + +from benchmarks.cp_layouts import query_layouts +from cula._metadata import balanced_shards +from cula.distributed import ( + AllGatherKimiCPTransport, + THDRedistributionPlan, + THDRedistributor, + THDSelectionCostModel, + build_kimi_cp_plan, + cudnn_kda_cp, +) +from cula.hybrid import ( + CsaCPSelectionPlan, + CsaCPSelectionWorkspace, + DsaSelectionPlan, + DsaSelectionWorkspace, + IndexPoolPlan, + cudnn_csa_select_cp, + cudnn_dsa_select, + cudnn_index_pool, +) +from cula.hybrid.index_pool import _load_cudnn_index_pool_ops + + +def validate_source_alignment(lengths, world): + """Make the same alignment decision on every balanced source rank.""" + if sum(lengths) % (4 * world) or any(n % 4 for n in lengths): + raise ValueError("source CP shards and documents must be block aligned") + + +def json_safe(value): + """Replace nonfinite measurements only in the serialized report copy.""" + if isinstance(value, dict): + return {key: json_safe(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [json_safe(item) for item in value] + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + +def metrics(actual, expected): + actual, expected = actual.float(), expected.float() + delta = actual - expected + return { + "max_abs": delta.abs().max().item(), + "relative_rms": (delta.norm() / expected.norm().clamp_min(1e-12)).item(), + "finite": bool(torch.isfinite(actual).all()), + } + + +def expand_blocks(block_ids, scores, doc_start, visible): + """Compact valid block slots, expand members, append the causal tail. + + Sorting by validity preserves each valid (id, score) pairing. The returned + block ids are still document-local; token ids are globally addressed. + Invalid slots remain -1 even when topk_length is supplied to cuDNN. + """ + order = (block_ids < 0).to(torch.int32).argsort(dim=-1, stable=True) + ids = block_ids.gather(1, order) + scores = scores.gather(1, order) + valid = ids >= 0 + nblocks = valid.sum(-1, dtype=torch.int32) + members = doc_start[:, None, None] + 4 * ids[:, :, None] + torch.arange(4, device=ids.device) + members = members.masked_fill(~valid[:, :, None], -1).flatten(1).to(torch.int32) + # The padded row is friendly to both the attention and teacher kernels. + width = math.ceil((members.shape[1] + 3) / 128) * 128 + tokens = torch.full((ids.shape[0], width), -1, dtype=torch.int32, device=ids.device) + tokens[:, : members.shape[1]] = members + slots = nblocks[:, None] * 4 + torch.arange(3, device=ids.device) + tail = doc_start[:, None] + visible[:, None] // 4 * 4 + torch.arange(3, device=ids.device) + tail = tail.masked_fill(torch.arange(3, device=ids.device)[None] >= visible[:, None] % 4, -1) + tokens.scatter_(1, slots.long(), tail.to(torch.int32)) + lengths = nblocks * 4 + visible % 4 + return ids, scores, members, tokens, lengths.to(torch.int32) + + +def reduce_scatter_fp32(value, local_rows): + out = torch.empty((local_rows, *value.shape[1:]), device=value.device, dtype=torch.float32) + dist.reduce_scatter_tensor(out, value.float().contiguous()) + return out + + +def query_metadata(layout, device): + """Materialize all span addresses with one transfer, including empty ranks.""" + sizes = torch.tensor([span.tokens for span in layout.spans], dtype=torch.int64, device="cpu") + starts = torch.tensor([span.sequence_start for span in layout.spans], dtype=torch.int32, device="cpu") + shifts = torch.tensor( + [span.token_start - offset for span, offset in zip(layout.spans, (0, *accumulate(sizes.tolist())))], + dtype=torch.int32, + device="cpu", + ) + n = layout.local_tokens + origin = torch.repeat_interleave(starts, sizes, output_size=n) + rows = torch.arange(n, dtype=torch.int32, device="cpu") + torch.repeat_interleave(shifts, sizes, output_size=n) + return torch.stack((origin, rows - origin + 1, rows)).to(device).unbind(0) + + +class FullCSA: + def __init__( + self, + lengths, + heads, + top_k, + device, + candidate_mib, + pool_backward_order="per-loss", + layout="contiguous", + redistribution_scope="selection", + planner_cost_model=None, + planner_strategy="balanced", + selection_api=False, + planner_config=None, + ): + import cudnn + + self.ops = cudnn.DSA + self.top_k = top_k + self.pool_backward_order = pool_backward_order + self.redistribution_scope = redistribution_scope + self.selection_api = selection_api + if selection_api and ( + redistribution_scope != "selection" or (layout == "planner" and planner_strategy != "selection") + ): + raise ValueError("the public API requires selection-only routing and the selection planner") + _, self.pool_backward = _load_cudnn_index_pool_ops() + torch.cuda.synchronize(device) + common_prepare_start = time.perf_counter() + self.rank, self.world = dist.get_rank(), dist.get_world_size() + validate_source_alignment(lengths, self.world) + self.cu = (0, *accumulate(lengths)) + self.plan = build_kimi_cp_plan(self.cu, group=dist.group.WORLD) + self.source_n, self.total = self.plan.local_tokens, self.cu[-1] + self.scale, self.heads = 576**-0.5, heads + n = self.source_n + pool_plan = IndexPoolPlan.from_cu_seqlens(self.plan.local_cu_seqlens, head_dim=128, ratio=4) + pool_cu_tokens, pool_cu_blocks = pool_plan.materialize(device) + torch.cuda.synchronize(device) + self.common_prepare_ms = (time.perf_counter() - common_prepare_start) * 1000 + # All random tensors are generated in SOURCE ownership, with identical + # RNG consumption across layouts. No target-layout synthetic inputs. + self.indexer = SimpleNamespace( + qi=torch.randn(n, 64, 128, dtype=torch.bfloat16, device=device), + local_kv=torch.randn(n, 128, dtype=torch.bfloat16, device=device), + pool_score=torch.randn(n, 128, dtype=torch.bfloat16, device=device), + ape=torch.randn(4, 128, dtype=torch.float32, device=device), + weights=torch.randn(n, 64, dtype=torch.float32, device=device), + global_kbar=torch.empty(self.total // 4, 128, dtype=torch.bfloat16, device=device), + pool_plan=pool_plan, + pool_cu_tokens=pool_cu_tokens, + pool_cu_blocks=pool_cu_blocks, + local_blocks=n // 4, + ) + b = self.indexer + # Match the native backward's BF16 weights contract from the outset; + # do not quantize an FP32 forward operand only for backward. + b.weights = (b.weights / (64 * math.sqrt(128))).bfloat16() + self.source_qi, self.source_weights = b.qi, b.weights + for x in (b.local_kv, b.pool_score, b.ape): + x.requires_grad_(True) + # APE is a replicated parameter, not an independent rank-local input. + dist.broadcast(b.ape.detach(), src=0) + self.source_q = torch.randn(n, heads, 576, device=device, dtype=torch.bfloat16) * 0.5 + self.kv = torch.randn(n, 576, device=device, dtype=torch.bfloat16) * 0.5 + self.source_dout = torch.randn(n, heads, 512, device=device, dtype=torch.bfloat16) + self.global_kv = torch.empty(self.total, 576, device=device, dtype=torch.bfloat16) + self.sink = torch.full((heads,), -torch.inf, device=device, dtype=torch.float32) + self.unit_grad = torch.ones((), device=device, dtype=torch.float32) + self._layout_options = dict( + device=device, + candidate_mib=candidate_mib, + layout=layout, + redistribution_scope=redistribution_scope, + planner_cost_model=planner_cost_model, + planner_strategy=planner_strategy, + selection_api=selection_api, + planner_config=planner_config, + ) + self._prepare_layout(**self._layout_options) + self.events = [] + self.last = {} + + def prepare_metadata(self): + """Rebuild fresh metadata after warmup, without regenerating inputs. + + Synchronization includes H2D metadata copies and workspace allocation; + compilation and RNG initialization are outside this preparation metric. + """ + device = self.source_q.device + torch.cuda.synchronize(device) + start = time.perf_counter() + self.plan = build_kimi_cp_plan(self.cu, group=dist.group.WORLD) + b = self.indexer + b.pool_plan = IndexPoolPlan.from_cu_seqlens(self.plan.local_cu_seqlens, head_dim=128, ratio=4) + b.pool_cu_tokens, b.pool_cu_blocks = b.pool_plan.materialize(device) + torch.cuda.synchronize(device) + self.common_prepare_ms = (time.perf_counter() - start) * 1000 + self._prepare_layout(**self._layout_options) + return dict(common_ms=self.common_prepare_ms, layout_ms=self.layout_prepare_ms) + + def _prepare_layout( + self, + *, + device, + candidate_mib, + layout, + redistribution_scope, + planner_cost_model, + planner_strategy, + selection_api, + planner_config, + ): + b, top_k = self.indexer, self.top_k + torch.cuda.synchronize(device) + prepare_start = time.perf_counter() + if planner_cost_model is not None and redistribution_scope != "selection": + raise ValueError("the selection cost model is valid only for selection-only redistribution") + if planner_strategy == "selection" and redistribution_scope != "selection": + raise ValueError("selection planning requires selection-only redistribution") + if selection_api: + cp_selection_plan = CsaCPSelectionPlan.from_cu_seqlens( + self.cu, + rank=self.rank, + world_size=self.world, + top_k=top_k, + policy=layout, + cost_model=planner_cost_model, + config=planner_config, + max_tokens_per_rank=self.total if layout == "zigzag" else None, + candidate_budget_bytes=None if candidate_mib == 0 else candidate_mib << 20, + ) + self.cp_selection = CsaCPSelectionWorkspace.from_plan(cp_selection_plan, device, group=dist.group.WORLD) + ranks, self.selected_policy = cp_selection_plan.ranks, cp_selection_plan.policy + else: + ranks, self.selected_policy = query_layouts( + self.cu, self.world, layout, cost_model=planner_cost_model, planner_strategy=planner_strategy + ) + if redistribution_scope == "queries" and any(r.local_tokens == 0 for r in ranks): + raise ValueError( + "query-wide redistribution requires nonempty attention ranks; use selection scope for empty ranks" + ) + self.selection_layout = ranks[self.rank] + source_offsets = balanced_shards(self.total, self.world) + if selection_api: + self.selection_exchange = self.cp_selection.exchange + else: + exchange_plan = THDRedistributionPlan.build(ranks, source_offsets, self.rank) + self.selection_exchange = THDRedistributor(exchange_plan, device, group=dist.group.WORLD) + if redistribution_scope == "selection": + attention_ranks, _ = query_layouts(self.cu, self.world, "contiguous") + self.layout = attention_ranks[self.rank] + self.exchange = THDRedistributor( + THDRedistributionPlan.build(attention_ranks, source_offsets, self.rank), device, group=dist.group.WORLD + ) + else: + self.layout = self.selection_layout + self.exchange = self.selection_exchange + self.n = self.layout.local_tokens + self.doc_start, self.visible, self.query_global_rows = query_metadata(self.layout, device) + key_prefix, key_ranges = [0], [] + for span in self.selection_layout.spans: + key_prefix.append(key_prefix[-1] + (span.sequence_end - span.sequence_start) // 4) + key_ranges.append((span.sequence_start // 4, span.sequence_end // 4)) + b.select_plan = None + b.select_workspace = None + if selection_api: + b.select_plan, b.select_workspace = cp_selection_plan.selection, self.cp_selection.selection + elif self.selection_layout.local_tokens: + b.select_plan = DsaSelectionPlan.from_cu_seqlens( + self.selection_layout.local_cu_seqlens, + key_prefix, + top_k=top_k, + block_size=4, + q_causal_offsets=self.selection_layout.causal_offsets, + candidate_budget_bytes=None if candidate_mib == 0 else candidate_mib << 20, + ) + b.select_workspace = DsaSelectionWorkspace.from_plan(b.select_plan, device) + self.selection_key_ids = None + if selection_api: + self.selection_key_slice, self.selection_key_ids = self.cp_selection.key_slice, self.cp_selection.key_indices + elif not key_ranges: + self.selection_key_slice = slice(0, 0) + elif all(left[1] == right[0] for left, right in zip(key_ranges, key_ranges[1:])): + self.selection_key_slice = slice(key_ranges[0][0], key_ranges[-1][1]) + else: + self.selection_key_ids = torch.cat([torch.arange(a, z, device="cpu") for a, z in key_ranges]).to(device) + torch.cuda.synchronize(device) + self.layout_prepare_ms = (time.perf_counter() - prepare_start) * 1000 + + def mark(self, name): + event = torch.cuda.Event(enable_timing=True) + event.record() + self.events.append((name, event)) + if getattr(self, "record_memory", False): + self.memory_stages[name] = allocator_usage() + + def forward(self): + b = self.indexer + self.events = [] + self.memory_stages = {} + self.mark("start") + # Release prior iteration references before allocating the next layout. + self.last = {} + with torch.cuda.nvtx.range("csa.query_redistribute"): + self.q = self.exchange.distribute(self.source_q) + selection_qi = self.source_qi if self.selection_api else self.selection_exchange.distribute(self.source_qi) + selection_weights = ( + self.source_weights if self.selection_api else self.selection_exchange.distribute(self.source_weights) + ) + b.qi = self.source_qi if self.redistribution_scope == "selection" else selection_qi + b.weights = self.source_weights if self.redistribution_scope == "selection" else selection_weights + self.mark("query_redistribute") + for x in (b.local_kv, b.pool_score, b.ape): + x.grad = None + with torch.cuda.nvtx.range("csa.pool"): + pooled = cudnn_index_pool( + b.local_kv, b.pool_score, b.ape, b.pool_cu_tokens, b.pool_cu_blocks, plan=b.pool_plan, deterministic=False + ) + self.mark("pool_fwd") + with torch.cuda.nvtx.range("csa.pool_gather"): + dist.all_gather_into_tensor(b.global_kbar, pooled.detach()) + self.mark("pool_gather") + with torch.cuda.nvtx.range("csa.selection_key_layout"): + select_keys = ( + None + if self.selection_api + else ( + b.global_kbar[self.selection_key_slice] + if self.selection_key_ids is None + else b.global_kbar.index_select(0, self.selection_key_ids) + ) + ) + self.mark("selection_key_layout") + with torch.cuda.nvtx.range("csa.selection"): + if self.selection_api: + selected = cudnn_csa_select_cp( + self.source_qi, + b.global_kbar[:, None], + self.source_weights, + workspace=self.cp_selection, + deterministic=False, + ) + elif b.select_plan is None: + selected = SimpleNamespace( + block_ids=torch.empty((0, self.top_k), device=selection_qi.device, dtype=torch.int32), + scores=torch.empty((0, self.top_k), device=selection_qi.device, dtype=torch.float32), + ) + else: + selected = cudnn_dsa_select( + selection_qi, + select_keys[:, None], + selection_weights, + plan=b.select_plan, + workspace=b.select_workspace, + deterministic=False, + ) + self.mark("selection") + with torch.cuda.nvtx.range("csa.selection_restore"): + if self.redistribution_scope == "selection" and not self.selection_api: + selected_ids = self.selection_exchange.restore(selected.block_ids) + selected_scores = self.selection_exchange.restore(selected.scores) + else: + selected_ids, selected_scores = selected.block_ids, selected.scores + del selection_qi, selection_weights, select_keys, selected + self.mark("selection_restore") + with torch.cuda.nvtx.range("csa.expand_tail"): + ids, scores, members, tokens, counts = expand_blocks(selected_ids, selected_scores, self.doc_start, self.visible) + block_ids = (ids + self.doc_start[:, None] // 4).masked_fill(ids < 0, -1) + self.mark("expand_tail") + with torch.cuda.nvtx.range("csa.kv_gather"): + dist.all_gather_into_tensor(self.global_kv, self.kv) + self.mark("kv_gather") + with torch.cuda.nvtx.range("csa.attention_fwd"): + fw = self.ops.sparse_attention_forward_wrapper( + self.q, self.global_kv, tokens, attn_sink=self.sink, topk_length=counts, softmax_scale=self.scale + ) + self.mark("attention_fwd") + with torch.cuda.nvtx.range("csa.teacher_kl"): + teacher = self.ops.sparse_attn_score_recompute_wrapper( + self.q[None], + self.global_kv[None], + fw["lse"][None], + members[None], + softmax_scale=self.scale, + qhead_per_kv_head=self.heads, + topk_indices_global=True, + )["target"][0] + target = teacher.view(self.n, -1, 4).amax(-1) + target = target / target.sum(-1, keepdim=True).clamp_min(1e-10) + valid = ids >= 0 + # Empty selected sets (first three queries) have zero auxiliary loss. + safe_scores = scores.masked_fill(~valid, -torch.inf) + safe_scores = torch.where(valid.any(-1, keepdim=True), safe_scores, torch.zeros_like(safe_scores)) + log_predict = safe_scores.log_softmax(-1).masked_fill(~valid, 0) + predict = log_predict.exp().masked_fill(~valid, 0) + loss = (target * (target.clamp_min(1e-10).log() - log_predict)).sum() / self.total + self.mark("teacher_kl") + with torch.cuda.nvtx.range("csa.output_restore"): + source_output = self.exchange.restore(fw["out"]) + self.mark("output_restore") + self.last = dict( + source_output=source_output, + pooled=pooled, + ids=ids, + block_ids=block_ids, + members=members, + tokens=tokens, + counts=counts, + target=target, + predict=predict, + loss=loss, + fw=fw, + ) + return source_output + + def backward(self): + b, r = self.indexer, self.last + with torch.cuda.nvtx.range("csa.dout_redistribute"): + self.dout = self.exchange.distribute(self.source_dout) + self.mark("dout_redistribute") + with torch.cuda.nvtx.range("csa.attention_bwd"): + bw = self.ops.sparse_attention_backward_wrapper( + self.q, + self.global_kv, + r["fw"]["out"], + self.dout, + r["fw"]["lse"], + self.sink, + r["tokens"], + softmax_scale=self.scale, + topk_length=r["counts"], + deterministic=False, + ) + self.mark("attention_bwd") + with torch.cuda.nvtx.range("csa.kv_reduce_scatter"): + local_dkv = reduce_scatter_fp32(bw["dkv"], self.source_n) + self.mark("kv_reduce_scatter") + with torch.cuda.nvtx.range("csa.indexer_bwd"): + # The backend overwrites target with its score gradient. Keep the + # reference target intact for numerical validation. + ibw = self.ops.indexer_backward_wrapper( + b.qi[None], + b.weights[None], + b.global_kbar[None], + r["target"][None].clone(), + r["predict"][None].clamp_min(1e-30).masked_fill(r["block_ids"][None] < 0, 0), + r["block_ids"][None], + self.unit_grad, + loss_coeff=self.n / self.total, + topk_indices_global=True, + deterministic=False, + d_index_k=torch.empty_like(b.global_kbar[None], dtype=torch.float32), + ) + self.mark("indexer_bwd") + with torch.cuda.nvtx.range("csa.pool_grad_exchange"): + if self.pool_backward_order == "sum-first": + local_dkbar = reduce_scatter_fp32(ibw["d_index_k"][0], b.local_blocks) + else: + # Preserve cuLA's gather-then-pool numerical ordering: do not + # merge loss-rank gradients before the native BF16 pool API. + incoming = ibw["d_index_k"][0].bfloat16() + received = torch.empty_like(incoming) + dist.all_to_all_single(received, incoming) + received = received.view(self.world, b.local_blocks, 128) + local_dkbar = received.float().sum(0) + self.mark("pool_grad_exchange") + with torch.cuda.nvtx.range("csa.pool_bwd"): + if self.pool_backward_order == "sum-first": + r["pooled"].backward(local_dkbar.bfloat16()) + else: + dkv = torch.zeros_like(b.local_kv, dtype=torch.float32) + ds = torch.zeros_like(b.pool_score, dtype=torch.float32) + da = torch.zeros_like(b.ape) + for loss_rank in range(self.world): + pool_bw = self.pool_backward( + b.local_kv, + b.pool_score, + b.ape, + b.pool_cu_tokens, + b.pool_cu_blocks, + received[loss_rank], + ratio=4, + head_dim=128, + coff=1, + deterministic=False, + ) + dkv.add_(pool_bw["grad_kv"]) + ds.add_(pool_bw["grad_score"]) + da.add_(pool_bw["grad_ape"]) + b.local_kv.grad = dkv.bfloat16() + b.pool_score.grad = ds.bfloat16() + b.ape.grad = da + dist.all_reduce(b.ape.grad) + self.mark("pool_bwd") + with torch.cuda.nvtx.range("csa.query_grad_restore"): + source_dq = self.exchange.restore(bw["dq"]) + source_dqi = self.exchange.restore(ibw["d_index_q"][0]) + source_dw = self.exchange.restore(ibw["d_weights"][0]) + self.mark("query_grad_restore") + r.update(source_dq=source_dq, source_dqi=source_dqi, source_dw=source_dw) + r.update(bw=bw, ibw=ibw, local_dkv=local_dkv, local_dkbar=local_dkbar) + + def timings(self): + return { + name + "_ms": previous.elapsed_time(event) for (_, previous), (name, event) in zip(self.events, self.events[1:]) + } + + +def check_reference(csa): + """Independent FP32 attention/teacher/KL and end-to-end pool gradients. + + Top-K is discrete: check the same selected set, plus exact document, + causality, uniqueness and complete/tail coverage. This oracle does not + independently validate selector ranking. + """ + b, r = csa.indexer, csa.last + ids, tokens = r["ids"], r["tokens"] + valid = tokens >= 0 + global_q = csa.query_global_rows + metadata_ok = bool(torch.all((~valid) | ((tokens >= csa.doc_start[:, None]) & (tokens <= global_q[:, None])))) + sorted_tokens = tokens.sort(-1).values + metadata_ok &= not bool(((sorted_tokens[:, 1:] == sorted_tokens[:, :-1]) & (sorted_tokens[:, 1:] >= 0)).any()) + metadata_ok &= bool(torch.equal(valid.sum(-1), r["counts"])) + metadata_ok &= bool(torch.equal((ids >= 0).sum(-1), (csa.visible // 4).clamp(max=ids.shape[1]))) + # Every causal tail token appears exactly once, even with no full block. + for offset in range(3): + tail = csa.doc_start + csa.visible // 4 * 4 + offset + found = (tokens == tail[:, None]).sum(-1) + metadata_ok &= bool(torch.all(found == (offset < csa.visible % 4).long())) + + kv = csa.global_kv.detach().float().requires_grad_(True) + kbar = b.global_kbar.detach().float().requires_grad_(True) + outputs, dq, dqi, dw, targets = [], [], [], [], [] + reference_loss = 0.0 + for start in range(0, csa.n, 16): + end = min(start + 16, csa.n) + ti = tokens[start:end].long() + mask = ti >= 0 + q = csa.q[start:end].float().detach().requires_grad_(True) + selected_kv = kv[ti.clamp_min(0)] + logits = torch.einsum("qhd,qkd->qhk", q, selected_kv) * csa.scale + probs = logits.masked_fill(~mask[:, None], -torch.inf).softmax(-1) + output = torch.einsum("qhk,qkv->qhv", probs, selected_kv[..., :512]) + output.backward(csa.dout[start:end].float()) + outputs.append(output.detach()) + dq.append(q.grad) + # Complete members occupy the prefix; causal tail starts after the last + # valid block. Mask before pooling to exclude tail slots from teacher. + teacher = probs.detach().sum(1)[:, : ids.shape[1] * 4].reshape(end - start, -1, 4).amax(-1) + teacher = teacher.masked_fill(ids[start:end] < 0, 0) + teacher = teacher / teacher.sum(-1, keepdim=True).clamp_min(1e-10) + targets.append(teacher) + qi = b.qi[start:end].float().detach().requires_grad_(True) + w = b.weights[start:end].float().detach().requires_grad_(True) + bi = r["block_ids"][start:end].long() + selected_kbar = kbar[bi.clamp_min(0)] + scores = (torch.einsum("qhd,qkd->qkh", qi, selected_kbar).relu() * w[:, None]).sum(-1) + valid_blocks = bi >= 0 + scores = scores.masked_fill(~valid_blocks, -torch.inf) + scores = torch.where(valid_blocks.any(-1, keepdim=True), scores, torch.zeros_like(scores)) + log_predict = scores.log_softmax(-1).masked_fill(~valid_blocks, 0) + loss = (teacher * (teacher.clamp_min(1e-10).log() - log_predict)).sum() / csa.total + loss.backward() + reference_loss += loss.item() + dqi.append(qi.grad) + dw.append(w.grad) + local_dkv = reduce_scatter_fp32(kv.grad, csa.source_n) + local_dkbar = reduce_scatter_fp32(kbar.grad, b.local_blocks) + pool_kv, pool_score, ape = [x.detach().float().requires_grad_(True) for x in (b.local_kv, b.pool_score, b.ape)] + weights = (pool_score.view(-1, 4, 128) + ape[None]).softmax(1) + pooled = (weights * pool_kv.view(-1, 4, 128)).sum(1) + pooled.backward(local_dkbar) + dist.all_reduce(ape.grad) + comparisons = { + "output": (r["source_output"], csa.exchange.restore(torch.cat(outputs))), + "dq": (r["source_dq"], csa.exchange.restore(torch.cat(dq))), + "dkv": (r["local_dkv"], local_dkv), + "teacher": (r["target"], torch.cat(targets)), + "d_index_q": (r["source_dqi"], csa.exchange.restore(torch.cat(dqi))), + "d_weights": (r["source_dw"], csa.exchange.restore(torch.cat(dw))), + "d_kbar": (r["local_dkbar"], local_dkbar), + "d_pool_kv": (b.local_kv.grad, pool_kv.grad), + "d_pool_score": (b.pool_score.grad, pool_score.grad), + "d_ape": (b.ape.grad, ape.grad), + } + errors = {key: metrics(actual, expected) for key, (actual, expected) in comparisons.items()} + # Fixed BF16 envelope, reported alongside raw errors; no threshold tuning. + passed = metadata_ok and all(x["finite"] and x["relative_rms"] < 0.03 for x in errors.values()) + loss_error = abs(r["loss"].item() - reference_loss) + passed &= loss_error < 1e-5 + return dict(pass_=passed, metadata_ok=metadata_ok, errors=errors, loss_abs_error=loss_error) + + +def check_redistribution(csa): + device = csa.source_q.device + source_rows = torch.arange(csa.plan.token_start, csa.plan.token_end, device=device) + source = (source_rows[:, None] * 3 + torch.arange(3, device=device)).float().requires_grad_(True) + ok = True + routes = [(csa.exchange, csa.layout)] + if csa.selection_exchange is not csa.exchange: + routes.append((csa.selection_exchange, csa.selection_layout)) + for exchange, layout in routes: + rows = ( + torch.cat([torch.arange(s.token_start, s.token_end, device=device) for s in layout.spans]) + if layout.spans + else torch.empty(0, device=device, dtype=torch.int64) + ) + expected = (rows[:, None] * 3 + torch.arange(3, device=device)).float() + source.grad = None + shuffled = exchange.distribute(source) + ok &= torch.equal(shuffled, expected) + shuffled.backward(expected * 2) + ok &= torch.equal(source.grad, source.detach() * 2) + leaf = shuffled.detach().requires_grad_(True) + restored = exchange.restore(leaf) + ok &= torch.equal(restored, source.detach()) + restored.backward(source.detach() * 3) + ok &= torch.equal(leaf.grad, expected * 3) + flag = torch.tensor(int(ok), device=device) + dist.all_reduce(flag, op=dist.ReduceOp.MIN) + if not flag.item(): + raise RuntimeError("row redistribution/inverse/autograd marker check failed") + return True + + +def compare_cpu_reference(actual, reference): + """Full-tensor error, streamed from lossless CPU storage; outside timing.""" + if actual.shape != reference.shape or actual.dtype != reference.dtype: + raise ValueError("baseline shape/dtype mismatch") + actual = actual.detach().reshape(-1) + reference = reference.reshape(-1) + stats = torch.zeros(3, device=actual.device, dtype=torch.float64) + finite = torch.ones((), device=actual.device, dtype=torch.bool) + for start in range(0, actual.numel(), 1 << 22): + a = actual[start : start + (1 << 22)].float() + b = reference[start : start + (1 << 22)].to(actual.device).float() + delta = a - b + stats[0] += delta.square().sum(dtype=torch.float64) + stats[1] += b.square().sum(dtype=torch.float64) + stats[2] = torch.maximum(stats[2], delta.abs().max().double()) + finite &= torch.isfinite(a).all() + values = stats.cpu().tolist() + return dict(relative_rms=math.sqrt(values[0] / max(values[1], 1e-24)), max_abs=values[2], finite=bool(finite)) + + +def fingerprint(tensor): + flat = tensor.detach().reshape(-1) + sums = torch.zeros(2, device=flat.device, dtype=torch.float64) + for start in range(0, flat.numel(), 1 << 22): + x = flat[start : start + (1 << 22)].float() + sums[0] += x.sum(dtype=torch.float64) + sums[1] += x.square().sum(dtype=torch.float64) + return sums.cpu().tolist() + + +def allocator_usage(): + return dict( + allocated_gib=torch.cuda.memory_allocated() / 2**30, + peak_allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + reserved_gib=torch.cuda.memory_reserved() / 2**30, + ) + + +def main(*, keep_group=False, baseline=None, save_baseline=False, comparison_name="layout_vs_contiguous"): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sequence-lengths", nargs="+", type=int, default=[4096]) + parser.add_argument("--heads", type=int, default=64) + parser.add_argument("--top-k", type=int, default=512) + parser.add_argument("--candidate-mib", type=int, default=0, help="0 uses cuLA's balanced scratch policy") + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--check", action="store_true") + parser.add_argument("--with-kda", action="store_true") + parser.add_argument("--kda-batch-invariant", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument( + "--kda-gate-scale", type=float, default=1.0, help="Scale negative KDA log-decay for sensitivity checks" + ) + parser.add_argument("--layout", choices=("contiguous", "zigzag", "planner"), default="contiguous") + parser.add_argument("--redistribution-scope", choices=("selection", "queries"), default="selection") + parser.add_argument("--planner-cost-model", help="JSON calibrated THDSelectionCostModel coefficients") + parser.add_argument("--planner-strategy", choices=("balanced", "selection"), default="balanced") + parser.add_argument("--selection-api", action="store_true", help="Use the public CSA CP selection API") + parser.add_argument("--memory-stages", action="store_true", help="Diagnostic allocator counters at each phase boundary") + parser.add_argument("--pool-backward-order", choices=("per-loss", "sum-first"), default="per-loss") + parser.add_argument("--output", required=True) + args = parser.parse_args() + if args.top_k <= 0 or args.top_k % 128 or args.warmups < 0 or args.repeats < 1: + parser.error("top-k must be positive and divisible by 128; warmups >= 0 and repeats >= 1") + if any(n <= 0 or n % 4 for n in args.sequence_lengths): + parser.error("this harness requires positive document lengths divisible by four") + if args.candidate_mib < 0: + parser.error("candidate-mib must be nonnegative") + if not math.isfinite(args.kda_gate_scale) or args.kda_gate_scale <= 0: + parser.error("kda-gate-scale must be finite and positive") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + if not dist.is_initialized(): + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=30)) + rank, world = dist.get_rank(), dist.get_world_size() + torch.manual_seed(20261112 + rank) + # Disable TF32 in the independent FP32 oracle. + torch.backends.cuda.matmul.allow_tf32 = False + cost_model = None + if args.planner_cost_model: + with open(args.planner_cost_model) as f: + parameters = json.load(f)["model"] + parameters["rank_node_ids"] = tuple(parameters["rank_node_ids"]) + cost_model = THDSelectionCostModel(**parameters) + csa = FullCSA( + args.sequence_lengths, + args.heads, + args.top_k, + device, + args.candidate_mib, + args.pool_backward_order, + args.layout, + args.redistribution_scope, + cost_model, + args.planner_strategy, + args.selection_api, + ) + csa.record_memory = args.memory_stages + redistribution_ok = check_redistribution(csa) + kda_prepare_ms = 0.0 + if args.with_kda: + plan = csa.plan + torch.cuda.synchronize(device) + kda_prepare_start = time.perf_counter() + transport = AllGatherKimiCPTransport.prepare(plan, heads=32, key_dim=128, value_dim=128, device=device) + local_cu = torch.tensor(plan.local_cu_seqlens, device=device, dtype=torch.int32) + torch.cuda.synchronize(device) + kda_prepare_ms = (time.perf_counter() - kda_prepare_start) * 1000 + q, k = [ + torch.nn.functional.normalize(torch.randn(csa.source_n, 32, 128, device=device), dim=-1).bfloat16() + for _ in range(2) + ] + v = torch.randn_like(q) + g = -0.01 - 0.04 * torch.rand(csa.source_n, 32, 128, device=device) + if args.kda_gate_scale != 1.0: + g *= args.kda_gate_scale + beta = torch.sigmoid(torch.randn(csa.source_n, 32, device=device)) + kda_inputs = [x.requires_grad_(True) for x in (q, k, v, g, beta)] + kda_do = torch.randn_like(v) + samples, check = [], None + for iteration in range(args.warmups + args.repeats): + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, kf, cb, end = [torch.cuda.Event(enable_timing=True) for _ in range(4)] + wall = time.perf_counter() + begin.record() + if args.with_kda: + for x in kda_inputs: + x.grad = None + with torch.cuda.nvtx.range("kda.forward"): + kda_out = cudnn_kda_cp( + *kda_inputs, + local_cu, + plan=plan, + group=dist.group.WORLD, + boundary_transport=transport, + batch_invariant=args.kda_batch_invariant, + ) + kf.record() + memory_after_kda_forward = allocator_usage() if args.memory_stages else None + csa.forward() + csa.backward() + cb.record() + if args.with_kda: + with torch.cuda.nvtx.range("kda.backward"): + kda_out.backward(kda_do) + end.record() + end.synchronize() + memory_after_kda_backward = allocator_usage() if args.memory_stages else None + sample = dict( + total_ms=begin.elapsed_time(end), + wall_ms=(time.perf_counter() - wall) * 1000, + kda_fwd_ms=begin.elapsed_time(kf), + kda_bwd_ms=cb.elapsed_time(end), + csa_ms=kf.elapsed_time(cb), + peak_allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + peak_reserved_gib=torch.cuda.max_memory_reserved() / 2**30, + **csa.timings(), + ) + if iteration >= args.warmups: + samples.append(sample) + if rank == 0: + print("ITER", iteration, json.dumps(json_safe(sample), allow_nan=False), flush=True) + if args.check and check is None: + check = check_reference(csa) + finite = all( + bool(torch.isfinite(x).all()) + for x in ( + csa.last["fw"]["out"], + csa.last["bw"]["dq"], + csa.last["local_dkv"], + csa.last["ibw"]["d_index_q"], + csa.last["ibw"]["d_weights"], + csa.last["local_dkbar"], + csa.last["target"], + csa.last["predict"], + csa.last["loss"], + csa.indexer.local_kv.grad, + csa.indexer.pool_score.grad, + csa.indexer.ape.grad, + ) + ) + if args.with_kda: + finite &= all(bool(torch.isfinite(x).all()) for x in [kda_out, *(x.grad for x in kda_inputs)]) + r, b = csa.last, csa.indexer + tensors = dict( + output=r["source_output"], + dq=r["source_dq"], + dkv=r["local_dkv"], + d_index_q=r["source_dqi"], + d_weights=r["source_dw"], + d_pool_kv=b.local_kv.grad, + d_pool_score=b.pool_score.grad, + d_ape=b.ape.grad, + ) + if args.with_kda: + tensors["kda_output"] = kda_out + tensors.update({"kda_" + key: x.grad for key, x in zip(("dq", "dk", "dv", "dg", "dbeta"), kda_inputs)}) + global_loss = r["loss"].detach().clone() + dist.all_reduce(global_loss) + errors = ( + None + if baseline is None + else {key: compare_cpu_reference(value, baseline["tensors"][key]) for key, value in tensors.items()} + ) + loss_delta = None if baseline is None else abs(global_loss.item() - baseline["global_loss"]) + # These operands verify identical source data across the three layouts. + inputs = dict( + q=csa.source_q, + kv=csa.kv, + qi=csa.source_qi, + weights=csa.source_weights, + pool_kv=b.local_kv, + pool_score=b.pool_score, + ape=b.ape, + dout=csa.source_dout, + ) + if args.with_kda: + inputs.update({"kda_" + key: x for key, x in zip(("q", "k", "v", "g", "beta"), kda_inputs)}) + inputs["kda_dout"] = kda_do + input_checksums = {key: fingerprint(value) for key, value in inputs.items()} + input_match = baseline is None or input_checksums == baseline["input_checksums"] + result = dict( + rank=rank, + samples=samples, + check=check, + finite=finite, + loss=global_loss.item(), + loss_abs_diff=loss_delta, + input_match=input_match, + redistribution_bitwise=redistribution_ok, + layout_prepare_ms=csa.layout_prepare_ms, + common_prepare_ms=csa.common_prepare_ms, + kda_prepare_ms=kda_prepare_ms, + total_prepare_ms=csa.common_prepare_ms + csa.layout_prepare_ms + kda_prepare_ms, + selected_policy=csa.selected_policy, + local_query_rows=csa.n, + spans=len(csa.layout.spans), + selection_spans=len(csa.selection_layout.spans), + selection_query_rows=csa.selection_layout.local_tokens, + remote_query_rows=csa.selection_exchange.plan.remote_rows, + selection_workspace_bytes=0 if b.select_plan is None else b.select_plan.candidate_workspace_floats * 4, + memory_stages=( + dict(kda_forward=memory_after_kda_forward, **csa.memory_stages, kda_backward=memory_after_kda_backward) + if args.memory_stages + else None + ), + ) + result[comparison_name] = errors + gathered = [None] * world + dist.all_gather_object(gathered, result) + if rank == 0: + critical = { + key: statistics.median(max(r["samples"][i][key] for r in gathered) for i in range(args.repeats)) + for key in samples[0] + } + report = dict(config=vars(args), world_size=world, critical_rank_median=critical, ranks=gathered) + report["planner_cost_model"] = None if cost_model is None else asdict(cost_model) + with open(args.output, "w") as f: + json.dump(json_safe(report), f, indent=2, allow_nan=False) + print("RESULT", json.dumps(json_safe(critical), allow_nan=False), flush=True) + okay = all( + r["finite"] + and r["input_match"] + and r["redistribution_bitwise"] + and (r["loss_abs_diff"] is None or 0 <= r["loss_abs_diff"] < 1e-5) + and (r["check"] is None or r["check"]["pass_"]) + and (r[comparison_name] is None or all(e["finite"] and e["relative_rms"] < 0.03 for e in r[comparison_name].values())) + for r in gathered + ) + saved = ( + None + if not save_baseline + else dict( + tensors={key: value.detach().cpu() for key, value in tensors.items()}, + global_loss=global_loss.item(), + input_checksums=input_checksums, + ) + ) + if not keep_group: + dist.destroy_process_group() + if not okay: + raise RuntimeError("Numerical check failed; raw per-rank errors were saved") + return saved + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_kda_independent_shards.py b/benchmarks/bench_kda_independent_shards.py new file mode 100644 index 00000000..02768e3b --- /dev/null +++ b/benchmarks/bench_kda_independent_shards.py @@ -0,0 +1,174 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Paired KDA check for globally document-local CP shards, including backward. + +The baseline conservatively forces the original boundary route. This compares +complete output/gradients on identical operands, retaining the baseline on CPU. +""" + +import argparse +import gc +import hashlib +import json +import os +import statistics +from dataclasses import replace +from itertools import accumulate +from pathlib import Path + +import torch +import torch.distributed as dist + +from benchmarks.bench_full_csa_cp import compare_cpu_reference, json_safe +from cula.distributed import AllGatherKimiCPTransport, KimiCPPlan, cudnn_kda_cp + + +def paired_samples(run, *, warmups, repeats): + """Warm both routes, then counterbalance each measured pair on all ranks.""" + if warmups < 0 or repeats < 1: + raise ValueError("warmups must be nonnegative and repeats positive") + samples = {"baseline": [], "candidate": []} + for warming, count in ((True, warmups), (False, repeats)): + for pair_index in range(count): + order = ("baseline", "candidate") if pair_index % 2 == 0 else ("candidate", "baseline") + for position, label in enumerate(order): + sample = run(label) + if not warming: + samples[label].append(dict(sample, pair_index=pair_index, execution_order=list(order), position=position)) + return samples + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=10) + args = parser.parse_args() + if args.warmups < 0 or args.repeats < 1: + parser.error("warmups must be nonnegative and repeats positive") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + torch.backends.cuda.matmul.allow_tf32 = False + device = torch.device("cuda", local_rank) + dist.init_process_group("nccl", device_id=device) + rank, world = dist.get_rank(), dist.get_world_size() + if world < 2 or (1 << 20) % world: + raise ValueError("require CP >= 2 dividing 1M") + n = (1 << 20) // world + cases = [(f"uniform-{length}", [length] * ((1 << 20) // length), False) for length in (1024, 4096, 16384)] + cases += [ + ("near-uniform", [15360, 17408] * 32, False), + ("weak-decay-uniform", [16384] * 64, True), + ("isolated-ranks-with-crossing-peer", [n] * (world - 2) + [2 * n], False), + ] + results = [] + for name, lengths, weak in cases: + plan = KimiCPPlan.from_global_cu_seqlens((0, *accumulate(lengths)), rank=rank, world_size=world) + transport = AllGatherKimiCPTransport.prepare(plan, heads=32, key_dim=128, value_dim=128, device=device) + cu = torch.tensor(plan.local_cu_seqlens, dtype=torch.int32, device=device) + torch.manual_seed(88171 + rank) + q, k = [torch.nn.functional.normalize(torch.randn(n, 32, 128, device=device), dim=-1).bfloat16() for _ in range(2)] + v = torch.randn_like(q) + g = ( + (-0.0001 - 0.0004 * torch.rand(n, 32, 128, device=device)) + if weak + else (-0.01 - 0.04 * torch.rand(n, 32, 128, device=device)) + ) + beta = torch.sigmoid(torch.randn(n, 32, device=device)) + inputs = [x.requires_grad_() for x in (q, k, v, g, beta)] + dout = torch.randn_like(v) + plans = {"baseline": replace(plan, requires_boundary_exchange=True), "candidate": plan} + + def timed_iteration(label, inputs=inputs, cu=cu, transport=transport, dout=dout): + for x in inputs: + x.grad = None + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + begin, forward, end = [torch.cuda.Event(enable_timing=True) for _ in range(3)] + begin.record() + out = cudnn_kda_cp(*inputs, cu, plan=plans[label], group=dist.group.WORLD, boundary_transport=transport) + forward.record() + out.backward(dout) + end.record() + end.synchronize() + sample = dict( + total_ms=begin.elapsed_time(end), + fwd_ms=begin.elapsed_time(forward), + bwd_ms=forward.elapsed_time(end), + allocated_gib=torch.cuda.max_memory_allocated() / 2**30, + ) + del out + return sample + + samples = paired_samples(timed_iteration, warmups=args.warmups, repeats=args.repeats) + del timed_iteration + # Run the numerical comparison separately, after both routes' timings. + # CPU copies and reference comparisons cannot perturb a measured pair. + reference = None + for label, active_plan in plans.items(): + for x in inputs: + x.grad = None + out = cudnn_kda_cp(*inputs, cu, plan=active_plan, group=dist.group.WORLD, boundary_transport=transport) + out.backward(dout) + tensors = dict(output=out, **{key: x.grad for key, x in zip(("dq", "dk", "dv", "dg", "dbeta"), inputs)}) + errors = {} if reference is None else {key: compare_cpu_reference(x, reference[key]) for key, x in tensors.items()} + if reference is None: + reference = {key: x.detach().cpu() for key, x in tensors.items()} + record = dict( + case=name, + label=label, + rank=rank, + requires_boundary_exchange=active_plan.requires_boundary_exchange, + intrinsic_requires_boundary_exchange=plan.requires_boundary_exchange, + samples=samples[label], + errors=errors, + ) + gathered = [None] * world + dist.all_gather_object(gathered, record) + passed = all(e["finite"] and e["max_abs"] == 0 for r in gathered for e in r["errors"].values()) + if rank == 0: + results.append( + dict( + case=name, + label=label, + ranks=gathered, + passed=passed, + total_ms=statistics.median( + max(r["samples"][i]["total_ms"] for r in gathered) for i in range(args.repeats) + ), + ) + ) + print(name, label, results[-1]["total_ms"], "exact", passed, flush=True) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps( + json_safe( + dict( + world_size=world, + timing_protocol="paired-alternating-v1", + source_sha256=hashlib.sha256( + (Path(__file__).resolve().parents[1] / "cula/distributed/kda.py").read_bytes() + ).hexdigest(), + results=results, + ) + ), + indent=2, + allow_nan=False, + ) + + "\n" + ) + if not passed: + raise RuntimeError("independent-shard path changed output or gradients; see saved errors") + del out, tensors + for x in inputs: + x.grad = None + del reference, inputs, q, k, v, g, beta, dout, transport, cu + gc.collect() + torch.cuda.empty_cache() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/bench_thd_transport.py b/benchmarks/bench_thd_transport.py new file mode 100644 index 00000000..9f069cf3 --- /dev/null +++ b/benchmarks/bench_thd_transport.py @@ -0,0 +1,204 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Calibrate selection transport independently of KDA/selection arrival skew. + +The caller supplies process-group topology. This uses the actual eager +redistributor and Q/weights/IDs/scores dtypes, with no attention kernels. +""" + +import argparse +import datetime +import gc +import json +import os +import statistics +from pathlib import Path + +import torch +import torch.distributed as dist + +from cula.distributed import THDRankLayout, THDRedistributionPlan, THDRedistributor, THDSpan + + +def transport_layout(world, rows, ranks_per_node, kind): + if world < 2 or rows < world or rows % world or rows % 4 or ranks_per_node < 1 or world % ranks_per_node: + raise ValueError("need complete nodes and row counts divisible by CP") + assignments = [[] for _ in range(world)] + for source in range(world): + if kind in ("mixed_dense_quarter", "mixed_dense_three_quarters"): + if ranks_per_node == world or rows % (4 * ranks_per_node): + raise ValueError("mixed validation needs multiple nodes and quarter-node aligned rows") + local_start = source // ranks_per_node * ranks_per_node + remote_start = (local_start + ranks_per_node) % world + remote_rows = rows // 4 * (1 if kind == "mixed_dense_quarter" else 3) + cursor = source * rows + for node_start, count in ((local_start, rows - remote_rows), (remote_start, remote_rows)): + for rank in range(node_start, node_start + ranks_per_node): + end = cursor + count // ranks_per_node + assignments[rank].append(THDSpan(0, cursor, end, 0, world * rows, rank)) + cursor = end + continue + if kind in ("intra_pair", "inter_pair"): + peer = 1 if kind == "intra_pair" else ranks_per_node + if peer >= world or (kind == "intra_pair" and ranks_per_node < 2): + raise ValueError("pair fixture needs two ranks on the declared link") + destinations = [peer if source == 0 else 0 if source == peer else source] + elif kind in ("intra", "intra_dense"): + node_start = source // ranks_per_node * ranks_per_node + destinations = ( + list(range(node_start, node_start + ranks_per_node)) + if kind == "intra_dense" + else [node_start + (source + 1) % ranks_per_node] + ) + if ranks_per_node == 1: + raise ValueError("intra-node calibration needs at least two ranks per node") + elif kind in ("inter", "inter_dense"): + if ranks_per_node == world: + raise ValueError("inter-node calibration needs at least two nodes") + next_node = ((source // ranks_per_node + 1) * ranks_per_node) % world + destinations = ( + list(range(next_node, next_node + ranks_per_node)) + if kind == "inter_dense" + else [(source + ranks_per_node) % world] + ) + elif kind == "striped": + destinations = list(range(world)) + else: + raise ValueError(f"unknown transport fixture: {kind}") + for i, rank in enumerate(destinations): + start = source * rows + i * rows // len(destinations) + end = source * rows + (i + 1) * rows // len(destinations) + assignments[rank].append(THDSpan(0, start, end, 0, world * rows, rank)) + return tuple(THDRankLayout(rank, tuple(spans)) for rank, spans in enumerate(assignments)) + + +def ordered(ranges): + cursor = 0 + for a, z in ranges: + if a != cursor: + return False + cursor += z - a + return True + + +def features(layouts, rows, rpn): + plans = [ + THDRedistributionPlan.build(layouts, tuple(i * rows for i in range(len(layouts) + 1)), r) for r in range(len(layouts)) + ] + edges = {(r, t): n for r, p in enumerate(plans) for t, n in enumerate(p.send_counts) if r != t and n} + intra_out = [0] * len(plans) + inter_out = intra_out.copy() + intra_in = intra_out.copy() + inter_in = intra_out.copy() + node_out = [0] * (len(plans) // rpn) + node_in = node_out.copy() + degree_out = [0] * len(plans) + degree_in = degree_out.copy() + for (s, t), n in edges.items(): + a, b = s // rpn, t // rpn + degree_out[s] += 1 + degree_in[t] += 1 + if a == b: + intra_out[s] += n + intra_in[t] += n + else: + inter_out[s] += n + inter_in[t] += n + node_out[a] += n + node_in[b] += n + return dict( + rank_intra_rows=intra_out + intra_in, + rank_inter_rows=inter_out + inter_in, + peer_rows=max(edges.values(), default=0), + node_rows=max(node_out + node_in), + peer_degree=max(degree_out + degree_in), + copies=max( + (0 if ordered(p.send_ranges) else p.source_rows) + (0 if ordered(p.receive_ranges) else p.target_rows) + for p in plans + ), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--validation-only", action="store_true") + parser.add_argument("--ranks-per-node", type=int, required=True) + parser.add_argument("--rows-per-rank", nargs="+", type=int, default=[256, 4096, 32768, 65536]) + parser.add_argument("--warmups", type=int, default=3) + parser.add_argument("--repeats", type=int, default=10) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.ranks_per_node < 1 or args.warmups < 0 or args.repeats < 1: + parser.error("require positive ranks-per-node/repeats and nonnegative warmups") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=30)) + world, rank = dist.get_world_size(), dist.get_rank() + if world < 2 or world % args.ranks_per_node: + raise ValueError("require CP >= 2 and complete nodes") + kinds = ["intra_pair", "intra", "intra_dense"] if args.ranks_per_node > 1 else [] + kinds += (["inter_pair", "inter", "inter_dense"] if world > args.ranks_per_node else []) + ["striped"] + sizes = args.rows_per_rank + if args.validation_only: + sizes, kinds = [8192, 49152], ["mixed_dense_quarter", "mixed_dense_three_quarters"] + # Validate every fixture before starting any timed GPU work. + for rows in sizes: + for kind in kinds: + transport_layout(world, rows, args.ranks_per_node, kind) + results = [] + for rows in sizes: + for kind in kinds: + layouts = transport_layout(world, rows, args.ranks_per_node, kind) + plan = THDRedistributionPlan.build(layouts, tuple(r * rows for r in range(world + 1)), rank) + exchange = THDRedistributor(plan, device, group=dist.group.WORLD) + marker = torch.arange(rank * rows, (rank + 1) * rows, device=device) + routed = exchange.distribute(marker) + restored = exchange.restore(routed) + expected = torch.cat([torch.arange(s.token_start, s.token_end, device=device) for s in layouts[rank].spans]) + okay = torch.tensor(int(torch.equal(marker, restored) and torch.equal(routed, expected)), device=device) + dist.all_reduce(okay, op=dist.ReduceOp.MIN) + if not okay.item(): + raise RuntimeError("transport calibration marker mismatch") + # H64/D128 BF16 query, BF16 weights; Top-K512 int32 IDs and FP32 scores. + source = [torch.zeros(rows, *shape, device=device, dtype=torch.bfloat16) for shape in [(64, 128), (64,)]] + target = [torch.zeros(rows, 512, device=device, dtype=dtype) for dtype in (torch.int32, torch.float32)] + samples = [] + for step in range(args.warmups + args.repeats): + dist.barrier(device_ids=[local_rank]) + torch.cuda.synchronize() + begin, end = [torch.cuda.Event(enable_timing=True) for _ in range(2)] + begin.record() + outputs = [exchange.distribute(x) for x in source] + [exchange.restore(x) for x in target] + end.record() + end.synchronize() + if step >= args.warmups: + samples.append(begin.elapsed_time(end)) + del outputs + gathered = [None] * world + dist.all_gather_object(gathered, samples) + record = dict( + kind=kind, + rows=rows, + features=features(layouts, rows, args.ranks_per_node), + samples_ms=gathered, + critical_median_ms=statistics.median(max(s[i] for s in gathered) for i in range(args.repeats)), + marker_exact=True, + heldout=args.validation_only or rows == 32768 or kind == "striped", + ) + results.append(record) + if rank == 0: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(dict(world=world, ranks_per_node=args.ranks_per_node, records=results), indent=2) + "\n" + ) + print(rows, kind, record["critical_median_ms"], flush=True) + del source, target, marker, routed, expected, restored, exchange + gc.collect() + torch.cuda.empty_cache() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/calibrate_thd_selection.py b/benchmarks/calibrate_thd_selection.py new file mode 100644 index 00000000..12e4c272 --- /dev/null +++ b/benchmarks/calibrate_thd_selection.py @@ -0,0 +1,188 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Fit explicit CPU planner coefficients from saved selection-routing runs. + +This does not run GPU work or tune kernels. Use separate, held-out THD inputs +to evaluate the resulting policy; calibration fit alone is not validation. +""" + +import argparse +import hashlib +import json +import math +import statistics +from itertools import accumulate, combinations +from pathlib import Path + +from benchmarks.cp_layouts import query_layouts +from cula.distributed.thd_planner import _candidate_work + + +def _solve(matrix, rhs): + rows = [list(row) + [value] for row, value in zip(matrix, rhs)] + for col in range(len(rhs)): + pivot = max(range(col, len(rhs)), key=lambda r: abs(rows[r][col])) + if abs(rows[pivot][col]) < 1e-14: + return None + rows[col], rows[pivot] = rows[pivot], rows[col] + scale = rows[col][col] + rows[col] = [x / scale for x in rows[col]] + for r in range(len(rhs)): + if r != col: + scale = rows[r][col] + rows[r] = [a - scale * b for a, b in zip(rows[r], rows[col])] + return [row[-1] for row in rows] + + +def nonnegative_fit(features, targets): + """Exact active-set enumeration for three nonnegative cost coefficients.""" + if not features or len(features) != len(targets) or any(len(row) != 3 for row in features): + raise ValueError("need matched, nonempty measurements with three features") + if any(not math.isfinite(v) or v < 0 for row in features for v in row) or any(not math.isfinite(v) for v in targets): + raise ValueError("fit inputs must be finite and feature counts nonnegative") + best, error = [0.0] * 3, sum(y * y for y in targets) + for size in range(1, 4): + for active in combinations(range(3), size): + matrix = [[sum(x[i] * x[j] for x in features) for j in active] for i in active] + rhs = [sum(x[i] * y for x, y in zip(features, targets)) for i in active] + solution = _solve(matrix, rhs) + if solution is None or any(x < 0 for x in solution): + continue + weights = [0.0] * 3 + for i, value in zip(active, solution): + weights[i] = value + loss = sum((sum(a * b for a, b in zip(x, weights)) - y) ** 2 for x, y in zip(features, targets)) + if loss < error: + best, error = weights, loss + return best, math.sqrt(error / len(targets)) + + +def routing_critical_ms(samples): + """Critical-path increments, excluding pre-existing rank arrival skew. + + Each sample starts at the iteration's post-barrier event. At every phase, + max(cumulative end) - max(cumulative start) is the phase's contribution to + the distributed makespan. Summing local exchange durations instead would + count waiting for slow KDA/selection ranks again as network time. + """ + phases = ( + "query_redistribute_ms", + "pool_fwd_ms", + "pool_gather_ms", + "selection_key_layout_ms", + "selection_ms", + "selection_restore_ms", + ) + if not samples or any(not math.isfinite(s[k]) or s[k] < 0 for s in samples for k in ("kda_fwd_ms", *phases)): + raise ValueError("routing phases must be finite, nonnegative and nonempty") + arrival = [s["kda_fwd_ms"] for s in samples] + duration = 0.0 + for phase in phases: + before = max(arrival) + arrival = [a + s[phase] for a, s in zip(arrival, samples)] + if phase in ("query_redistribute_ms", "selection_key_layout_ms", "selection_restore_ms"): + duration += max(arrival) - before + return duration + + +def calibrate(paths, ranks_per_node, margin=0.05): + features, targets, rates, sources, digests = [], [], [], [], {} + world = None + # Current benchmark: BF16 H64/D128 Q, BF16 H64 weights, + # int32 IDs + FP32 scores for Top-K 512. + payload = 2 * 64 * 128 + 2 * 64 + (4 + 4) * 512 + for path in paths: + raw = json.loads(path.read_text()) + config = raw["config"] + if config["check"]: + continue + if config.get("redistribution_scope") != "selection" or raw.get("planner_cost_model") is not None: + raise ValueError("calibrate from unmodeled selection-only runs, not query-wide or already calibrated runs") + if config["top_k"] != 512: + raise ValueError("this benchmark profile requires indexer H64/D128 and Top-K 512") + if world is not None and world != raw["world_size"]: + raise ValueError("cannot combine different CP group sizes") + world = raw["world_size"] + bounds = (0, *accumulate(config["sequence_lengths"])) + ranks, selected = query_layouts(bounds, world, config["layout"]) + observed = sorted(raw["ranks"], key=lambda x: x["rank"]) + if [r["rank"] for r in observed] != list(range(world)): + raise ValueError("calibration requires exactly one result per process-group rank") + if any(not all(r[k] for k in ("finite", "input_match", "redistribution_bitwise")) for r in observed): + raise ValueError("cannot calibrate from a failed numerical/redistribution check") + if any( + not e["finite"] or not math.isfinite(e["relative_rms"]) or e["relative_rms"] >= 0.03 + for r in observed + for e in (r["layout_vs_contiguous"] or {}).values() + ): + raise ValueError("cannot calibrate from failed output/gradient comparisons") + if any(len(r["samples"]) != config["repeats"] for r in observed): + raise ValueError("incomplete timing samples") + if selected != observed[0]["selected_policy"]: + raise ValueError("source planner differs from the measured assignment") + for layout, result in zip(ranks, observed): + candidates = sum(_candidate_work(s.causal_offset, s.tokens, 4) for s in layout.spans) + features.append([candidates / 1e9, layout.local_tokens / 1e6, len(layout.spans) / 1000]) + targets.append(statistics.median(s["selection_ms"] for s in result["samples"])) + repeats = len(observed[0]["samples"]) + duration = statistics.median(routing_critical_ms([r["samples"][i] for r in observed]) for i in range(repeats)) + remote_rows = max(r["remote_query_rows"] for r in observed) + if remote_rows: + if not math.isfinite(duration) or duration <= 0: + raise ValueError("routing timings must be finite and positive") + rates.append(remote_rows * payload / (duration * 1e6)) + sources.append(str(path)) + digests[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest() + if not features or not rates or world is None: + raise ValueError("need measured selection work and at least one nonidentity routing case") + if ranks_per_node < 1 or world % ranks_per_node: + raise ValueError("ranks_per_node must divide the measured group size") + coefficients, rmse = nonnegative_fit(features, targets) + # The measurements do not isolate NVLink vs inter-node links. Use one + # conservative effective rate, including packing, for both. Cumulative + # critical increments remove pre-existing arrival skew from the fit. + # Do not present it as a physical-link bandwidth measurement. + rate = min(rates) + return dict( + schema_version=2, + model=dict( + candidate_ns=coefficients[0] / 1000, + query_ns=coefficients[1], + span_us=coefficients[2], + bytes_per_query=payload, + intra_node_gbps=rate, + inter_node_gbps=rate, + rank_node_ids=[r // ranks_per_node for r in range(world)], + packing_ns_per_query=0.0, + exchange_us=0.0, + min_gain_fraction=margin, + ), + calibration=dict( + sources=sources, + source_sha256=digests, + samples=len(targets), + selection_fit_rmse_ms=rmse, + observed_routing_effective_gbps=rates, + transport_note="Critical phase increments remove pre-existing rank arrival skew. Shared conservative effective rate includes packing; links were not isolated.", + validation_note="Evaluate held-out layouts and seeds; fitting is not a performance guarantee.", + ), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--records", type=Path, nargs="+", required=True) + parser.add_argument("--ranks-per-node", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + paths = sorted({p for root in args.records for p in root.glob("layout-1m-*.json")}) + result = calibrate(paths, args.ranks_per_node) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["model"], indent=2)) + print("selection fit RMSE ms", result["calibration"]["selection_fit_rmse_ms"]) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/check_csa_cp_selection.py b/benchmarks/check_csa_cp_selection.py new file mode 100644 index 00000000..7cb386fb --- /dev/null +++ b/benchmarks/check_csa_cp_selection.py @@ -0,0 +1,98 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Native distributed checks of source order, subgroups and retained results.""" + +import argparse +import datetime +import json +import math +import os +from itertools import accumulate +from pathlib import Path + +import torch +import torch.distributed as dist + +from cula.hybrid import CsaCPSelectionPlan, CsaCPSelectionWorkspace, cudnn_csa_select_cp + + +def canonical(result): + ids, order = result.block_ids.sort(-1) + scores = result.scores.gather(1, order).masked_fill(ids < 0, 0) + return ids, scores + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--weights-dtype", choices=("bf16", "fp32"), default="bf16") + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + device = torch.device("cuda", torch.cuda.current_device()) + dist.init_process_group("nccl", timeout=datetime.timedelta(minutes=5)) + world = dist.get_world_size() + if world < 4 or world % 2: + raise ValueError("launch an even number of >= 4 ranks") + groups = [dist.new_group(list(range(parity, world, 2))) for parity in (0, 1)] + group = groups[dist.get_rank() % 2] + rank, size = dist.get_rank(group), dist.get_world_size(group) + records = [] + # Aligned but unequal source lengths, document-crossing shards, and empty + # zigzag destinations. All policies must restore the same native result. + for lengths in ([4] * (2 * size), [4, 4 * (size + 1)], [4096], [4] * 20 + [984] * 2): + cu = (0, *accumulate(lengths)) + plan = CsaCPSelectionPlan.from_cu_seqlens(cu, rank=rank, world_size=size, top_k=512, policy="contiguous") + torch.manual_seed(20261116 + dist.get_rank()) + n = plan.exchange.source_rows + qi = torch.randn(n, 64, 128, dtype=torch.bfloat16, device=device) + dtype = torch.bfloat16 if args.weights_dtype == "bf16" else torch.float32 + weights = torch.randn(n, 64, dtype=dtype, device=device) / 64 + # Distinct replicated key sets per subgroup catch use of WORLD. + torch.manual_seed(713 + dist.get_rank() % 2) + keys = torch.randn(cu[-1] // 4, 1, 128, dtype=torch.bfloat16, device=device) + baseline = None + for policy in ("contiguous", "zigzag"): + plan = CsaCPSelectionPlan.from_cu_seqlens( + cu, rank=rank, world_size=size, top_k=512, policy=policy, max_tokens_per_rank=cu[-1] + ) + workspace = CsaCPSelectionWorkspace.from_plan(plan, device, group=group) + result = cudnn_csa_select_cp(qi, keys, weights, workspace=workspace) + ids, scores = canonical(result) + saved_ids, saved_scores = result.block_ids.clone(), result.scores.clone() + cudnn_csa_select_cp(qi * 2, keys, weights, workspace=workspace) + retained = torch.equal(saved_ids, result.block_ids) and torch.equal(saved_scores, result.scores) + if baseline is None: + baseline = ids.clone(), scores.clone() + ids_equal = torch.equal(ids, baseline[0]) + delta = (scores - baseline[1]).float() + relative = (delta.norm() / baseline[1].norm().clamp_min(1e-12)).item() + okay = retained and ids_equal and bool(torch.isfinite(scores).all()) and relative < 1e-5 + records.append( + dict( + lengths=lengths, + policy=policy, + weights_dtype=args.weights_dtype, + source_rows=n, + target_rows=plan.exchange.target_rows, + retained=retained, + ids_equal=ids_equal, + score_relative_rms=relative if math.isfinite(relative) else None, + okay=okay, + ) + ) + all_records = [None] * world + dist.all_gather_object(all_records, records) + if dist.get_rank() == 0: + out = Path(os.environ.get("CSA_CHECK_OUTPUT", "csa-cp-checks.json")) + out.write_text(json.dumps(all_records, indent=2, allow_nan=False) + "\n") + print("CHECKS", sum(len(r) for r in all_records), "PASS", all(x["okay"] for r in all_records for x in r), flush=True) + if not all(x["okay"] for r in all_records for x in r): + raise RuntimeError("CSA CP selection qualification failed") + for g in groups: + if g != dist.GroupMember.NON_GROUP_MEMBER: + dist.destroy_process_group(g) + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cp_layouts.py b/benchmarks/cp_layouts.py new file mode 100644 index 00000000..5370d052 --- /dev/null +++ b/benchmarks/cp_layouts.py @@ -0,0 +1,37 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Comparable contiguous, per-document zigzag and adaptive THD layouts.""" + +from cula._metadata import balanced_shards +from cula.distributed import THDRankLayout, THDSpan, plan_thd_layout, plan_thd_selection, zigzag_thd_layout + + +def query_layouts(boundaries, world, policy, *, cost_model=None, planner_strategy="balanced"): + if policy == "planner": + if planner_strategy == "selection": + if cost_model is None: + raise ValueError("selection planning requires calibrated costs") + source_rows = 4 * ((boundaries[-1] // 4 + world - 1) // world) + plan = plan_thd_selection(boundaries, world_size=world, cost_model=cost_model, max_tokens_per_rank=2 * source_rows) + elif planner_strategy == "balanced": + plan = plan_thd_layout(boundaries, world_size=world, cost_model=cost_model) + else: + raise ValueError(f"unknown planner strategy: {planner_strategy}") + return plan.ranks, plan.policy + assignments = [[] for _ in range(world)] + if policy == "contiguous": + offsets = tuple(n * 4 for n in balanced_shards(boundaries[-1] // 4, world)) + for rank, (left, right) in enumerate(zip(offsets, offsets[1:])): + for seq, (start, end) in enumerate(zip(boundaries, boundaries[1:])): + a, z = max(left, start), min(right, end) + if a < z: + assignments[rank].append(THDSpan(seq, a, z, start, end, rank)) + elif policy == "zigzag": + return zigzag_thd_layout(boundaries, world_size=world), policy + else: + raise ValueError(f"unknown layout: {policy}") + ranks = tuple( + THDRankLayout(rank, tuple(sorted(spans, key=lambda s: s.token_start))) for rank, spans in enumerate(assignments) + ) + return ranks, policy diff --git a/benchmarks/fit_thd_transport.py b/benchmarks/fit_thd_transport.py new file mode 100644 index 00000000..ce2f9ea2 --- /dev/null +++ b/benchmarks/fit_thd_transport.py @@ -0,0 +1,128 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Fit nonnegative effective transport costs from independent routing sweeps. + +SciPy is a calibration-only dependency. Production planning does not import it. +Held-out fixtures are reported, never included in the optimization objective. +""" + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np + + +def predict(features, coefficients, overlap_links=False): + """Same ordered-phase resource model used by THDSelectionCostModel.""" + intra, inter, peer, node, packing, startup, fixed = coefficients + unit = 65536 + combine = max if overlap_links else lambda a, b: a + b + rank = max(combine(a * intra, b * inter) for a, b in zip(features["rank_intra_rows"], features["rank_inter_rows"])) / unit + transfer = max(rank, features["peer_rows"] * peer / unit, features["node_rows"] * node / unit) + return transfer + features["copies"] * packing / unit + features["peer_degree"] * startup + fixed + + +def fit(raw, profile, *, overlap_links=False): + from scipy.optimize import differential_evolution, least_squares + + world, per_node = raw["world"], raw["ranks_per_node"] + if ( + isinstance(world, bool) + or not isinstance(world, int) + or isinstance(per_node, bool) + or not isinstance(per_node, int) + or world < 2 + or per_node < 1 + or world % per_node + ): + raise ValueError("calibration topology requires CP >= 2 and complete nodes") + rows = [r for r in raw["records"] if not r["heldout"]] + if len(rows) < 6 or any( + not r["marker_exact"] or not np.isfinite(r["critical_median_ms"]) or r["critical_median_ms"] <= 0 + for r in raw["records"] + ): + raise ValueError("need complete, finite positive transport timings and exact row markers") + y = np.array([r["critical_median_ms"] for r in rows]) + + def residual(x): + return (np.array([predict(r["features"], x, overlap_links) for r in rows]) - y) / np.maximum(y, 0.25) + + # Inactive max-resource branches have zero derivative. A bounded global + # initialization avoids silently accepting a rank-bandwidth coefficient + # that never became active during local least-squares optimization. + scale = max(y) + startup_limit = min(r["critical_median_ms"] / max(r["features"]["peer_degree"], 1) for r in rows) + bounds = [(0, 2 * scale)] * 5 + [(0, startup_limit), (0, min(y))] + initialized = differential_evolution( + lambda x: np.sum(residual(x) ** 2), bounds, seed=1, popsize=10, maxiter=300, tol=1e-8, polish=False + ) + best = least_squares(residual, initialized.x, bounds=(0, np.inf), max_nfev=4000, ftol=1e-10, xtol=1e-10, gtol=1e-10) + x = best.x + # Ordered sweep fixtures elide every permutation. Their timings cannot + # identify copy bandwidth; use a separately measured packing coefficient. + if not any(r["features"]["copies"] for r in rows): + x[4] = profile["model"].get("packing_ns_per_query", 0) * 65536 / 1e6 + # A single-node calibration cannot identify inter-node bandwidth. + # Use the same unused fallback rate, not an arbitrary optimizer parameter. + # The sweep assigns contiguous group ranks to nodes. Export that same + # topology rather than retaining the input profile's deployment mapping. + model = dict( + profile["model"], + overlap_intra_inter=overlap_links, + rank_node_ids=[rank // per_node for rank in range(world)], + ) + factor = 65536 * model["bytes_per_query"] / 1e6 + model.update( + intra_node_gbps=factor / max(x[0], 1e-10), + inter_node_gbps=factor / max(x[1] if raw["world"] > raw["ranks_per_node"] else x[0], 1e-10), + peer_gbps=factor / max(x[2], 1e-10), + inter_node_aggregate_gbps=factor / max(x[3], 1e-10) if raw["world"] > raw["ranks_per_node"] else None, + packing_ns_per_query=x[4] * 1e6 / 65536, + peer_startup_us=x[5] * 1000, + exchange_us=x[6] * 1000, + ) + predictions = [ + dict( + kind=r["kind"], + rows=r["rows"], + heldout=r["heldout"], + actual_ms=r["critical_median_ms"], + predicted_ms=predict(r["features"], x, overlap_links), + relative_error=predict(r["features"], x, overlap_links) / r["critical_median_ms"] - 1, + ) + for r in raw["records"] + ] + return dict( + profile, + schema_version=5, + model=model, + transport_calibration=dict( + method="Nonnegative max-resource fit; independent eager transport timings", + coefficients_ms_per_65536_rows=x.tolist(), + predictions=predictions, + ranks_per_node=raw["ranks_per_node"], + world_size=raw["world"], + heldout_rule="32768 rows or striped layout; never used in fitting", + note="Effective coefficients include eager implementation costs; not physical bandwidth or universal optimality guarantees.", + ), + ) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--records", type=Path, required=True) + p.add_argument("--profile", type=Path, required=True) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--overlap-links", action="store_true") + a = p.parse_args() + result = fit(json.loads(a.records.read_text()), json.loads(a.profile.read_text()), overlap_links=a.overlap_links) + result["transport_calibration"]["source_sha256"] = hashlib.sha256(a.records.read_bytes()).hexdigest() + a.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["model"], indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/profiles/README.md b/benchmarks/profiles/README.md new file mode 100644 index 00000000..0a33d33a --- /dev/null +++ b/benchmarks/profiles/README.md @@ -0,0 +1,23 @@ +# THD selection profile examples + +These are explicit profiles from the 2026-11-15 real-CP experiment, not library +defaults or physical-link bandwidth specifications. They apply to indexer +H64/D128, Top-K512, BF16 query/weights and int32 IDs/FP32 scores, at 1M total +tokens with eager PyTorch/NCCL variable-split all-to-all. + +- `thd_selection_h20_cp16.json`: two nodes, eight H20-3e ranks per node. +- `thd_selection_b300_cp8.json`: one node, eight B300 ranks. The inter-node + rate is unused; it is not a cross-node Blackwell measurement. + +The profiles retain compute-fit provenance and single-peer/dense/striped +transport samples. `min_gain_ms` is a conservative **single-use preparation +budget**, 40 ms and 20 ms respectively, not a measured kernel cost. A caller +reusing a plan can explicitly amortize it; zero keeps throughput-only planning. +No plan is selected by a case name, document-count lookup, or hardware detector. + +Pass one explicitly to `bench_csa_layout_matrix.py --planner-cost-model` with +`--planner-strategy selection`. Recalibrate with `calibrate_thd_selection.py` +and the independent `bench_thd_transport.py` / `fit_thd_transport.py` workflow for other indexer shapes, process-group topology, +message sizes or NCCL/software configuration. See +[the experiment report](../../docs/THD_SELECTION_RESULTS.md) for measured +latency, precision, memory and limitations. diff --git a/benchmarks/profiles/thd_selection_b300_cp8.json b/benchmarks/profiles/thd_selection_b300_cp8.json new file mode 100644 index 00000000..bfc73640 --- /dev/null +++ b/benchmarks/profiles/thd_selection_b300_cp8.json @@ -0,0 +1,646 @@ +{ + "schema_version": 5, + "model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "min_gain_ms": 20, + "overlap_intra_inter": false, + "peer_gbps": 408.72149682964135, + "inter_node_aggregate_gbps": null, + "peer_startup_us": 7.460149512894069e-21 + }, + "calibration": { + "sources": [ + "layout-1m-20s2l-contiguous.json", + "layout-1m-20s2l-zigzag.json", + "layout-1m-count-mixed-256-1long-contiguous.json", + "layout-1m-count-mixed-256-1long-zigzag.json", + "layout-1m-count-mixed-512-2long-contiguous.json", + "layout-1m-count-mixed-512-2long-zigzag.json", + "layout-1m-count-tiny-mix-512-contiguous.json", + "layout-1m-count-tiny-mix-512-zigzag.json", + "layout-1m-count-uniform-256-contiguous.json", + "layout-1m-count-uniform-256-zigzag.json", + "layout-1m-single-contiguous.json", + "layout-1m-single-zigzag.json", + "layout-1m-uniform-contiguous.json", + "layout-1m-uniform-zigzag.json" + ], + "source_sha256": { + "layout-1m-20s2l-contiguous.json": "31c62725b5ff3567bd916db656b186051ca80f30c37f0cb3a1f50837311f14e9", + "layout-1m-20s2l-zigzag.json": "7e1fff2be4be88a44ed5670becc04946bdaec464ee4a9ddcb5a501976d0de389", + "layout-1m-count-mixed-256-1long-contiguous.json": "54a36612d35922e9a2d8032b4a641a0afc3484191ce9d3fa19d9328c3efd22e2", + "layout-1m-count-mixed-256-1long-zigzag.json": "73c6f8f9e70a9c1e6896ea4ca412f8fe467ff3525fc757329c8ee9520f39c377", + "layout-1m-count-mixed-512-2long-contiguous.json": "f8786cc6ff782bf0b48bcf3d1acfac5de0d328216e0790d29852f139ddbaaf07", + "layout-1m-count-mixed-512-2long-zigzag.json": "3d08c30f9b05728c06c3564ea7b88f27a7bd523b0de17f1df612818504d702a4", + "layout-1m-count-tiny-mix-512-contiguous.json": "644d77f672d4a16a7cc27a8cb68e0bd9f6a55221d5d6b533b22b9514eccb6473", + "layout-1m-count-tiny-mix-512-zigzag.json": "5bf51c02bb8a083e8e19614aef4720efc3cfbc413ac3780fdba924c9b8849695", + "layout-1m-count-uniform-256-contiguous.json": "dfda88005564f268e7140f5e42c60fc8a7a670402cba201a811b7f093e084b6b", + "layout-1m-count-uniform-256-zigzag.json": "2450fdc87d83207dd66e333bffbbb3f89123b18afbfdac6b6d0bfa3f649c91af", + "layout-1m-single-contiguous.json": "ed4598764f4760f8467d40e9a351cc5586129a3a23180fb0a35bf71aedb10d6f", + "layout-1m-single-zigzag.json": "b6ab3b9f7cf1a89dd1ebb1d76980781276d728e559e8e8b7ee23f15de557c112", + "layout-1m-uniform-contiguous.json": "da9f168ffb4109bcc6bab5e6f7112d9ea99741406fcbcacd9fdb57f84952cfca", + "layout-1m-uniform-zigzag.json": "2ce0a59edde0db530c63d5c20a09bccaa41a9865b109ad5e10bda46a0d43f246" + }, + "samples": 112, + "selection_fit_rmse_ms": 4.720245923945194, + "observed_routing_effective_gbps": [ + 305.18791099916496, + 331.06902736714636, + 390.5160264901337, + 387.31209063170303, + 442.2725855454942, + 330.9084931142364, + 441.57185423427615 + ], + "transport_note": "Critical phase increments remove pre-existing rank arrival skew. Shared conservative effective rate includes packing; links were not isolated.", + "validation_note": "Evaluate held-out layouts and seeds; fitting is not a performance guarantee." + }, + "packing_calibration": { + "world": 8, + "records": [ + { + "rows": 256, + "critical_median_ms": 0.043455999344587326, + "ns_per_query": 169.74999743979424, + "samples_ms": [ + [ + 0.04495999962091446, + 0.04396799951791763, + 0.04649600014090538, + 0.04335999861359596, + 0.04291199892759323, + 0.04259200021624565, + 0.04383999854326248, + 0.04214400053024292, + 0.043327998369932175, + 0.0424639992415905 + ], + [ + 0.04476799815893173, + 0.0435199998319149, + 0.048895999789237976, + 0.042208001017570496, + 0.042047999799251556, + 0.04230400174856186, + 0.042047999799251556, + 0.04211200028657913, + 0.04320000112056732, + 0.042047999799251556 + ], + [ + 0.04095999896526337, + 0.03951999917626381, + 0.040383998304605484, + 0.03872000053524971, + 0.038495998829603195, + 0.038047999143600464, + 0.03923200070858002, + 0.039103999733924866, + 0.03836800158023834, + 0.03961599990725517 + ], + [ + 0.04243199899792671, + 0.04185599833726883, + 0.04134399816393852, + 0.03903999924659729, + 0.03766399994492531, + 0.03961599990725517, + 0.04838399961590767, + 0.038656000047922134, + 0.039103999733924866, + 0.03852799907326698 + ], + [ + 0.04032000154256821, + 0.03932800143957138, + 0.04016000032424927, + 0.039712000638246536, + 0.038656000047922134, + 0.041600000113248825, + 0.03907199949026108, + 0.03929600119590759, + 0.03862399980425835, + 0.03827200084924698 + ], + [ + 0.040608000010252, + 0.03936000168323517, + 0.04022400081157684, + 0.03920000046491623, + 0.038336001336574554, + 0.03948799893260002, + 0.040063999593257904, + 0.040192000567913055, + 0.03984000161290169, + 0.039103999733924866 + ], + [ + 0.044544000178575516, + 0.04294399917125702, + 0.04396799951791763, + 0.04281599819660187, + 0.04262400045990944, + 0.04320000112056732, + 0.04339199885725975, + 0.04259200021624565, + 0.04355200007557869, + 0.041919998824596405 + ], + [ + 0.04012800008058548, + 0.03999999910593033, + 0.03891199827194214, + 0.03891199827194214, + 0.03743999823927879, + 0.03852799907326698, + 0.038176000118255615, + 0.03779200091958046, + 0.03862399980425835, + 0.037856001406908035 + ] + ], + "marker_exact": true + }, + { + "rows": 4096, + "critical_median_ms": 0.058239998295903206, + "ns_per_query": 14.218749583960744, + "samples_ms": [ + [ + 0.07593599706888199, + 0.05734400078654289, + 0.05615999922156334, + 0.0570559985935688, + 0.05584000051021576, + 0.0575999990105629, + 0.05711999908089638, + 0.05859199911355972, + 0.05488000065088272, + 0.05523199960589409 + ], + [ + 0.07100799679756165, + 0.05907199904322624, + 0.05846399813890457, + 0.05632000043988228, + 0.05567999929189682, + 0.05801599845290184, + 0.057023998349905014, + 0.05523199960589409, + 0.05689600110054016, + 0.05907199904322624 + ], + [ + 0.0663679987192154, + 0.05756799876689911, + 0.05552000179886818, + 0.05375999957323074, + 0.05369599908590317, + 0.05728000029921532, + 0.05644800141453743, + 0.053247999399900436, + 0.055296000093221664, + 0.05552000179886818 + ], + [ + 0.06598400324583054, + 0.05648000165820122, + 0.05686400085687637, + 0.055743999779224396, + 0.05571199953556061, + 0.05580800026655197, + 0.05587200075387955, + 0.055296000093221664, + 0.0541439987719059, + 0.055424001067876816 + ], + [ + 0.06329599767923355, + 0.05728000029921532, + 0.05596800148487091, + 0.056223999708890915, + 0.05305600166320801, + 0.056671999394893646, + 0.05353600159287453, + 0.05331199988722801, + 0.05516799911856651, + 0.05398400127887726 + ], + [ + 0.06611199676990509, + 0.05660799890756607, + 0.05644800141453743, + 0.05475199967622757, + 0.05417599901556969, + 0.056992001831531525, + 0.05615999922156334, + 0.05369599908590317, + 0.05526399984955788, + 0.05612799897789955 + ], + [ + 0.06188800185918808, + 0.05804799869656563, + 0.05782400071620941, + 0.05564799904823303, + 0.05523199960589409, + 0.05686400085687637, + 0.05711999908089638, + 0.05398400127887726, + 0.057631999254226685, + 0.056543998420238495 + ], + [ + 0.05507199838757515, + 0.056992001831531525, + 0.056223999708890915, + 0.05503999814391136, + 0.0533440001308918, + 0.05676800012588501, + 0.05478399991989136, + 0.05350400134921074, + 0.05503999814391136, + 0.054687999188899994 + ] + ], + "marker_exact": true + }, + { + "rows": 32768, + "critical_median_ms": 0.2985280007123947, + "ns_per_query": 9.110351584240561, + "samples_ms": [ + [ + 0.29609599709510803, + 0.3033280074596405, + 0.30131199955940247, + 0.29923200607299805, + 0.2980160117149353, + 0.2963840067386627, + 0.29635199904441833, + 0.29721599817276, + 0.30006399750709534, + 0.2970559895038605 + ], + [ + 0.29766398668289185, + 0.2970559895038605, + 0.2961919903755188, + 0.29846400022506714, + 0.2975040078163147, + 0.2964800000190735, + 0.29865598678588867, + 0.2950400114059448, + 0.29760000109672546, + 0.2967680096626282 + ], + [ + 0.2961919903755188, + 0.3020800054073334, + 0.29583999514579773, + 0.2961919903755188, + 0.297760009765625, + 0.2956799864768982, + 0.297791987657547, + 0.29552000761032104, + 0.2959679961204529, + 0.2950400114059448 + ], + [ + 0.29769599437713623, + 0.29654398560523987, + 0.29683199524879456, + 0.29657599329948425, + 0.2956480085849762, + 0.2948159873485565, + 0.296671986579895, + 0.29606398940086365, + 0.29574400186538696, + 0.2954240143299103 + ], + [ + 0.296671986579895, + 0.2959040105342865, + 0.2970240116119385, + 0.29583999514579773, + 0.2957119941711426, + 0.2964160144329071, + 0.29657599329948425, + 0.2970559895038605, + 0.2959679961204529, + 0.29574400186538696 + ], + [ + 0.2969599962234497, + 0.2972800135612488, + 0.29712000489234924, + 0.2977280020713806, + 0.29600000381469727, + 0.295199990272522, + 0.29580798745155334, + 0.2954240143299103, + 0.29609599709510803, + 0.29740801453590393 + ], + [ + 0.29606398940086365, + 0.2993279993534088, + 0.29760000109672546, + 0.29651200771331787, + 0.29712000489234924, + 0.29686400294303894, + 0.29721599817276, + 0.29654398560523987, + 0.2970240116119385, + 0.29840001463890076 + ], + [ + 0.2945280075073242, + 0.2952960133552551, + 0.2947840094566345, + 0.2972480058670044, + 0.29631999135017395, + 0.2950400114059448, + 0.29500800371170044, + 0.29440000653266907, + 0.29449599981307983, + 0.2954240143299103 + ] + ], + "marker_exact": true + }, + { + "rows": 131072, + "critical_median_ms": 1.1141119599342346, + "ns_per_query": 8.499999694322469, + "samples_ms": [ + [ + 1.115007996559143, + 1.1131839752197266, + 1.1146240234375, + 1.113759994506836, + 1.1129599809646606, + 1.1128000020980835, + 1.113919973373413, + 1.1228480339050293, + 1.1110080480575562, + 1.1130239963531494 + ], + [ + 1.1131199598312378, + 1.1105279922485352, + 1.1111680269241333, + 1.1111040115356445, + 1.1095679998397827, + 1.1131839752197266, + 1.1103999614715576, + 1.1095679998397827, + 1.1111680269241333, + 1.107103943824768 + ], + [ + 1.1123839616775513, + 1.111583948135376, + 1.1119040250778198, + 1.1112639904022217, + 1.1103999614715576, + 1.1127040386199951, + 1.1113280057907104, + 1.1091200113296509, + 1.111807942390442, + 1.1102080345153809 + ], + [ + 1.107424020767212, + 1.1072640419006348, + 1.1094720363616943, + 1.10806405544281, + 1.1081279516220093, + 1.1077439785003662, + 1.1066880226135254, + 1.1055999994277954, + 1.1091519594192505, + 1.1077439785003662 + ], + [ + 1.1094720363616943, + 1.1099519729614258, + 1.1125439405441284, + 1.111232042312622, + 1.1100480556488037, + 1.1092480421066284, + 1.1102080345153809, + 1.1091840267181396, + 1.112831950187683, + 1.1085439920425415 + ], + [ + 1.111807942390442, + 1.1117440462112427, + 1.1116160154342651, + 1.1108479499816895, + 1.1138559579849243, + 1.1098239421844482, + 1.1092480421066284, + 1.1085760593414307, + 1.114367961883545, + 1.1143039464950562 + ], + [ + 1.113312005996704, + 1.1125760078430176, + 1.1120959520339966, + 1.1128640174865723, + 1.111840009689331, + 1.1126400232315063, + 1.1093120574951172, + 1.112064003944397, + 1.1125760078430176, + 1.1100159883499146 + ], + [ + 1.1059839725494385, + 1.106719970703125, + 1.106112003326416, + 1.1084480285644531, + 1.1059199571609497, + 1.1046080589294434, + 1.1082559823989868, + 1.10806405544281, + 1.10697603225708, + 1.1062719821929932 + ] + ], + "marker_exact": true + } + ] + }, + "gain_policy": "Retain declared 5% / 20 ms single-use migration headroom; no heldout E2E tuning.", + "transport_calibration": { + "method": "Nonnegative max-resource fit; independent eager transport timings", + "coefficients_ms_per_65536_rows": [ + 2.0728231593528488, + 5.25403206689115, + 3.3043671509231323, + 8.017647329379857, + 0.5570559799671173, + 7.460149512894069e-24, + 0.1813690545886826 + ], + "predictions": [ + { + "kind": "intra_pair", + "rows": 256, + "heldout": false, + "actual_ms": 0.17163199931383133, + "predicted_ms": 0.1942767387719761, + "relative_error": 0.13193774790642965 + }, + { + "kind": "intra", + "rows": 256, + "heldout": false, + "actual_ms": 0.17878399789333344, + "predicted_ms": 0.1942767387719761, + "relative_error": 0.08665619440888661 + }, + { + "kind": "intra_dense", + "rows": 256, + "heldout": false, + "actual_ms": 0.1780639961361885, + "predicted_ms": 0.1884538993716269, + "relative_error": 0.05834926465141166 + }, + { + "kind": "striped", + "rows": 256, + "heldout": true, + "actual_ms": 0.18012800067663193, + "predicted_ms": 0.1884538993716269, + "relative_error": 0.04622212351061261 + }, + { + "kind": "intra_pair", + "rows": 4096, + "heldout": false, + "actual_ms": 0.6821600198745728, + "predicted_ms": 0.38789200152137837, + "relative_error": -0.431376817432515 + }, + { + "kind": "intra", + "rows": 4096, + "heldout": false, + "actual_ms": 1.0989919900894165, + "predicted_ms": 0.38789200152137837, + "relative_error": -0.6470474716655409 + }, + { + "kind": "intra_dense", + "rows": 4096, + "heldout": false, + "actual_ms": 0.2651199996471405, + "predicted_ms": 0.29472657111579154, + "relative_error": 0.11167234274311899 + }, + { + "kind": "striped", + "rows": 4096, + "heldout": true, + "actual_ms": 0.26867200434207916, + "predicted_ms": 0.29472657111579154, + "relative_error": 0.09697536904715665 + }, + { + "kind": "intra_pair", + "rows": 32768, + "heldout": true, + "actual_ms": 2.0732479095458984, + "predicted_ms": 1.8335526300502487, + "relative_error": -0.11561341911501066 + }, + { + "kind": "intra", + "rows": 32768, + "heldout": true, + "actual_ms": 2.6429920196533203, + "predicted_ms": 1.8335526300502487, + "relative_error": -0.3062587338834437 + }, + { + "kind": "intra_dense", + "rows": 32768, + "heldout": true, + "actual_ms": 1.1571199893951416, + "predicted_ms": 1.088229186805554, + "relative_error": -0.05953643807121389 + }, + { + "kind": "striped", + "rows": 32768, + "heldout": true, + "actual_ms": 1.1546720266342163, + "predicted_ms": 1.088229186805554, + "relative_error": -0.05754260802726663 + }, + { + "kind": "intra_pair", + "rows": 131072, + "heldout": false, + "actual_ms": 6.2092320919036865, + "predicted_ms": 6.790103356434948, + "relative_error": 0.09354961385461302 + }, + { + "kind": "intra", + "rows": 131072, + "heldout": false, + "actual_ms": 6.001280069351196, + "predicted_ms": 6.790103356434948, + "relative_error": 0.1314425052602206 + }, + { + "kind": "intra_dense", + "rows": 131072, + "heldout": false, + "actual_ms": 4.027600049972534, + "predicted_ms": 3.808809583456168, + "relative_error": -0.05432278870834217 + }, + { + "kind": "striped", + "rows": 131072, + "heldout": true, + "actual_ms": 4.0313920974731445, + "predicted_ms": 3.808809583456168, + "relative_error": -0.05521232086466865 + } + ], + "ranks_per_node": 8, + "world_size": 8, + "heldout_rule": "32768 rows or striped layout; never used in fitting", + "note": "Effective coefficients include eager implementation costs; not physical bandwidth or universal optimality guarantees.", + "source_sha256": "a965b310b107366576791580b49e74f6b987ffd4307107518666fd444c061f11" + }, + "cudnn_source": "855ccd77ba9efed22441fd454f7cc0206a068e94" +} diff --git a/benchmarks/profiles/thd_selection_h20_cp16.json b/benchmarks/profiles/thd_selection_h20_cp16.json new file mode 100644 index 00000000..4c6bbae6 --- /dev/null +++ b/benchmarks/profiles/thd_selection_h20_cp16.json @@ -0,0 +1,1168 @@ +{ + "schema_version": 5, + "model": { + "candidate_ns": 0.12856368106272384, + "query_ns": 9.515262475665075, + "span_us": 6.381563913018475, + "bytes_per_query": 20608, + "intra_node_gbps": 43.3481035770435, + "inter_node_gbps": 20.68645784116552, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "packing_ns_per_query": 14.45166026314837, + "exchange_us": 3.9301084700615614, + "min_gain_fraction": 0.05, + "min_gain_ms": 40, + "overlap_intra_inter": true, + "peer_gbps": 12.233793384063304, + "inter_node_aggregate_gbps": null, + "peer_startup_us": 25.098063419906637 + }, + "calibration": { + "sources": [ + "layout-1m-20s2l-contiguous.json", + "layout-1m-20s2l-zigzag.json", + "layout-1m-count-mixed-256-1long-contiguous.json", + "layout-1m-count-mixed-256-1long-zigzag.json", + "layout-1m-count-mixed-512-2long-contiguous.json", + "layout-1m-count-mixed-512-2long-zigzag.json", + "layout-1m-count-tiny-mix-512-contiguous.json", + "layout-1m-count-tiny-mix-512-zigzag.json", + "layout-1m-count-uniform-256-contiguous.json", + "layout-1m-count-uniform-256-zigzag.json", + "layout-1m-single-contiguous.json", + "layout-1m-single-zigzag.json", + "layout-1m-uniform-contiguous.json", + "layout-1m-uniform-zigzag.json" + ], + "source_sha256": { + "layout-1m-20s2l-contiguous.json": "5088f225f871b029b39697501736b82dd8d3823ed48c24c15ff90174c4c15d1f", + "layout-1m-20s2l-zigzag.json": "40cab9e437964fdf61d1c3a8e7f0972a709075099299241182035e44986490ae", + "layout-1m-count-mixed-256-1long-contiguous.json": "8ba6524aa34c393f62c7231df830e30e27265f02d4c458d90521bd5b5d2f2c08", + "layout-1m-count-mixed-256-1long-zigzag.json": "3762489337627a368407b485c2a7ed02d60fada369c7d637d5c4e947bfbfbc8d", + "layout-1m-count-mixed-512-2long-contiguous.json": "d77f701c8180e3ec93b0f72551cf64cb04c76f7cf4ce4f7531eaa32b72258298", + "layout-1m-count-mixed-512-2long-zigzag.json": "e34a2928e4ee83ea68742c2963b75a9eb059c3fd7c3ad2de95a714497977d41b", + "layout-1m-count-tiny-mix-512-contiguous.json": "ee7b848ac033ced04bb4ca6af87b50fd6a41dd3b38e4d48579fad45bf2cef6d2", + "layout-1m-count-tiny-mix-512-zigzag.json": "f563c2d0c1a3dc1043d0d358891250297ab9d1aecd7bd69ee723f23a68792eb4", + "layout-1m-count-uniform-256-contiguous.json": "76a035301c254c6a000cac83629f85caac91079b07d2d79e25a6b67aad355d0c", + "layout-1m-count-uniform-256-zigzag.json": "7854f5a37814e4d97760324d11054bac40fee1bc10d9262589b005ee105451ac", + "layout-1m-single-contiguous.json": "628d7b57b8edf1a2191e75bc666ed9dcc37e9ecb875297f68590bb0623cf682a", + "layout-1m-single-zigzag.json": "58786fa4f74b6cc644d2143647f65b644cab602254d6fa201ee5dd2c414d1e97", + "layout-1m-uniform-contiguous.json": "70e57ca6facc163aea921505d2bf839b611618f9088f440a414c3a4f1be84301", + "layout-1m-uniform-zigzag.json": "26d5ecbaf81f5c0311cb08dcc218c1ecb3dd569a442358e61bc790aa7710c965" + }, + "samples": 224, + "selection_fit_rmse_ms": 2.949003900413, + "observed_routing_effective_gbps": [ + 21.003766295884223, + 22.729257916064537, + 37.48939138205569, + 38.61166479509942, + 39.58319101689069, + 20.138357216688668, + 39.28377120939799 + ], + "transport_note": "Critical phase increments remove pre-existing rank arrival skew. Shared conservative effective rate includes packing; links were not isolated.", + "validation_note": "Evaluate held-out layouts and seeds; fitting is not a performance guarantee." + }, + "packing_calibration": { + "world": 16, + "records": [ + { + "rows": 256, + "critical_median_ms": 0.04848000034689903, + "ns_per_query": 189.37500135507435, + "samples_ms": [ + [ + 0.04598399996757507, + 0.045152001082897186, + 0.04572800174355507, + 0.04447999969124794, + 0.043136000633239746, + 0.04396799951791763, + 0.04396799951791763, + 0.04816000163555145, + 0.04335999861359596, + 0.04307200014591217 + ], + [ + 0.04633599892258644, + 0.0459199994802475, + 0.04368000105023384, + 0.04447999969124794, + 0.04291199892759323, + 0.04335999861359596, + 0.042688000947237015, + 0.04633599892258644, + 0.04416000097990036, + 0.042080000042915344 + ], + [ + 0.05820799991488457, + 0.04732799902558327, + 0.045632001012563705, + 0.04473600164055824, + 0.043807998299598694, + 0.04886399954557419, + 0.04492799937725067, + 0.04790399968624115, + 0.04390399903059006, + 0.04342399910092354 + ], + [ + 0.04710400104522705, + 0.04825599864125252, + 0.045024000108242035, + 0.04383999854326248, + 0.043616000562906265, + 0.04636799916625023, + 0.04451199993491173, + 0.04790399968624115, + 0.04374400153756142, + 0.04224000126123428 + ], + [ + 0.04668800160288811, + 0.04707200080156326, + 0.0451200008392334, + 0.04467200115323067, + 0.044415999203920364, + 0.04374400153756142, + 0.04297599941492081, + 0.04601600021123886, + 0.04335999861359596, + 0.043136000633239746 + ], + [ + 0.04652800038456917, + 0.04665600135922432, + 0.04476799815893173, + 0.045632001012563705, + 0.04307200014591217, + 0.043935999274253845, + 0.0432640016078949, + 0.048128001391887665, + 0.04371200129389763, + 0.04371200129389763 + ], + [ + 0.04787199944257736, + 0.04896000027656555, + 0.045504000037908554, + 0.044544000178575516, + 0.04399999976158142, + 0.044544000178575516, + 0.043807998299598694, + 0.05251200124621391, + 0.04623999819159508, + 0.04435199871659279 + ], + [ + 0.04713600128889084, + 0.04572800174355507, + 0.04396799951791763, + 0.04499199986457825, + 0.04342399910092354, + 0.04467200115323067, + 0.04368000105023384, + 0.04822399839758873, + 0.043296001851558685, + 0.043296001851558685 + ], + [ + 0.0461760014295578, + 0.047168001532554626, + 0.045504000037908554, + 0.044576000422239304, + 0.04374400153756142, + 0.0435199998319149, + 0.043776001781225204, + 0.04355200007557869, + 0.043455999344587326, + 0.04355200007557869 + ], + [ + 0.048608001321554184, + 0.047520000487565994, + 0.0480320006608963, + 0.04726399853825569, + 0.0459199994802475, + 0.04630399867892265, + 0.04588799923658371, + 0.04608000069856644, + 0.04524800181388855, + 0.04572800174355507 + ], + [ + 0.046720001846551895, + 0.05827200040221214, + 0.04604800045490265, + 0.04473600164055824, + 0.043487999588251114, + 0.04447999969124794, + 0.04390399903059006, + 0.044064000248909, + 0.04335999861359596, + 0.04390399903059006 + ], + [ + 0.04927999898791313, + 0.04636799916625023, + 0.044704001396894455, + 0.04438399896025658, + 0.04556800052523613, + 0.043616000562906265, + 0.04368000105023384, + 0.04310400038957596, + 0.04249599948525429, + 0.04275200143456459 + ], + [ + 0.045024000108242035, + 0.04435199871659279, + 0.04403200000524521, + 0.04339199885725975, + 0.04230400174856186, + 0.042367998510599136, + 0.04275200143456459, + 0.04278400167822838, + 0.04310400038957596, + 0.04281599819660187 + ], + [ + 0.04681599885225296, + 0.04726399853825569, + 0.047200001776218414, + 0.04620800167322159, + 0.045951999723911285, + 0.04569600149989128, + 0.04473600164055824, + 0.04566400125622749, + 0.04473600164055824, + 0.045471999794244766 + ], + [ + 0.052032001316547394, + 0.05023999884724617, + 0.048576001077890396, + 0.04838399961590767, + 0.04835199937224388, + 0.047359999269247055, + 0.045791998505592346, + 0.047520000487565994, + 0.046560000628232956, + 0.046751998364925385 + ], + [ + 0.0488319993019104, + 0.04864000156521797, + 0.04806400090456009, + 0.04652800038456917, + 0.04614400118589401, + 0.047231998294591904, + 0.04668800160288811, + 0.047520000487565994, + 0.046592000871896744, + 0.046431999653577805 + ] + ], + "marker_exact": true + }, + { + "rows": 4096, + "critical_median_ms": 0.07840000092983246, + "ns_per_query": 19.140625227009878, + "samples_ms": [ + [ + 0.07708799839019775, + 0.07785599678754807, + 0.07744000107049942, + 0.07568000257015228, + 0.07692799717187881, + 0.07631999999284744, + 0.07599999755620956, + 0.07487999647855759, + 0.07599999755620956, + 0.07561600208282471 + ], + [ + 0.07827199995517731, + 0.07753600180149078, + 0.07715199887752533, + 0.07635200023651123, + 0.07795199751853943, + 0.07503999769687653, + 0.07667200267314911, + 0.07628799974918365, + 0.0772479996085167, + 0.07475200295448303 + ], + [ + 0.07763200253248215, + 0.07756800204515457, + 0.07676800340414047, + 0.07718399912118912, + 0.0764480009675026, + 0.07462400197982788, + 0.07756800204515457, + 0.07507199794054031, + 0.07577600330114365, + 0.0783040001988411 + ], + [ + 0.07705599814653397, + 0.07638400048017502, + 0.0761599987745285, + 0.07587199658155441, + 0.07625599950551987, + 0.07558400183916092, + 0.08560000360012054, + 0.07823999971151352, + 0.07606399804353714, + 0.07568000257015228 + ], + [ + 0.07744000107049942, + 0.0759039968252182, + 0.07744000107049942, + 0.07580800354480743, + 0.07577600330114365, + 0.07587199658155441, + 0.07689599692821503, + 0.07612799853086472, + 0.07648000121116638, + 0.07568000257015228 + ], + [ + 0.0783040001988411, + 0.07827199995517731, + 0.0777600035071373, + 0.07657600194215775, + 0.07651200145483017, + 0.07692799717187881, + 0.07747200131416321, + 0.0761599987745285, + 0.07683199644088745, + 0.07651200145483017 + ], + [ + 0.078015998005867, + 0.07558400183916092, + 0.07727999985218048, + 0.07648000121116638, + 0.0764480009675026, + 0.07676800340414047, + 0.07654400169849396, + 0.07503999769687653, + 0.07631999999284744, + 0.07583999633789062 + ], + [ + 0.07817599922418594, + 0.07705599814653397, + 0.07731200009584427, + 0.07638400048017502, + 0.07753600180149078, + 0.07612799853086472, + 0.07740800082683563, + 0.07532799988985062, + 0.07638400048017502, + 0.07580800354480743 + ], + [ + 0.07648000121116638, + 0.07555200159549713, + 0.07622399926185608, + 0.07526399940252304, + 0.07571200281381607, + 0.07446400076150894, + 0.07676800340414047, + 0.0745600014925003, + 0.07558400183916092, + 0.07446400076150894 + ], + [ + 0.07887999713420868, + 0.07868800312280655, + 0.07820799946784973, + 0.07571200281381607, + 0.077504001557827, + 0.07660800218582153, + 0.07740800082683563, + 0.07651200145483017, + 0.07903999835252762, + 0.07625599950551987 + ], + [ + 0.0769599974155426, + 0.07635200023651123, + 0.07849600166082382, + 0.0753600001335144, + 0.07622399926185608, + 0.07568000257015228, + 0.07625599950551987, + 0.07491199672222137, + 0.0772159993648529, + 0.07609599828720093 + ], + [ + 0.07740800082683563, + 0.07609599828720093, + 0.07680000364780426, + 0.07491199672222137, + 0.07753600180149078, + 0.07599999755620956, + 0.0772479996085167, + 0.07689599692821503, + 0.07747200131416321, + 0.07548800110816956 + ], + [ + 0.07760000228881836, + 0.07667200267314911, + 0.0761599987745285, + 0.07622399926185608, + 0.07715199887752533, + 0.07552000135183334, + 0.0769599974155426, + 0.07603199779987335, + 0.07628799974918365, + 0.07491199672222137 + ], + [ + 0.07868800312280655, + 0.07683199644088745, + 0.07734400033950806, + 0.07760000228881836, + 0.07804799824953079, + 0.07744000107049942, + 0.07779199630022049, + 0.07606399804353714, + 0.07811199873685837, + 0.07689599692821503 + ], + [ + 0.07788799703121185, + 0.07583999633789062, + 0.07788799703121185, + 0.07513599842786789, + 0.07596799731254578, + 0.07603199779987335, + 0.0769599974155426, + 0.07568000257015228, + 0.07542400062084198, + 0.07513599842786789 + ], + [ + 0.07872000336647034, + 0.07734400033950806, + 0.07689599692821503, + 0.07756800204515457, + 0.07791999727487564, + 0.07654400169849396, + 0.07779199630022049, + 0.07593599706888199, + 0.07756800204515457, + 0.077504001557827 + ] + ], + "marker_exact": true + }, + { + "rows": 32768, + "critical_median_ms": 0.495168000459671, + "ns_per_query": 15.111328139028046, + "samples_ms": [ + [ + 0.4846400022506714, + 0.48422399163246155, + 0.4835520088672638, + 0.4837760031223297, + 0.4890559911727905, + 0.4880639910697937, + 0.48630398511886597, + 0.4833280146121979, + 0.4864319860935211, + 0.4830079972743988 + ], + [ + 0.4832319915294647, + 0.48476800322532654, + 0.4800640046596527, + 0.48451200127601624, + 0.4832319915294647, + 0.48390400409698486, + 0.5678719878196716, + 0.49299201369285583, + 0.48403200507164, + 0.4846400022506714 + ], + [ + 0.482015997171402, + 0.4830720126628876, + 0.48156800866127014, + 0.48368000984191895, + 0.4830079972743988, + 0.481471985578537, + 0.48044800758361816, + 0.48524799942970276, + 0.4825280010700226, + 0.48230400681495667 + ], + [ + 0.48281601071357727, + 0.4836159944534302, + 0.4824639856815338, + 0.4837439954280853, + 0.482015997171402, + 0.48240000009536743, + 0.48211199045181274, + 0.4838080108165741, + 0.4822719991207123, + 0.4832639992237091 + ], + [ + 0.48342400789260864, + 0.48368000984191895, + 0.48368000984191895, + 0.4848960041999817, + 0.4822719991207123, + 0.4830400049686432, + 0.659168004989624, + 0.4891839921474457, + 0.4853439927101135, + 0.4845759868621826 + ], + [ + 0.4935680031776428, + 0.4917759895324707, + 0.4864000082015991, + 0.4891520142555237, + 0.48416000604629517, + 0.4912639856338501, + 0.4907520115375519, + 0.4926080107688904, + 0.4842880070209503, + 0.48710399866104126 + ], + [ + 0.48550400137901306, + 0.4867840111255646, + 0.48364800214767456, + 0.48230400681495667, + 0.4846400022506714, + 0.48447999358177185, + 0.4832639992237091, + 0.48524799942970276, + 0.4850879907608032, + 0.4859839975833893 + ], + [ + 0.4856320023536682, + 0.48576000332832336, + 0.483487993478775, + 0.48399999737739563, + 0.48284798860549927, + 0.48473599553108215, + 0.4832639992237091, + 0.48416000604629517, + 0.4830720126628876, + 0.48576000332832336 + ], + [ + 0.48124799132347107, + 0.4818879961967468, + 0.4793280065059662, + 0.48153600096702576, + 0.48051199316978455, + 0.4798080027103424, + 0.4789760112762451, + 0.4806399941444397, + 0.48051199316978455, + 0.48150399327278137 + ], + [ + 0.4848000109195709, + 0.48655998706817627, + 0.4845759868621826, + 0.48390400409698486, + 0.4891839921474457, + 0.4854080080986023, + 0.483487993478775, + 0.4849280118942261, + 0.4829440116882324, + 0.4832000136375427 + ], + [ + 0.48240000009536743, + 0.48524799942970276, + 0.4813440144062042, + 0.48604801297187805, + 0.48550400137901306, + 0.4848960041999817, + 0.4816319942474365, + 0.4835520088672638, + 0.48028799891471863, + 0.48287999629974365 + ], + [ + 0.5635200142860413, + 0.488319993019104, + 0.48419201374053955, + 0.4840959906578064, + 0.48099198937416077, + 0.4856320023536682, + 0.48393601179122925, + 0.4859839975833893, + 0.4830079972743988, + 0.4822080135345459 + ], + [ + 0.48470398783683777, + 0.485152006149292, + 0.4819520115852356, + 0.4846400022506714, + 0.48284798860549927, + 0.48265600204467773, + 0.48451200127601624, + 0.48364800214767456, + 0.4830720126628876, + 0.4837439954280853 + ], + [ + 0.4851840138435364, + 0.4853760004043579, + 0.4843519926071167, + 0.4835520088672638, + 0.4819520115852356, + 0.4843840003013611, + 0.4817599952220917, + 0.49718400835990906, + 0.4858880043029785, + 0.48393601179122925 + ], + [ + 0.4880000054836273, + 0.4862079918384552, + 0.48396798968315125, + 0.48185598850250244, + 0.4809280037879944, + 0.4840959906578064, + 0.4854719936847687, + 0.4825280010700226, + 0.4824639856815338, + 0.4829440116882324 + ], + [ + 0.4848319888114929, + 0.48502400517463684, + 0.484607994556427, + 0.4850560128688812, + 0.481471985578537, + 0.48585599660873413, + 0.4817599952220917, + 0.5250239968299866, + 0.512224018573761, + 0.49856001138687134 + ] + ], + "marker_exact": true + }, + { + "rows": 65536, + "critical_median_ms": 0.9471040070056915, + "ns_per_query": 14.45166026314837, + "samples_ms": [ + [ + 0.9458879828453064, + 0.9456959962844849, + 0.9457280039787292, + 0.9461119771003723, + 0.9447680115699768, + 0.9431679844856262, + 0.9455680251121521, + 0.9448959827423096, + 0.9446079730987549, + 0.9431679844856262 + ], + [ + 0.9470400214195251, + 0.9518719911575317, + 0.9548479914665222, + 0.9454399943351746, + 0.9466879963874817, + 0.9461759924888611, + 0.9453759789466858, + 0.9455040097236633, + 0.9439039826393127, + 0.9450560212135315 + ], + [ + 0.9453759789466858, + 0.9442880153656006, + 0.9488000273704529, + 0.9441919922828674, + 0.9446719884872437, + 0.9445440173149109, + 0.9446719884872437, + 0.9427520036697388, + 0.9429119825363159, + 0.9422720074653625 + ], + [ + 0.9474560022354126, + 0.9570239782333374, + 0.948095977306366, + 0.9434239864349365, + 0.9453439712524414, + 0.9433599710464478, + 0.9449920058250427, + 0.9431359767913818, + 0.9460800290107727, + 0.9411839842796326 + ], + [ + 0.9461759924888611, + 0.9458240270614624, + 0.9471359848976135, + 0.9443519711494446, + 0.9463359713554382, + 0.9470080137252808, + 0.9451839923858643, + 0.9464960098266602, + 0.9453439712524414, + 0.9448639750480652 + ], + [ + 0.9487360119819641, + 0.9451199769973755, + 0.9447360038757324, + 0.9431040287017822, + 0.945792019367218, + 0.9423360228538513, + 0.9433919787406921, + 0.9450240135192871, + 0.9434239864349365, + 0.9451839923858643 + ], + [ + 0.9472640156745911, + 0.9432960152626038, + 0.9454399943351746, + 0.94377601146698, + 0.9430080056190491, + 0.9435520172119141, + 0.9461759924888611, + 0.9435840249061584, + 0.9455680251121521, + 0.944320023059845 + ], + [ + 0.9450240135192871, + 0.9433919787406921, + 0.9450240135192871, + 0.9413440227508545, + 0.9461439847946167, + 0.9453120231628418, + 0.9457600116729736, + 0.9438080191612244, + 0.9462720155715942, + 0.9450240135192871 + ], + [ + 0.9440640211105347, + 0.9446719884872437, + 0.9467840194702148, + 0.9463679790496826, + 0.9463679790496826, + 0.9440640211105347, + 0.9461119771003723, + 0.941536009311676, + 0.9463040232658386, + 0.9431040287017822 + ], + [ + 0.9471679925918579, + 0.9464640021324158, + 0.9434880018234253, + 0.9440000057220459, + 0.9467200040817261, + 0.9445120096206665, + 0.9435200095176697, + 0.9442880153656006, + 0.9439359903335571, + 0.9440640211105347 + ], + [ + 0.9433919787406921, + 0.943231999874115, + 0.9468160271644592, + 0.9446079730987549, + 0.9435520172119141, + 0.9436159729957581, + 0.9437119960784912, + 0.9442560076713562, + 0.9421759843826294, + 0.9425600171089172 + ], + [ + 0.9450560212135315, + 0.9459840059280396, + 0.9427840113639832, + 0.9410880208015442, + 0.9449920058250427, + 0.9431040287017822, + 0.9437440037727356, + 0.9419519901275635, + 0.9431679844856262, + 0.9477120041847229 + ], + [ + 0.9472320079803467, + 0.9468799829483032, + 0.9467520117759705, + 0.944383978843689, + 0.9447360038757324, + 0.9464960098266602, + 0.9450880289077759, + 0.9472000002861023, + 0.9467520117759705, + 0.9451519846916199 + ], + [ + 0.9446719884872437, + 0.9464640021324158, + 0.944927990436554, + 0.9429759979248047, + 0.9435840249061584, + 0.943231999874115, + 0.9460480213165283, + 0.944096028804779, + 0.9459519982337952, + 0.9434239864349365 + ], + [ + 0.9466239809989929, + 0.9458240270614624, + 0.944927990436554, + 0.9418560266494751, + 0.9429759979248047, + 0.9424960017204285, + 0.9442560076713562, + 0.9439039826393127, + 0.9446399807929993, + 0.9423999786376953 + ], + [ + 0.946943998336792, + 0.9460480213165283, + 0.9494720101356506, + 0.9453120231628418, + 0.9448320269584656, + 0.9444159865379333, + 0.9444800019264221, + 0.944159984588623, + 0.9457600116729736, + 0.9445120096206665 + ] + ], + "marker_exact": true + } + ] + }, + "gain_policy": "Retain declared 5% / 40 ms single-use migration headroom; no heldout E2E tuning.", + "transport_calibration": { + "method": "Nonnegative max-resource fit; independent eager transport timings", + "coefficients_ms_per_65536_rows": [ + 31.15628543240907, + 65.28744062274444, + 110.39632970746038, + 4.357277541747443, + 0.9471040070056915, + 0.025098063419906638, + 0.003930108470061562 + ], + "predictions": [ + { + "kind": "intra_pair", + "rows": 256, + "heldout": false, + "actual_ms": 0.3941280096769333, + "predicted_ms": 0.4602638348097353, + "relative_error": 0.16780290542408682 + }, + { + "kind": "intra", + "rows": 256, + "heldout": false, + "actual_ms": 0.3874559998512268, + "predicted_ms": 0.4602638348097353, + "relative_error": 0.1879125242258859 + }, + { + "kind": "intra_dense", + "rows": 256, + "heldout": false, + "actual_ms": 0.2534399926662445, + "predicted_ms": 0.28610776238346247, + "relative_error": 0.12889745368734373 + }, + { + "kind": "inter_pair", + "rows": 256, + "heldout": false, + "actual_ms": 0.6665279865264893, + "predicted_ms": 0.4602638348097353, + "relative_error": -0.309460601634552 + }, + { + "kind": "inter", + "rows": 256, + "heldout": false, + "actual_ms": 0.7499519884586334, + "predicted_ms": 0.4602638348097353, + "relative_error": -0.38627559911440523 + }, + { + "kind": "inter_dense", + "rows": 256, + "heldout": false, + "actual_ms": 0.6300639808177948, + "predicted_ms": 0.4597436807619102, + "relative_error": -0.2703222295532852 + }, + { + "kind": "striped", + "rows": 256, + "heldout": true, + "actual_ms": 0.44177600741386414, + "predicted_ms": 0.507915592234959, + "relative_error": 0.1497129398408774 + }, + { + "kind": "intra_pair", + "rows": 4096, + "heldout": false, + "actual_ms": 6.484655857086182, + "predicted_ms": 6.928798778606242, + "relative_error": 0.06849136350616325 + }, + { + "kind": "intra", + "rows": 4096, + "heldout": false, + "actual_ms": 6.525344133377075, + "predicted_ms": 6.928798778606242, + "relative_error": 0.061828868636291556 + }, + { + "kind": "intra_dense", + "rows": 4096, + "heldout": false, + "actual_ms": 1.9262880086898804, + "predicted_ms": 1.883475911994279, + "relative_error": -0.022225179465618417 + }, + { + "kind": "inter_pair", + "rows": 4096, + "heldout": false, + "actual_ms": 7.343456029891968, + "predicted_ms": 6.928798778606242, + "relative_error": -0.056466226473997905 + }, + { + "kind": "inter", + "rows": 4096, + "heldout": false, + "actual_ms": 7.85427188873291, + "predicted_ms": 6.928798778606242, + "relative_error": -0.11783054154953243 + }, + { + "kind": "inter_dense", + "rows": 4096, + "heldout": false, + "actual_ms": 4.239135980606079, + "predicted_ms": 4.285179654750842, + "relative_error": 0.010861570460445558 + }, + { + "kind": "striped", + "rows": 4096, + "heldout": true, + "actual_ms": 2.2811520099639893, + "predicted_ms": 2.420633579229425, + "relative_error": 0.06114523217049328 + }, + { + "kind": "intra_pair", + "rows": 32768, + "heldout": true, + "actual_ms": 50.98955154418945, + "predicted_ms": 55.22719302562015, + "relative_error": 0.08310803592297145 + }, + { + "kind": "intra", + "rows": 32768, + "heldout": true, + "actual_ms": 51.211055755615234, + "predicted_ms": 55.22719302562015, + "relative_error": 0.0784232469092292 + }, + { + "kind": "intra_dense", + "rows": 32768, + "heldout": true, + "actual_ms": 14.252928256988525, + "predicted_ms": 13.810491429088376, + "relative_error": -0.031041819612276034 + }, + { + "kind": "inter_pair", + "rows": 32768, + "heldout": true, + "actual_ms": 56.860286712646484, + "predicted_ms": 55.22719302562015, + "relative_error": -0.02872116518299428 + }, + { + "kind": "inter", + "rows": 32768, + "heldout": true, + "actual_ms": 61.266624450683594, + "predicted_ms": 55.22719302562015, + "relative_error": -0.09857620652701149 + }, + { + "kind": "inter_dense", + "rows": 32768, + "heldout": true, + "actual_ms": 30.82592010498047, + "predicted_ms": 32.84843492720153, + "relative_error": 0.06561085006816358 + }, + { + "kind": "striped", + "rows": 32768, + "heldout": true, + "actual_ms": 15.788095951080322, + "predicted_ms": 16.702261215454772, + "relative_error": 0.05790218574849093 + }, + { + "kind": "intra_pair", + "rows": 65536, + "heldout": false, + "actual_ms": 102.41177368164062, + "predicted_ms": 110.42535787935034, + "relative_error": 0.07824866135627051 + }, + { + "kind": "intra", + "rows": 65536, + "heldout": false, + "actual_ms": 102.45684814453125, + "predicted_ms": 110.42535787935034, + "relative_error": 0.07777430088009618 + }, + { + "kind": "intra_dense", + "rows": 65536, + "heldout": false, + "actual_ms": 28.466943740844727, + "predicted_ms": 27.441366305767346, + "relative_error": -0.03602695970505143 + }, + { + "kind": "inter_pair", + "rows": 65536, + "heldout": false, + "actual_ms": 114.2823371887207, + "predicted_ms": 110.42535787935034, + "relative_error": -0.03374956624312919 + }, + { + "kind": "inter", + "rows": 65536, + "heldout": false, + "actual_ms": 122.30096054077148, + "predicted_ms": 110.42535787935034, + "relative_error": -0.09710146681523546 + }, + { + "kind": "inter_dense", + "rows": 65536, + "heldout": false, + "actual_ms": 60.029855728149414, + "predicted_ms": 65.49215523857376, + "relative_error": 0.09099304744560532 + }, + { + "kind": "striped", + "rows": 65536, + "heldout": true, + "actual_ms": 31.16423988342285, + "predicted_ms": 33.02412137114088, + "relative_error": 0.05967998881652026 + } + ], + "ranks_per_node": 8, + "world_size": 16, + "heldout_rule": "32768 rows or striped layout; never used in fitting", + "note": "Effective coefficients include eager implementation costs; not physical bandwidth or universal optimality guarantees.", + "source_sha256": "be7130a522549221bc2c1d2b538da8101ef5f7c3f640d6f5bbb83cb573bf0de5", + "node_limit_note": "Removed node envelope mathematically dominated by per-rank inter-node limits for this topology; it is not independently identified by these fixtures." + }, + "transport_validation": { + "source_sha256": "ef98c8708210f495df09c0c78d7e2d099cfb9713213b671abde4da5ad9da50e9", + "records": [ + { + "kind": "mixed_dense_quarter", + "rows": 8192, + "actual_ms": 2.960976004600525, + "predicted_ms": 3.054578100021679, + "relative_error": 0.031611906099787035 + }, + { + "kind": "mixed_dense_three_quarters", + "rows": 8192, + "actual_ms": 6.393343925476074, + "predicted_ms": 6.619486619026664, + "relative_error": 0.03537158272519347 + }, + { + "kind": "mixed_dense_quarter", + "rows": 49152, + "actual_ms": 16.762415885925293, + "predicted_ms": 16.425463301286772, + "relative_error": -0.020101671914813046 + }, + { + "kind": "mixed_dense_three_quarters", + "rows": 49152, + "actual_ms": 36.45444869995117, + "predicted_ms": 37.81491441531668, + "relative_error": 0.03731960745211671 + } + ] + }, + "cudnn_source": "855ccd77ba9efed22441fd454f7cc0206a068e94" +} diff --git a/benchmarks/summarize_csa_layout_matrix.py b/benchmarks/summarize_csa_layout_matrix.py new file mode 100644 index 00000000..7d55f168 --- /dev/null +++ b/benchmarks/summarize_csa_layout_matrix.py @@ -0,0 +1,136 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Summarize complete paired full-CSA matrices, retaining worst-rank errors.""" + +import argparse +import json +import math +import statistics +from pathlib import Path + + +def summarize(directory): + grouped = {} + for path in sorted(directory.glob("layout-1m-*.json")): + raw = json.loads(path.read_text()) + layout = raw["config"]["layout"] + name = path.stem.removeprefix("layout-").removesuffix("-" + layout) + ranks = raw["ranks"] + if sorted(r["rank"] for r in ranks) != list(range(raw["world_size"])): + raise ValueError(f"incomplete rank results: {path}") + if any( + not all(r[k] for k in ("finite", "input_match", "redistribution_bitwise")) + or (r.get("check") is not None and not r["check"]["pass_"]) + for r in ranks + ): + raise ValueError(f"failed correctness check: {path}") + repeats = raw["config"]["repeats"] + if not repeats or any(len(r["samples"]) != repeats for r in ranks): + raise ValueError(f"incomplete timing samples: {path}") + if any( + not math.isfinite(s[k]) or s[k] < 0 + for r in ranks + for s in r["samples"] + for k in ("total_ms", "peak_allocated_gib", "peak_reserved_gib") + ) or any( + not math.isfinite(r[k]) or r[k] < 0 + for r in ranks + for k in ("layout_prepare_ms", "common_prepare_ms", "kda_prepare_ms", "total_prepare_ms") + ): + raise ValueError(f"invalid timing/memory samples: {path}") + if any( + not math.isclose(r["total_prepare_ms"], r["layout_prepare_ms"] + r["common_prepare_ms"] + r["kda_prepare_ms"]) + for r in ranks + ): + raise ValueError(f"inconsistent preparation accounting: {path}") + errors = {} + for rank in ranks: + if layout != "contiguous": + required = {"output", "dq", "dkv", "d_index_q", "d_weights", "d_pool_kv", "d_pool_score", "d_ape"} + if raw["config"].get("with_kda"): + required |= {"kda_output", "kda_dq", "kda_dk", "kda_dv", "kda_dg", "kda_dbeta"} + if not required.issubset(rank.get("layout_vs_contiguous") or {}): + raise ValueError(f"missing output/gradient baseline comparison: {path}") + delta = rank.get("loss_abs_diff") + if delta is None or not math.isfinite(delta) or not 0 <= delta < 1e-5: + raise ValueError(f"failed loss comparison: {path}") + for tensor, error in (rank["layout_vs_contiguous"] or {}).items(): + if not error["finite"] or not 0 <= error["relative_rms"] < 0.03: + raise ValueError(f"failed precision comparison: {path}: {tensor}") + errors[tensor] = max(errors.get(tensor, 0.0), error["relative_rms"]) + critical = [max(r["samples"][i]["total_ms"] for r in ranks) for i in range(repeats)] + row = dict( + source=str(path), + total_ms=statistics.median(critical), + min_ms=min(critical), + max_ms=max(critical), + allocated_gib=max(s["peak_allocated_gib"] for r in ranks for s in r["samples"]), + reserved_gib=max(s["peak_reserved_gib"] for r in ranks for s in r["samples"]), + prepare_ms=max(r["total_prepare_ms"] for r in ranks), + layout_prepare_ms=max(r["layout_prepare_ms"] for r in ranks), + policy=ranks[0]["selected_policy"], + relative_rms=errors, + loss_abs_diff=max((r["loss_abs_diff"] or 0.0) for r in ranks), + workspace_gib=max(r["selection_workspace_bytes"] for r in ranks) / 2**30, + max_spans=max(r["selection_spans"] for r in ranks), + ) + contract = { + key: raw["config"].get(key) + for key in ( + "heads", + "top_k", + "with_kda", + "kda_batch_invariant", + "kda_gate_scale", + "candidate_mib", + "redistribution_scope", + "pool_backward_order", + "repeats", + "warmups", + ) + } + group = grouped.setdefault( + name, dict(lengths=raw["config"]["sequence_lengths"], world_size=raw["world_size"], contract=contract, layouts={}) + ) + if ( + group["lengths"] != raw["config"]["sequence_lengths"] + or group["world_size"] != raw["world_size"] + or group["contract"] != contract + ): + raise ValueError(f"unpaired shape/group: {name}") + group["layouts"][layout] = row + if not grouped: + raise ValueError("no 1M results found") + for name, case in grouped.items(): + if set(case["layouts"]) != {"contiguous", "zigzag", "planner"}: + raise ValueError(f"incomplete comparison: {name}") + c, z, p = (case["layouts"][k] for k in ("contiguous", "zigzag", "planner")) + case["speedup_vs_contiguous"] = c["total_ms"] / p["total_ms"] + case["gap_to_best_baseline_pct"] = 100 * (p["total_ms"] / min(c["total_ms"], z["total_ms"]) - 1) + case["prepare_plus_iteration_ms"] = {k: v["prepare_ms"] + v["total_ms"] for k, v in case["layouts"].items()} + first = case["prepare_plus_iteration_ms"] + case["prepare_inclusive_gap_pct"] = 100 * (first["planner"] / min(first["contiguous"], first["zigzag"]) - 1) + return grouped + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + results = summarize(args.directory) + if args.output: + args.output.write_text(json.dumps(results, indent=2) + "\n") + print("| Case | Docs | Contiguous ms | Zigzag ms | Planner ms | Speedup | Planner allocated GiB | Prepare ms | Policy |") + print("|---|---:|---:|---:|---:|---:|---:|---:|---|") + for name, case in results.items(): + c, z, p = (case["layouts"][k] for k in ("contiguous", "zigzag", "planner")) + print( + f"| {name} | {len(case['lengths'])} | {c['total_ms']:.2f} | {z['total_ms']:.2f} | {p['total_ms']:.2f} | " + f"{case['speedup_vs_contiguous']:.3f}x | {p['allocated_gib']:.2f} | {p['prepare_ms']:.2f} | {p['policy']} |" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/thd_1m_cases.py b/benchmarks/thd_1m_cases.py new file mode 100644 index 00000000..72a806d2 --- /dev/null +++ b/benchmarks/thd_1m_cases.py @@ -0,0 +1,259 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Reproducible 1M THD cases inspired by a character-length count histogram. + +The screenshot supplies no exact percentages or tokenizer. Weights below are +explicit synthetic priors, not recovered measurements. Long-document spikes +are additional stress cases, not claims about the screenshot's hidden tail. +""" + +import argparse +import hashlib +import json +import math +import random +from dataclasses import asdict, dataclass, replace +from itertools import accumulate +from pathlib import Path + +TOTAL_TOKENS = 1 << 20 +DEFAULT_SEED = 20261112 +# (lower exclusive, upper inclusive); k=1024 is a synthetic convention. +CHARACTER_BINS = ((10, 100), (100, 1024), (1024, 2048), (2048, 4096), (4096, 8192), (8192, 16384)) +COUNT_PRIORS = { + "tiny90": (9000, 500, 200, 200, 80, 20), + "mix80": (8000, 700, 500, 500, 250, 50), + "mix70": (7000, 1200, 800, 600, 300, 100), +} + + +@dataclass(frozen=True) +class THDCase: + name: str + lengths: tuple[int, ...] + prior: str + character_divisor: int + long_documents: tuple[int, ...] + order: str + seed: int + final_document_truncation: int = 0 + + +def _fill(budget, prior, divisor, seed): + """Draw aligned lengths and truncate at most the last draw to fill exactly.""" + rng = random.Random(seed) + cumulative = tuple(accumulate(COUNT_PRIORS[prior])) + lengths, remaining, truncation = [], budget, 0 + while remaining: + draw = rng.randrange(cumulative[-1]) + bucket = next(i for i, bound in enumerate(cumulative) if draw < bound) + lower, upper = CHARACTER_BINS[bucket] + # Uniform aligned token proxies within each scaled character bin. + minimum = max(1, lower // (4 * divisor) + 1) + maximum = max(minimum, upper // (4 * divisor)) + proposed = 4 * rng.randint(minimum, maximum) + length = min(proposed, remaining) + lengths.append(length) + remaining -= length + truncation += proposed - length + return lengths, truncation + + +def _ordered(lengths, order, seed): + values = sorted(lengths) + if order == "ascending": + return tuple(values) + if order == "descending": + return tuple(reversed(values)) + if order == "alternating": + # Smallest, largest, second-smallest, second-largest, ... + return tuple( + v + for i in range((len(values) + 1) // 2) + for v in ((values[i], values[-1 - i]) if i != len(values) - 1 - i else (values[i],)) + ) + if order != "shuffled": + raise ValueError(f"unknown order: {order}") + random.Random(seed + 1).shuffle(values) + return tuple(values) + + +def statistics_cases(seed=DEFAULT_SEED): + """Return 18 block-aligned cases; same-histogram order variants share data.""" + cases = [THDCase("1m-stat-tiny64", (64,) * (TOTAL_TOKENS // 64), "fixed64", 1, (), "constant", seed)] + specifications = [ + ("tiny90", "tiny90", 1, ()), + ("mix80", "mix80", 1, ()), + ("mix70", "mix70", 1, ()), + ("mix80-div2", "mix80", 2, ()), + ("mix80-div4", "mix80", 4, ()), + ("mix80-1x128k", "mix80", 1, (128 << 10,)), + ("mix80-1x256k", "mix80", 1, (256 << 10,)), + ("mix80-1x512k", "mix80", 1, (512 << 10,)), + ("mix80-2x256k", "mix80", 1, (256 << 10,) * 2), + ("mix80-4x128k", "mix80", 1, (128 << 10,) * 4), + ("mix80-8x64k", "mix80", 1, (64 << 10,) * 8), + ] + for name, prior, divisor, long_docs in specifications: + small, truncated = _fill(TOTAL_TOKENS - sum(long_docs), prior, divisor, seed) + lengths = _ordered([*small, *long_docs], "shuffled", seed) + cases.append(THDCase(f"1m-stat-{name}", lengths, prior, divisor, long_docs, "shuffled", seed, truncated)) + for name in ("1m-stat-mix80", "1m-stat-mix80-2x256k"): + original = next(case for case in cases if case.name == name) + for order in ("ascending", "descending", "alternating"): + cases.append( + replace(original, name=f"{name}-{order}", lengths=_ordered(original.lengths, order, seed), order=order) + ) + return tuple(cases) + + +def counted_cases(seed=DEFAULT_SEED): + """1M with 256--1024 documents, controlling count independently of length.""" + + def balanced(tokens, count): + blocks, extra = divmod(tokens // 4, count) + if tokens % 4 or blocks < 1: + raise ValueError("invalid aligned token budget") + return [4 * (blocks + (i < extra)) for i in range(count)] + + tiny512 = [64] * 410 + tiny1000 = [64] * 800 + specs = [ + ("uniform-256", [4096] * 256, ()), + ("uniform-1024", [1024] * 1024, ()), + ("near-uniform-512", [1536, 2560] * 256, ()), + ("tiny-mix-512", tiny512 + [8192] * 90 + balanced(TOTAL_TOKENS - sum(tiny512) - 8192 * 90, 12), ()), + ("tiny-mix-1000", tiny1000 + [2048] * 190 + balanced(TOTAL_TOKENS - sum(tiny1000) - 2048 * 190, 10), ()), + ("mixed-256-1long", [64] * 205 + [8192] * 50, (TOTAL_TOKENS - 64 * 205 - 8192 * 50,)), + ("mixed-512-2long", tiny512 + balanced(TOTAL_TOKENS // 2 - sum(tiny512), 100), (256 << 10,) * 2), + ("mixed-1000-1long", tiny1000 + balanced(TOTAL_TOKENS // 2 - sum(tiny1000), 199), (512 << 10,)), + ("mixed-512-4long", tiny512 + balanced(TOTAL_TOKENS // 2 - sum(tiny512), 98), (128 << 10,) * 4), + ] + cases = [ + THDCase(f"1m-count-{name}", _ordered([*short, *long], "shuffled", seed), "fixed-count", 1, long, "shuffled", seed) + for name, short, long in specs + ] + base = next(c for c in cases if c.name == "1m-count-mixed-512-2long") + for order in ("ascending", "descending"): + cases.append(replace(base, name=f"{base.name}-{order}", lengths=_ordered(base.lengths, order, seed), order=order)) + return tuple(cases) + + +def heldout_cases(seed=20261114): + """Uncalibrated, variable-length cases with 257--1000 documents. + + Both lengths and order change with the seed. Budget apportionment uses + largest remainders after reserving one block per sample, so it cannot + introduce empty documents or hide padding tokens in a final correction. + """ + rng = random.Random(seed) + + def allocate(tokens, weights): + blocks = tokens // 4 - len(weights) + total_weight = sum(weights) + quotas = [blocks * weight / total_weight for weight in weights] + values = [math.floor(quota) for quota in quotas] + for i in sorted(range(len(weights)), key=lambda i: (-(quotas[i] - values[i]), i))[: blocks - sum(values)]: + values[i] += 1 + return [4 * (value + 1) for value in values] + + specs = [] + for count, sigma in ((257, 1.2), (1000, 1.8)): + specs.append((f"lognormal-{count}", allocate(TOTAL_TOKENS, [rng.lognormvariate(0, sigma) for _ in range(count)]), ())) + specs.append(("jittered-768", allocate(TOTAL_TOKENS, [rng.uniform(0.7, 1.3) for _ in range(768)]), ())) + tiny = [4 * rng.randint(3, 25) for _ in range(614)] + specs.append( + ("tiny-tail-768", tiny + allocate(TOTAL_TOKENS - sum(tiny), [rng.lognormvariate(0, 2) for _ in range(154)]), ()) + ) + short = [4 * rng.randint(3, 25) for _ in range(410)] + [4 * rng.randint(128, 1024) for _ in range(101)] + fraction = rng.uniform(0.2, 0.8) + long = tuple(allocate(TOTAL_TOKENS - sum(short), [fraction, 1 - fraction])) + specs.append(("trimodal-513", short + list(long), long)) + return tuple( + THDCase(f"1m-heldout-{name}", _ordered(lengths, "shuffled", seed), "heldout-weighted", 1, long, "shuffled", seed) + for name, lengths, long in specs + ) + + +def summarize(case): + lengths = case.lengths + boundaries = (0, 10, 100, 1024, 2048, 4096, 8192, 16384, 65536, 131072, 262144, 524288, TOTAL_TOKENS) + histogram = [] + for lower, upper in zip(boundaries, boundaries[1:]): + values = [n for n in lengths if lower < n <= upper] + histogram.append( + dict( + lower_exclusive=lower, + upper_inclusive=upper, + documents=len(values), + tokens=sum(values), + document_fraction=len(values) / len(lengths), + token_fraction=sum(values) / TOTAL_TOKENS, + ) + ) + return dict( + documents=len(lengths), + total_tokens=sum(lengths), + min_tokens=min(lengths), + max_tokens=max(lengths), + tiny_document_fraction=sum(n <= 100 for n in lengths) / len(lengths), + tiny_token_fraction=sum(n for n in lengths if n <= 100) / TOTAL_TOKENS, + explicit_long_token_fraction=sum(case.long_documents) / TOTAL_TOKENS, + lengths_sha256=hashlib.sha256(json.dumps(lengths, separators=(",", ":")).encode()).hexdigest(), + token_length_histogram=histogram, + ) + + +def manifest(seed=DEFAULT_SEED, suite="statistics"): + if suite not in ("statistics", "counted", "heldout", "all"): + raise ValueError(f"unknown THD suite: {suite}") + cases = () + if suite in ("statistics", "all"): + cases += statistics_cases(seed) + if suite in ("counted", "all"): + cases += counted_cases(seed) + if suite in ("heldout", "all"): + cases += heldout_cases(seed) + return dict( + schema_version=1, + provenance="Synthetic priors inspired by a character-length document-count screenshot; not tokenized corpus data.", + assumptions={ + "total_tokens": TOTAL_TOKENS, + "alignment": 4, + "k": 1024, + "character_bins_lower_exclusive_upper_inclusive": CHARACTER_BINS, + "document_count_prior_weights": COUNT_PRIORS, + "character_divisor": "1/2/4 are sensitivity assumptions, not a measured tokenizer conversion.", + "within_bin": "Uniform over aligned token lengths after scaling.", + "packing": "Fill short-mixture budget, truncate only the last draw, add explicit long docs, then reorder.", + "tail": "64K-512K spikes are extra stress cases; screenshot tail probabilities are unknown.", + "fixed_count": "256/512/1000/1024 documents; tiny mixtures use about 80% 64-token documents by count, with the remaining budget assigned explicitly. These are token stress shapes, not a measured character-to-token conversion.", + }, + cases=[dict(**asdict(case), statistics=summarize(case)) for case in cases], + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--suite", choices=("statistics", "counted", "heldout", "all"), default="statistics") + parser.add_argument("--output", type=Path, help="Write exact lengths, assumptions and count/token histograms as JSON") + args = parser.parse_args() + data = manifest(args.seed, args.suite) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(data, indent=2) + "\n") + print("| Case | Documents | <=100 count % | <=100 token % | Max tokens | Explicit long token % |") + print("|---|---:|---:|---:|---:|---:|") + for case in data["cases"]: + stats = case["statistics"] + print( + f"| {case['name']} | {stats['documents']} | {100 * stats['tiny_document_fraction']:.2f} | " + f"{100 * stats['tiny_token_fraction']:.2f} | {stats['max_tokens']} | " + f"{100 * stats['explicit_long_token_fraction']:.1f} |" + ) + + +if __name__ == "__main__": + main() diff --git a/cula/_metadata.py b/cula/_metadata.py index 850058fc..e6a02500 100644 --- a/cula/_metadata.py +++ b/cula/_metadata.py @@ -1,7 +1,7 @@ # Copyright 2025-2026 Ant Group Co., Ltd. # SPDX-License-Identifier: Apache-2.0 -"""CPU-only packed-sequence and contiguous-shard metadata helpers.""" +"""Host sequence/shard planning and explicit device index materialization.""" from __future__ import annotations @@ -56,4 +56,24 @@ def validate_shards(shard_offsets: Sequence[int], *, total_tokens: int, world_si return offsets +def _range_indices(ranges: Sequence[tuple[int, int]], device: torch.device) -> torch.Tensor: + """Materialize validated half-open ranges without a large CPU index array. + + CUDA receives only O(ranges) host metadata. The known output size avoids + repeat_interleave's device-to-host size synchronization. This helper does + not synchronize streams; callers retain normal PyTorch stream ordering. + """ + sizes, shifts, total = [], [], 0 + for start, end in ranges: + sizes.append(end - start) + shifts.append(start - total) + total += end - start + if not total: + return torch.empty(0, dtype=torch.int64, device=device) + metadata = torch.tensor((sizes, shifts), dtype=torch.int64, device=device) + result = torch.arange(total, dtype=torch.int64, device=device) + result += torch.repeat_interleave(metadata[1], metadata[0], output_size=total) + return result + + __all__ = ["as_boundaries", "balanced_shards", "validate_shards"] diff --git a/cula/distributed/__init__.py b/cula/distributed/__init__.py index 7930bbde..41e289c1 100644 --- a/cula/distributed/__init__.py +++ b/cula/distributed/__init__.py @@ -9,13 +9,17 @@ cudnn_kda_cp, ) from cula.distributed.plan import KimiCPPlan, build_kimi_cp_plan +from cula.distributed.redistribute import THDRedistributionPlan, THDRedistributor from cula.distributed.thd_planner import ( THDLayoutMetrics, THDLayoutPlan, THDPlannerConfig, THDRankLayout, + THDSelectionCostModel, THDSpan, plan_thd_layout, + plan_thd_selection, + zigzag_thd_layout, ) __all__ = [ @@ -26,8 +30,13 @@ "THDLayoutPlan", "THDPlannerConfig", "THDRankLayout", + "THDRedistributionPlan", + "THDRedistributor", + "THDSelectionCostModel", "THDSpan", "build_kimi_cp_plan", "cudnn_kda_cp", "plan_thd_layout", + "plan_thd_selection", + "zigzag_thd_layout", ] diff --git a/cula/distributed/kda.py b/cula/distributed/kda.py index 916583c5..3abbf43b 100644 --- a/cula/distributed/kda.py +++ b/cula/distributed/kda.py @@ -719,7 +719,12 @@ def backward(ctx, grad_output: torch.Tensor): # The reverse carry is determined only by the first local span. cuDNN # backward apply consumes the summary transition in its stored M^T domain. first_end = plan.first_boundary_end - g_local, transition_backward, backward_context = kimi_delta_attention_summary_bwd( + # Native context contains per-token intermediates, not just the boundary + # summary. A prefix summary's context cannot serve the full THD shard. + # Keep the prefix-only scan for multi-document shards and let apply + # prepare its own intermediates; reuse only for identical token ranges. + reuse_context = first_end == q.shape[0] + summary_outputs = kimi_delta_attention_summary_bwd( q[:first_end], k[:first_end], g[:first_end], @@ -728,11 +733,13 @@ def backward(ctx, grad_output: torch.Tensor): cu_seqlens[:2], scale=options.scale, output_transition=True, - output_context=True, + output_context=reuse_context, checkpoint_every_n_tokens=options.checkpoint_every_n_tokens, plan_name=options.summary_plan_name, **options.kwargs(a_log, dt_bias), ) + g_local, transition_backward = summary_outputs[:2] + backward_context = summary_outputs[2] if reuse_context else None following_h, following_transition = _validate_boundary_summaries( ctx.boundary_transport.boundary_summaries( g_local[0], @@ -868,6 +875,11 @@ def cudnn_kda_cp( layers. Inputs use cuDNN's canonical THD contract; framework-specific layout and gate conversions belong in the framework adapter. + If every shard boundary is a document boundary, the plan marks the whole + group as independent. Execution then uses native local KDA/autograd and + skips boundary summaries and transport. An isolated rank alone cannot + enable this path while other ranks still require an exchange. + ``boundary_transport`` defaults to exact ordered summary all-gather with lazy storage. Large-CP callers should prepare one :class:`AllGatherKimiCPTransport` per concurrent CUDA stream and reuse it @@ -916,7 +928,13 @@ def cudnn_kda_cp( plan_name=plan_name, summary_plan_name=summary_plan_name, ) - if plan.world_size == 1: + if plan.world_size == 1 or not plan.requires_boundary_exchange: + # Every shard boundary is a document boundary. This is the same + # independent native recurrence as CP1, including native autograd; + # no forward or backward state can cross ranks. The plan flag is + # identical on all peers, so no rank skips a required collective. + if plan.world_size > 1 and _resolve_kimi_cp_backend(backend, q.device) == "hopper-state-probe": + _validate_hopper_state_probe_options(options, a_log, dt_bias) kimi_delta_attention, _ = _load_cudnn_kda_ops() attention_outputs = kimi_delta_attention( q, diff --git a/cula/distributed/plan.py b/cula/distributed/plan.py index 3890e70d..9e8d6132 100644 --- a/cula/distributed/plan.py +++ b/cula/distributed/plan.py @@ -29,6 +29,10 @@ class KimiCPPlan: plan records their token spans and the ordered rank chains that contribute incoming forward state or outgoing backward state. Tensor shapes and process-group identity are checked again at execution time. + + ``requires_boundary_exchange`` is a group-wide property derived from all + shard boundaries, identical on every rank. Manual plans default to the + conservative exchanging path. """ rank: int @@ -42,6 +46,9 @@ class KimiCPPlan: last_boundary_start: int pre_num_ranks: int post_num_ranks: int + # Group-wide, not rank-local: one isolated rank must still participate if + # any other rank needs the boundary collective. Manual plans stay safe. + requires_boundary_exchange: bool = True @property def local_tokens(self) -> int: @@ -130,6 +137,9 @@ def from_global_cu_seqlens( if pre_num_ranks < 0 or post_num_ranks < 0: raise RuntimeError("internal error while deriving Kimi CP boundary rank ranges") + document_boundaries = set(boundaries) + requires_boundary_exchange = any(offset not in document_boundaries for offset in offsets[1:-1]) + return cls( rank=rank, world_size=world_size, @@ -142,6 +152,7 @@ def from_global_cu_seqlens( last_boundary_start=local_boundaries[-2], pre_num_ranks=pre_num_ranks, post_num_ranks=post_num_ranks, + requires_boundary_exchange=requires_boundary_exchange, ) diff --git a/cula/distributed/redistribute.py b/cula/distributed/redistribute.py new file mode 100644 index 00000000..61c81aa5 --- /dev/null +++ b/cula/distributed/redistribute.py @@ -0,0 +1,183 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Lossless row redistribution from contiguous CP shards to THD span layouts. + +Only row ownership/order changes. Floating point values are neither reduced +nor cast. The backward is the exact inverse exchange. Preparation is CPU +metadata work plus device index construction; execution is eager PyTorch/NCCL +and allocates outputs. No CUDA Graph contract is implied. +""" + +from dataclasses import dataclass + +import torch +import torch.distributed as dist +from torch.autograd.function import once_differentiable + +from cula._metadata import _range_indices +from cula.distributed.thd_planner import THDRankLayout + + +@dataclass(frozen=True) +class THDRedistributionPlan: + rank: int + world_size: int + source_rows: int + target_rows: int + send_counts: tuple[int, ...] + receive_counts: tuple[int, ...] + send_ranges: tuple[tuple[int, int], ...] + receive_ranges: tuple[tuple[int, int], ...] + identity: bool + + @classmethod + def build(cls, ranks: tuple[THDRankLayout, ...], source_offsets: tuple[int, ...], rank: int): + world = len(ranks) + if world < 1 or len(source_offsets) != world + 1 or source_offsets[0] != 0: + raise ValueError("source_offsets must describe one contiguous shard per rank starting at zero") + if any(isinstance(x, bool) or not isinstance(x, int) for x in source_offsets): + raise TypeError("source offsets must be integers") + if any(a > b for a, b in zip(source_offsets, source_offsets[1:])): + raise ValueError("source offsets must be nondecreasing") + if isinstance(rank, bool) or not isinstance(rank, int) or not 0 <= rank < world: + raise ValueError("invalid rank") + intervals = [] + for owner, layout in enumerate(ranks): + if isinstance(layout.rank, bool) or not isinstance(layout.rank, int): + raise TypeError("layout ranks must be integers") + if layout.rank != owner: + raise ValueError("rank layouts must be in group-rank order") + for span in layout.spans: + if any( + isinstance(value, bool) or not isinstance(value, int) + for value in (span.owner, span.token_start, span.token_end, span.sequence_start, span.sequence_end) + ): + raise TypeError("span owners and interval bounds must be integers") + if span.owner != owner: + raise ValueError("span owner does not match its rank layout") + if not (span.sequence_start <= span.token_start < span.token_end <= span.sequence_end): + raise ValueError("span must be a nonempty interval within its sequence") + intervals.append((span.token_start, span.token_end)) + cursor = 0 + for start, end in sorted(intervals): + if start != cursor: + raise ValueError("target layout must cover every source row exactly once, without gaps or overlap") + cursor = end + if cursor != source_offsets[-1]: + raise ValueError("target layout does not cover the source extent") + + send_ranges, receive_ranges = [], [] + send_counts, receive_counts = [0] * world, [0] * world + identity = True + left, right = source_offsets[rank : rank + 2] + for destination, layout in enumerate(ranks): + target_offset = 0 + for span in layout.spans: + # Global identity is required: one locally unchanged rank must + # still join the collective if any peer changes ownership. + identity &= span.token_start == source_offsets[destination] + target_offset + a, z = max(left, span.token_start), min(right, span.token_end) + if a < z: + send_counts[destination] += z - a + send_ranges.append((a - left, z - left)) + target_offset += span.tokens + identity &= target_offset == source_offsets[destination + 1] - source_offsets[destination] + + # Receive buffer groups rows by source, then by destination span order. + for source, (left, right) in enumerate(zip(source_offsets, source_offsets[1:])): + target_offset = 0 + for span in ranks[rank].spans: + a, z = max(left, span.token_start), min(right, span.token_end) + if a < z: + receive_counts[source] += z - a + receive_ranges.append((target_offset + a - span.token_start, target_offset + z - span.token_start)) + target_offset += span.tokens + return cls( + rank, + world, + source_offsets[rank + 1] - source_offsets[rank], + ranks[rank].local_tokens, + tuple(send_counts), + tuple(receive_counts), + tuple(send_ranges), + tuple(receive_ranges), + bool(identity), + ) + + @property + def remote_rows(self): + return sum(self.send_counts) - self.send_counts[self.rank] + + +def _permutations(ranges, rows, device): + # Avoid allocating or applying an index tensor for a naturally ordered side. + cursor, ordered = 0, True + for start, end in ranges: + ordered &= start == cursor + cursor = end + if ordered and cursor == rows: + return None, None + order = _range_indices(ranges, device) + if order.numel() != rows: + raise ValueError("incomplete permutation") + inverse = torch.empty_like(order) + inverse[order] = torch.arange(rows, device=device) + return order, inverse + + +class THDRedistributor: + """Prepared eager exchange with explicit forward and inverse mappings. + + Every rank must prepare from identical global layouts and call exchanges + in the same order. Trailing shapes and dtypes must agree across peers. + The process group must stay alive for every backward using this object. + """ + + def __init__(self, plan: THDRedistributionPlan, device, *, group=None): + if dist.get_world_size(group) != plan.world_size or dist.get_rank(group) != plan.rank: + raise ValueError("plan identity does not match the process group") + self.plan, self.group, self.device = plan, group, torch.device(device) + if self.device.type == "cuda" and self.device.index is None: + self.device = torch.device("cuda", torch.cuda.current_device()) + self.send_order, self.send_inverse = _permutations(plan.send_ranges, plan.source_rows, self.device) + self.receive_order, self.receive_inverse = _permutations(plan.receive_ranges, plan.target_rows, self.device) + + def _exchange(self, tensor, reverse): + plan = self.plan + rows = plan.target_rows if reverse else plan.source_rows + if tensor.ndim < 1 or tensor.shape[0] != rows or tensor.device != self.device: + raise ValueError(f"expected {rows} rows on {self.device}, got {tuple(tensor.shape)} on {tensor.device}") + if not tensor.is_contiguous(): + raise ValueError("redistribution requires contiguous rows; explicit packing belongs to the caller") + if plan.identity: + return tensor + order = self.receive_order if reverse else self.send_order + inverse = self.send_inverse if reverse else self.receive_inverse + send_counts, receive_counts = ( + (plan.receive_counts, plan.send_counts) if reverse else (plan.send_counts, plan.receive_counts) + ) + packed = tensor if order is None else tensor.index_select(0, order) + received = tensor.new_empty((sum(receive_counts), *tensor.shape[1:])) + dist.all_to_all_single( + received, packed, output_split_sizes=list(receive_counts), input_split_sizes=list(send_counts), group=self.group + ) + return received if inverse is None else received.index_select(0, inverse) + + def distribute(self, tensor): + return _Redistribute.apply(tensor, self, False) + + def restore(self, tensor): + return _Redistribute.apply(tensor, self, True) + + +class _Redistribute(torch.autograd.Function): + @staticmethod + def forward(ctx, tensor, exchange, reverse): + ctx.exchange, ctx.reverse = exchange, reverse + return exchange._exchange(tensor, reverse) + + @staticmethod + @once_differentiable + def backward(ctx, grad): + return ctx.exchange._exchange(grad.contiguous(), not ctx.reverse), None, None diff --git a/cula/distributed/thd_planner.py b/cula/distributed/thd_planner.py index b7646fc0..5b8890a1 100644 --- a/cula/distributed/thd_planner.py +++ b/cula/distributed/thd_planner.py @@ -11,9 +11,12 @@ from __future__ import annotations +from bisect import bisect_left, bisect_right from collections import defaultdict from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace +from heapq import heapify, heappop, heappush +from itertools import groupby from math import ceil, isfinite from operator import index @@ -75,7 +78,7 @@ class THDLayoutMetrics: @dataclass(frozen=True, slots=True) class THDLayoutPlan: - """Complete global span assignment selected by :func:`plan_thd_layout`.""" + """Complete global span assignment selected by a THD planner.""" world_size: int block_size: int @@ -83,6 +86,7 @@ class THDLayoutPlan: ranks: tuple[THDRankLayout, ...] metrics: THDLayoutMetrics evaluated_scores: tuple[tuple[str, float], ...] + score_unit: str = "normalized" def for_rank(self, rank: int) -> THDRankLayout: if not isinstance(rank, int) or not 0 <= rank < self.world_size: @@ -106,6 +110,176 @@ class THDPlannerConfig: token_weight: float = 1.0 causal_weight: float = 1.0 boundary_weight: float = 0.02 + min_chunk_tokens: int = 0 + """Minimum newly created selection span; zero preserves the legacy search. + + Existing source fragments and complete short documents remain legal. + This does not change the explicitly requested full-zigzag baseline. + """ + + +@dataclass(frozen=True, slots=True) +class THDSelectionCostModel: + """Calibrated eager selection-routing costs; no hardware detection or I/O. + + Coefficients must describe the caller's indexer shape and transport. Q/O + attention and backward stay in source ownership. ``bytes_per_query`` sums + outgoing indexer Q/weights and returning IDs/scores, without dtype changes. + Bandwidths are effective per-rank GB/s, not advertised link speeds. Group + rank -> node IDs is explicit; the source is a contiguous balanced shard. + ``min_gain_fraction`` requires headroom over contiguous before migrating. + ``min_gain_ms`` additionally requires an absolute predicted saving per use; + callers may price one-time preparation divided by expected layout reuse. + ``peer_gbps`` optionally prices the maximum aggregated source/destination + edge, since sparse peer traffic may not saturate aggregate rank bandwidth. + ``inter_node_aggregate_gbps`` optionally bounds shared node injection and + receive bandwidth. ``peer_startup_us`` prices the busiest rank's nonempty + peer count, summed over the ordered forward/return collectives. Calibrate + these independently of kernel arrival waits; they are effective costs, + not physical-link specifications. Unspecified limits preserve old profiles. + """ + + candidate_ns: float + query_ns: float + span_us: float + bytes_per_query: int + intra_node_gbps: float + inter_node_gbps: float + rank_node_ids: tuple[int, ...] + packing_ns_per_query: float = 0.0 + exchange_us: float = 0.0 + min_gain_fraction: float = 0.05 + peer_gbps: float | None = None + min_gain_ms: float = 0.0 + peer_startup_us: float = 0.0 + inter_node_aggregate_gbps: float | None = None + overlap_intra_inter: bool = False + + def __post_init__(self): + if not isinstance(self.overlap_intra_inter, bool): + raise TypeError("overlap_intra_inter must be a boolean") + for name in ( + "candidate_ns", + "query_ns", + "span_us", + "packing_ns_per_query", + "exchange_us", + "min_gain_fraction", + "min_gain_ms", + "peer_startup_us", + ): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite and nonnegative") + for name in ("intra_node_gbps", "inter_node_gbps"): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive") + if self.min_gain_fraction >= 1: + raise ValueError("min_gain_fraction must be below one") + for name in ("peer_gbps", "inter_node_aggregate_gbps"): + value = getattr(self, name) + if value is not None and ( + isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(value) or value <= 0 + ): + raise ValueError(f"{name} must be finite and positive when supplied") + if isinstance(self.bytes_per_query, bool) or not isinstance(self.bytes_per_query, int) or self.bytes_per_query <= 0: + raise ValueError("bytes_per_query must be a positive integer") + if not isinstance(self.rank_node_ids, tuple) or not self.rank_node_ids: + raise TypeError("rank_node_ids must be a nonempty tuple") + if any(isinstance(n, bool) or not isinstance(n, int) or n < 0 for n in self.rank_node_ids): + raise ValueError("node ids must be nonnegative integers") + + def _migration_threshold_ms(self, baseline_ms): + return baseline_ms - max(baseline_ms * self.min_gain_fraction, self.min_gain_ms) + + def _outgoing_capacity_gbps(self, rank): + kinds = {node == self.rank_node_ids[rank] for peer, node in enumerate(self.rank_node_ids) if peer != rank} + rates = [self.intra_node_gbps if local else self.inter_node_gbps for local in kinds] + return sum(rates) if self.overlap_intra_inter and rates else max(rates, default=self.intra_node_gbps) + + def estimate_ms(self, ranks: tuple[THDRankLayout, ...], *, block_size: int) -> float: + world = len(ranks) + if len(self.rank_node_ids) != world: + raise ValueError("cost-model node map must have one entry per process-group rank") + total = sum(r.local_tokens for r in ranks) + offsets = tuple(x * block_size for x in balanced_shards(total // block_size, world)) + compute, incoming, outgoing = [0.0] * world, [0.0] * world, [0.0] * world + intra_in, intra_out, inter_in, inter_out = ([0.0] * world for _ in range(4)) + peer_rows = defaultdict(int) + source_cursor = list(offsets[:-1]) + send_ordered, receive_ordered = [True] * world, [True] * world + identity = True + for rank, layout in enumerate(ranks): + cursor = offsets[rank] + last_source = -1 + for span in layout.spans: + start, end = span.token_start, span.token_end + tokens = end - start + compute[rank] += ( + self.candidate_ns * _candidate_work(start - span.sequence_start, tokens, block_size) / 1e6 + + self.query_ns * tokens / 1e6 + + self.span_us / 1e3 + ) + identity &= start == cursor + cursor += tokens + # Pinned source spans are common in mixed packs. Their exact + # one-source intersection needs neither a bisect nor a walk. + if offsets[rank] <= start and end <= offsets[rank + 1]: + send_ordered[rank] &= start == source_cursor[rank] + source_cursor[rank] += tokens + receive_ordered[rank] &= rank >= last_source + last_source = rank + continue + for source, rows in _source_intersections(span, offsets): + send_ordered[source] &= max(offsets[source], span.token_start) == source_cursor[source] + source_cursor[source] += rows + receive_ordered[rank] &= source >= last_source + last_source = source + if source != rank: + gbps = ( + self.intra_node_gbps + if self.rank_node_ids[source] == self.rank_node_ids[rank] + else self.inter_node_gbps + ) + cost = rows * self.bytes_per_query / (gbps * 1e6) + outgoing[source] += cost + incoming[rank] += cost + local = self.rank_node_ids[source] == self.rank_node_ids[rank] + (intra_out if local else inter_out)[source] += cost + (intra_in if local else inter_in)[rank] += cost + peer_rows[source, rank] += rows + identity &= cursor == offsets[rank + 1] + if identity: + return max(compute) + # Execution has ordered exchange/compute/return phases. Model their + # critical durations separately rather than assuming unmeasured overlap. + transfer = max(*incoming, *outgoing) + if self.overlap_intra_inter: + transfer = max(*intra_in, *intra_out, *inter_in, *inter_out) + if self.peer_gbps is not None: + transfer = max(transfer, max(peer_rows.values(), default=0) * self.bytes_per_query / (self.peer_gbps * 1e6)) + node_in, node_out = defaultdict(int), defaultdict(int) + send_peers, receive_peers = [0] * world, [0] * world + for (source, target), rows in peer_rows.items(): + send_peers[source] += 1 + receive_peers[target] += 1 + a, b = self.rank_node_ids[source], self.rank_node_ids[target] + if a != b: + node_out[a] += rows + node_in[b] += rows + if self.inter_node_aggregate_gbps is not None: + node_rows = max((*node_in.values(), *node_out.values()), default=0) + transfer = max(transfer, node_rows * self.bytes_per_query / (self.inter_node_aggregate_gbps * 1e6)) + transfer += max(*send_peers, *receive_peers) * self.peer_startup_us / 1e3 + # Match the redistributor's no-op permutation elimination. A local + # forward pack and its inverse restore together copy bytes_per_query; + # the destination-side pair does the same only if receive order differs. + copies = max( + (offsets[r + 1] - offsets[r]) * (not send_ordered[r]) + layout.local_tokens * (not receive_ordered[r]) + for r, layout in enumerate(ranks) + ) + return max(compute) + transfer + copies * self.packing_ns_per_query / 1e6 + self.exchange_us / 1e3 @dataclass(frozen=True, slots=True) @@ -131,6 +305,45 @@ def _candidate_work(start: int, tokens: int, block_size: int) -> int: return _candidate_prefix(start + tokens, block_size) - _candidate_prefix(start, block_size) +def _source_intersections(span: THDSpan, offsets: tuple[int, ...]): + for source in range(bisect_right(offsets, span.token_start) - 1, bisect_left(offsets, span.token_end)): + yield source, min(offsets[source + 1], span.token_end) - max(offsets[source], span.token_start) + + +def _transfer_lower_bound(assignments, world, block_size, model, compute_mean, baseline): + """Optimistic lower bound on critical compute plus outgoing traffic. + + To reduce source rank i's compute below C, some candidate/token work must + leave it. A moved row can remove no more than the largest causal position + on that source. Ignore span overhead, incoming traffic, packing, capacity + and fixed exchange cost: all make this a lower bound, not a prediction. + Minimizing the maximum of affine functions only needs their intersections + and the interval endpoints. It remains valid for unequal destination rows. + """ + compute, density = [0.0] * world, [0.0] * world + for item, rank in assignments: + compute[rank] += (model.candidate_ns * item.causal_work + model.query_ns * item.tokens) / 1e6 + density[rank] = max( + density[rank], + (model.candidate_ns * ((item.token_end - item.sequence_start) // block_size) + model.query_ns) / 1e6, + ) + lines = [(1.0, 0.0)] + for rank in range(world): + if not density[rank]: + continue + rate = model._outgoing_capacity_gbps(rank) + slope = model.bytes_per_query / (rate * 1e6 * density[rank]) + lines.append((1 - slope, slope * compute[rank])) + points = [compute_mean, baseline] + for i, (a, b) in enumerate(lines): + for c, d in lines[:i]: + if a != c: + cross = (d - b) / (a - c) + if compute_mean < cross < baseline: + points.append(cross) + return min(max(a * x + b for a, b in lines) for x in points) + + def _validate_inputs( global_cu_seqlens: torch.Tensor | Sequence[int], *, @@ -149,6 +362,13 @@ def _validate_inputs( raise ValueError(f"{name} must be a positive integer") if config.min_split_tokens % config.block_size: raise ValueError("min_split_tokens must be block aligned") + if ( + isinstance(config.min_chunk_tokens, bool) + or not isinstance(config.min_chunk_tokens, int) + or config.min_chunk_tokens < 0 + or config.min_chunk_tokens % config.block_size + ): + raise ValueError("min_chunk_tokens must be nonnegative and block aligned") if any((right - left) % config.block_size for left, right in zip(boundaries, boundaries[1:])): raise ValueError("every sequence length must be block aligned") if boundaries[-1] // config.block_size < world_size: @@ -186,7 +406,8 @@ def _contiguous_items(boundaries: tuple[int, ...], world_size: int, block_size: shard_offsets = tuple(value * block_size for value in block_shards) result = [] for rank, (shard_start, shard_end) in enumerate(zip(shard_offsets, shard_offsets[1:])): - for sequence, (sequence_start, sequence_end) in enumerate(zip(boundaries, boundaries[1:])): + for sequence in range(bisect_right(boundaries, shard_start) - 1, bisect_left(boundaries, shard_end)): + sequence_start, sequence_end = boundaries[sequence : sequence + 2] start = max(shard_start, sequence_start) end = min(shard_end, sequence_end) if start < end: @@ -200,6 +421,7 @@ def _adaptive_items( config: THDPlannerConfig, *, chunks_per_rank: int, + sequences: set[int] | None = None, ) -> list[_PlanningItem]: block_size = config.block_size total_blocks = boundaries[-1] // block_size @@ -207,13 +429,16 @@ def _adaptive_items( target_blocks = max(1, ceil(total_blocks / requested_grid)) min_split_blocks = config.min_split_tokens // block_size result = [] - for sequence, (sequence_start, sequence_end) in enumerate(zip(boundaries, boundaries[1:])): + for sequence in range(len(boundaries) - 1) if sequences is None else sorted(sequences): + sequence_start, sequence_end = boundaries[sequence : sequence + 2] length_blocks = (sequence_end - sequence_start) // block_size pieces = min(requested_grid, max(1, ceil(length_blocks / target_blocks))) # A sequence shorter than two useful spans stays whole. This is what # prevents short THD samples from paying a max-CP fragmentation tax. if length_blocks < 2 * min_split_blocks: pieces = 1 + if config.min_chunk_tokens: + pieces = min(pieces, max(1, length_blocks // (config.min_chunk_tokens // block_size))) offsets = balanced_shards(length_blocks, pieces) for left, right in zip(offsets, offsets[1:]): start = sequence_start + left * block_size @@ -223,7 +448,7 @@ def _adaptive_items( # Keep all ranks non-empty even for small batches containing fewer than W # sequences. The minimum span is a performance preference, not a semantic # restriction, so only this feasibility repair may split below it. - while len(result) < world_size: + while sequences is None and len(result) < world_size: splittable = [item for item in result if item.tokens >= 2 * block_size] if not splittable: raise ValueError("block alignment leaves fewer non-empty spans than ranks") @@ -275,23 +500,28 @@ def _greedy_assign(items: list[_PlanningItem], world_size: int) -> list[tuple[_P reverse=True, ) result = [] - for item in ordered: - owner = min( - range(world_size), - key=lambda rank: ( - max( - (token_loads[rank] + item.tokens) / ideal_tokens, - (causal_loads[rank] + item.causal_work) / ideal_causal, - ), - (token_loads[rank] + item.tokens) / ideal_tokens + (causal_loads[rank] + item.causal_work) / ideal_causal, - item_counts[rank], - rank, - ), - ) - result.append((item, owner)) - token_loads[owner] += item.tokens - causal_loads[owner] += item.causal_work - item_counts[owner] += 1 + for (tokens, work), group in groupby(ordered, key=lambda item: (item.tokens, item.causal_work)): + + def pressure(rank): + token_pressure = (token_loads[rank] + tokens) / ideal_tokens + causal_pressure = (causal_loads[rank] + work) / ideal_causal + return max(token_pressure, causal_pressure), token_pressure + causal_pressure, item_counts[rank], rank + + # Consecutive equal-cost pieces change only the selected rank's key. + # A heap retains the exact LPT ordering without scanning all CP ranks + # for every tiny document. Unique-cost pieces keep the simple minimum. + pieces = list(group) + heap = [pressure(rank) for rank in range(world_size)] + if len(pieces) > 1: + heapify(heap) + for item in pieces: + owner = heappop(heap)[-1] if len(pieces) > 1 else min(heap)[-1] + result.append((item, owner)) + token_loads[owner] += tokens + causal_loads[owner] += work + item_counts[owner] += 1 + if len(pieces) > 1: + heappush(heap, pressure(owner)) return result @@ -341,7 +571,19 @@ def _rebalance_tokens( loads[owner] += item.tokens causal_loads[owner] += item.causal_work ideal_causal = sum(causal_loads) / world_size - repaired = list(assignments) + # Stable serials preserve the old global-list tie order, while looking only + # at the selected donor avoids rescanning every unrelated short document. + bins = [dict() for _ in range(world_size)] + shapes = [defaultdict(list) for _ in range(world_size)] + + def insert(owner, serial, item): + bins[owner][serial] = item + shape = (item.tokens, item.token_start - item.sequence_start) + heappush(shapes[owner][shape], (item.sequence, item.token_start, serial)) + + for serial, (item, owner) in enumerate(assignments): + insert(owner, serial, item) + next_serial = len(assignments) while loads != capacities: receivers = [rank for rank in range(world_size) if loads[rank] < capacities[rank]] @@ -355,55 +597,56 @@ def _rebalance_tokens( loads[donor] - capacities[donor], ) - choices = [] - for assignment_index, (item, owner) in enumerate(repaired): - if owner != donor: - continue + other_pressure = max( + (causal_loads[r] / ideal_causal for r in range(world_size) if r not in (donor, receiver)), default=0.0 + ) + best = None + # Equal (length, causal offset) pieces have identical whole/prefix/ + # suffix costs. Only the earliest semantic tie-break can win, so one + # heap head represents all equivalent short documents exactly. + for heap in shapes[donor].values(): + serial = heap[0][2] + item = bins[donor][serial] transfer_tokens = min(item.tokens, transfer_limit) - if transfer_tokens == item.tokens: - variants = ((item, None),) - else: - variants = tuple( - _split_item( - item, - transfer_tokens, - take_prefix=take_prefix, - block_size=block_size, - ) - for take_prefix in (True, False) + split = transfer_tokens != item.tokens + for take_prefix in (True, False) if split else (True,): + moved_start = item.token_start if take_prefix else item.token_end - transfer_tokens + moved_work = ( + _candidate_work(moved_start - item.sequence_start, transfer_tokens, block_size) + if split + else item.causal_work ) - for moved, remainder in variants: - next_donor_causal = causal_loads[donor] - moved.causal_work - next_receiver_causal = causal_loads[receiver] + moved.causal_work causal_pressure = max( - next_donor_causal / ideal_causal, - next_receiver_causal / ideal_causal, - *(causal_loads[rank] / ideal_causal for rank in range(world_size) if rank not in (donor, receiver)), + (causal_loads[donor] - moved_work) / ideal_causal, + (causal_loads[receiver] + moved_work) / ideal_causal, + other_pressure, ) - choices.append( - ( - causal_pressure, - remainder is not None, - -moved.tokens, - moved.sequence, - moved.token_start, - assignment_index, - moved, - remainder, - ) - ) - if not choices: + choice = (causal_pressure, split, -transfer_tokens, item.sequence, moved_start, serial, take_prefix) + if best is None or choice < best: + best = choice + if best is None: raise RuntimeError("internal error: an overloaded rank has no movable THD span") - *_, assignment_index, moved, remainder = min(choices) - repaired.pop(assignment_index) + _, split, negative_tokens, _, _, serial, take_prefix = best + item = bins[donor].pop(serial) + shape = (item.tokens, item.token_start - item.sequence_start) + heappop(shapes[donor][shape]) + if not shapes[donor][shape]: + del shapes[donor][shape] + moved, remainder = ( + _split_item(item, -negative_tokens, take_prefix=take_prefix, block_size=block_size) if split else (item, None) + ) if remainder is not None: - repaired.append((remainder, donor)) - repaired.append((moved, receiver)) + insert(donor, next_serial, remainder) + next_serial += 1 + insert(receiver, next_serial, moved) + next_serial += 1 loads[donor] -= moved.tokens loads[receiver] += moved.tokens causal_loads[donor] -= moved.causal_work causal_loads[receiver] += moved.causal_work - return repaired + return [ + (item, owner) for _, item, owner in sorted((i, item, r) for r, items in enumerate(bins) for i, item in items.items()) + ] def _coalesce(assignments: list[tuple[_PlanningItem, int]]) -> list[THDSpan]: @@ -502,11 +745,91 @@ def _build_candidate( return policy, spans, metrics +def _minimum_assignment(costs: list[list[float]]) -> tuple[int, ...]: + """Minimum-cost bin -> rank permutation, O(W^3), with stable ties. + + The primal-dual augmenting-path algorithm keeps rank ownership bijective. + It needs no optional optimizer dependency and never enumerates W! maps. + """ + size = len(costs) + u, v = [0.0] * (size + 1), [0.0] * (size + 1) + matched, previous = [0] * (size + 1), [0] * (size + 1) + for row in range(1, size + 1): + matched[0] = row + column = 0 + best = [float("inf")] * (size + 1) + used = [False] * (size + 1) + while True: + used[column] = True + active = matched[column] + step, next_column = float("inf"), 0 + for candidate in range(1, size + 1): + if not used[candidate]: + reduced = costs[active - 1][candidate - 1] - u[active] - v[candidate] + if reduced < best[candidate]: + best[candidate], previous[candidate] = reduced, column + if best[candidate] < step: + step, next_column = best[candidate], candidate + for candidate in range(size + 1): + if used[candidate]: + u[matched[candidate]] += step + v[candidate] -= step + else: + best[candidate] -= step + column = next_column + if matched[column] == 0: + break + while column: + parent = previous[column] + matched[column] = matched[parent] + column = parent + assignment = [0] * size + for column in range(1, size + 1): + assignment[matched[column] - 1] = column - 1 + return tuple(assignment) + + +def _localize_spans(spans: list[THDSpan], world: int, block_size: int, model: THDSelectionCostModel) -> list[THDSpan]: + """Relabel equal-capacity bins to minimize aggregate transfer cost. + + Computation and fragmentation stay identical. This minimizes a sum, not + the critical path; the caller must still score the result against the + original candidate with the full max-rank model. + """ + offsets = tuple(x * block_size for x in balanced_shards(sum(s.tokens for s in spans) // block_size, world)) + overlaps = [[0] * world for _ in range(world)] + for span in spans: + for source, rows in _source_intersections(span, offsets): + overlaps[span.owner][source] += rows + capacities = defaultdict(list) + for rank, (a, z) in enumerate(zip(offsets, offsets[1:])): + capacities[z - a].append(rank) + owners = list(range(world)) + for peers in capacities.values(): + costs = [ + [ + sum( + rows + / (model.intra_node_gbps if model.rank_node_ids[s] == model.rank_node_ids[r] else model.inter_node_gbps) + for s, rows in enumerate(overlaps[owner]) + if s != r + ) + for r in peers + ] + for owner in peers + ] + assignment = _minimum_assignment(costs) + for owner, destination in zip(peers, assignment): + owners[owner] = peers[destination] + return [replace(span, owner=owners[span.owner]) for span in spans] + + def plan_thd_layout( global_cu_seqlens: torch.Tensor | Sequence[int], *, world_size: int, config: THDPlannerConfig | None = None, + cost_model: THDSelectionCostModel | None = None, ) -> THDLayoutPlan: """Plan a block-aligned THD layout and select the best modeled candidate. @@ -518,16 +841,44 @@ def plan_thd_layout( config = THDPlannerConfig() if config is None else config if not isinstance(config, THDPlannerConfig): raise TypeError("config must be a THDPlannerConfig") + if cost_model is not None and not isinstance(cost_model, THDSelectionCostModel): + raise TypeError("cost_model must be a THDSelectionCostModel") boundaries = _validate_inputs(global_cu_seqlens, world_size=world_size, config=config) + if cost_model is not None and len(cost_model.rank_node_ids) != world_size: + raise ValueError("cost-model node map must have one entry per process-group rank") + source_items = _contiguous_items(boundaries, world_size, config.block_size) contiguous = _build_candidate( - _contiguous_items(boundaries, world_size, config.block_size), + source_items, policy="contiguous", world_size=world_size, config=config, ) + if cost_model is not None and contiguous is not None: + policy, spans, metrics = contiguous + ranks = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world_size)) + baseline_ms = cost_model.estimate_ms(ranks, block_size=config.block_size) + # Every exact partition has the same candidate/token sums and at least + # one span per document. Even free transport cannot beat their mean. + lower_bound = ( + cost_model.candidate_ns * sum(metrics.causal_loads) / 1e6 + + cost_model.query_ns * boundaries[-1] / 1e6 + + cost_model.span_us * (len(boundaries) - 1) / 1e3 + ) / world_size + lower_bound = _transfer_lower_bound(source_items, world_size, config.block_size, cost_model, lower_bound, baseline_ms) + if lower_bound >= cost_model._migration_threshold_ms(baseline_ms): + return THDLayoutPlan( + world_size, + config.block_size, + policy, + ranks, + replace(metrics, score=baseline_ms), + ((policy, baseline_ms),), + "ms", + ) adaptive_candidates = [] seen_assignments = set() + seen_items = set() for chunks_per_rank in range(1, config.max_chunks_per_rank + 1): adaptive_items = _adaptive_items( boundaries, @@ -535,6 +886,10 @@ def plan_thd_layout( config, chunks_per_rank=chunks_per_rank, ) + item_signature = tuple(adaptive_items) + if item_signature in seen_items: + continue + seen_items.add(item_signature) adaptive = _build_candidate( _rebalance_tokens( _greedy_assign(adaptive_items, world_size), @@ -554,6 +909,35 @@ def plan_thd_layout( candidates = [candidate for candidate in (contiguous, *adaptive_candidates) if candidate is not None] if not candidates: raise RuntimeError("no feasible THD layout candidate") + if cost_model is not None: + localized = [] + for policy, spans, _ in candidates: + if policy == "contiguous": + continue + nearby = _localize_spans(spans, world_size, config.block_size, cost_model) + if nearby != spans: + localized.append( + ( + policy + "-local", + nearby, + _metrics(nearby, world_size=world_size, block_size=config.block_size, config=config), + ) + ) + candidates += localized + candidates = [ + ( + policy, + spans, + replace( + metrics, + score=cost_model.estimate_ms( + tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world_size)), + block_size=config.block_size, + ), + ), + ) + for policy, spans, metrics in candidates + ] selected_policy, selected_spans, selected_metrics = min( candidates, key=lambda candidate: ( @@ -562,6 +946,10 @@ def plan_thd_layout( candidate[0], ), ) + if cost_model is not None: + baseline = next(c for c in candidates if c[0] == "contiguous") + if selected_metrics.score >= cost_model._migration_threshold_ms(baseline[2].score): + selected_policy, selected_spans, selected_metrics = baseline ranks = tuple( THDRankLayout( rank, @@ -581,6 +969,470 @@ def plan_thd_layout( ranks, selected_metrics, tuple((policy, metrics.score) for policy, _, metrics in candidates), + "ms" if cost_model is not None else "normalized", + ) + + +def _selection_assign(items, *, pinned_loads, pinned_compute, world, capacity, offsets, model, block_size): + """Keep cheap source spans local; place long-span work under a row cap. + + Unlike a shared KDA/CSA layout, selection ownership need not have equal + token counts. Its invariant is bounded rows and exact coverage. Placement + scores ordered compute/transport phases using the same calibrated units + as final candidate selection, with stable load/locality tie breaks. + """ + # Copy the small per-rank state; all grids share the same pinned spans. + loads, compute = list(pinned_loads), list(pinned_compute) + incoming, outgoing = [0.0] * world, [0.0] * world + intra_in, intra_out, inter_in, inter_out = ([0.0] * world for _ in range(4)) + peer_rows = [[0] * world for _ in range(world)] + peer_peak = 0 + send_peers, receive_peers = [0] * world, [0] * world + degree_peak, node_peak = 0, 0 + node_in, node_out = defaultdict(int), defaultdict(int) + # Pinned spans initialize loads but are materialized once by the caller. + result = [] + + def work(item): + return model.candidate_ns * item.causal_work / 1e6 + model.query_ns * item.tokens / 1e6 + model.span_us / 1e3 + + for original in sorted(items, key=lambda item: (-work(item), -item.tokens, item.sequence, item.token_start)): + item = original + while item is not None: + # A cap is not an exact bin size. Split only when no remaining + # bin can hold this item, keeping short sequences intact otherwise. + free = max(capacity - n for n in loads) + if free <= 0: + raise RuntimeError("internal error: selection row capacity exhausted") + placed, remainder = ( + _split_item(item, free, take_prefix=True, block_size=block_size) if item.tokens > free else (item, None) + ) + intersections = tuple(_source_intersections(placed, offsets)) + cost = work(placed) + max_compute, max_incoming, max_outgoing = max(compute), max(incoming), max(outgoing) + max_copies = max(offsets[r + 1] - offsets[r] + n for r, n in enumerate(loads)) + if model.overlap_intra_inter: + max_incoming, max_outgoing = max(*intra_in, *inter_in), max(*intra_out, *inter_out) + best = None + for rank in range(world): + if loads[rank] + placed.tokens > capacity: + continue + additions = [] + moved = 0 + for source, rows in intersections: + if source == rank: + continue + rate = ( + model.intra_node_gbps + if model.rank_node_ids[source] == model.rank_node_ids[rank] + else model.inter_node_gbps + ) + additions.append((source, rows * model.bytes_per_query / (rate * 1e6))) + moved += rows + received = sum(value for _, value in additions) + transfer = max(max_incoming, incoming[rank] + received, max_outgoing) + transfer = max(transfer, max((outgoing[s] + value for s, value in additions), default=0.0)) + if model.overlap_intra_inter: + local_received = sum(v for s, v in additions if model.rank_node_ids[s] == model.rank_node_ids[rank]) + remote_received = sum(v for s, v in additions if model.rank_node_ids[s] != model.rank_node_ids[rank]) + transfer = max( + max_incoming, max_outgoing, intra_in[rank] + local_received, inter_in[rank] + remote_received + ) + transfer = max( + transfer, + max( + ( + (intra_out if model.rank_node_ids[s] == model.rank_node_ids[rank] else inter_out)[s] + v + for s, v in additions + ), + default=0.0, + ), + ) + candidate_peer_peak = max( + peer_peak, max((peer_rows[s][rank] + n for s, n in intersections if s != rank), default=0) + ) + if model.peer_gbps is not None: + transfer = max(transfer, candidate_peer_peak * model.bytes_per_query / (model.peer_gbps * 1e6)) + new_edges = [s for s, n in intersections if s != rank and not peer_rows[s][rank]] + candidate_degree = max( + degree_peak, + receive_peers[rank] + len(new_edges), + max((send_peers[s] + 1 for s in new_edges), default=0), + ) + candidate_node = node_peak + if model.inter_node_aggregate_gbps is not None: + node_additions = defaultdict(int) + for s, n in intersections: + if model.rank_node_ids[s] != model.rank_node_ids[rank]: + node_additions[model.rank_node_ids[s]] += n + candidate_node = max( + node_peak, + node_in[model.rank_node_ids[rank]] + sum(node_additions.values()), + max((node_out[n] + rows for n, rows in node_additions.items()), default=0), + ) + transfer = max(transfer, candidate_node * model.bytes_per_query / (model.inter_node_aggregate_gbps * 1e6)) + transfer += candidate_degree * model.peer_startup_us / 1e3 + # Partial placement uses an upper bound on copies. Final + # scoring checks the actual source/destination permutations. + copies = max(max_copies, offsets[rank + 1] - offsets[rank] + loads[rank] + placed.tokens) + critical = max(max_compute, compute[rank] + cost) + transfer + if candidate_peer_peak: + critical += copies * model.packing_ns_per_query / 1e6 + model.exchange_us / 1e3 + key = (critical, compute[rank] + cost + incoming[rank] + received, moved, loads[rank], rank) + if best is None or key < best[0]: + best = key, rank, additions, received, candidate_peer_peak, candidate_degree, candidate_node + _, owner, additions, received, peer_peak, degree_peak, node_peak = best + result.append((placed, owner)) + loads[owner] += placed.tokens + compute[owner] += cost + incoming[owner] += received + for source, value in additions: + outgoing[source] += value + local = model.rank_node_ids[source] == model.rank_node_ids[owner] + (intra_out if local else inter_out)[source] += value + (intra_in if local else inter_in)[owner] += value + for source, rows in intersections: + if source != owner: + if not peer_rows[source][owner]: + send_peers[source] += 1 + receive_peers[owner] += 1 + peer_rows[source][owner] += rows + a, b = model.rank_node_ids[source], model.rank_node_ids[owner] + if a != b: + node_out[a] += rows + node_in[b] += rows + item = remainder + return result + + +def _paired_selection_items(boundaries, movable, world, block_size, offsets, model): + """Pair early/late chunks only within long documents, keeping shorts local.""" + result = [] + for sequence in sorted(movable): + start, end = boundaries[sequence : sequence + 2] + # Unlike balanced_shards, empty pieces are legitimate for tiny custom + # split thresholds. They never become empty THDSpan objects. + base, remainder = divmod((end - start) // block_size, 2 * world) + cuts = [start + (i * base + min(i, remainder)) * block_size for i in range(2 * world + 1)] + pairs = [] + for rank in range(world): + pairs.append( + [ + _make_item(sequence, cuts[i], cuts[i + 1], start, end, block_size) + for i in (rank, 2 * world - 1 - rank) + if cuts[i] < cuts[i + 1] + ] + ) + costs = [] + for pair in pairs: + overlaps = [0] * world + for item in pair: + for source, rows in _source_intersections(item, offsets): + overlaps[source] += rows + costs.append( + [ + sum( + rows + / ( + model.intra_node_gbps + if model.rank_node_ids[source] == model.rank_node_ids[rank] + else model.inter_node_gbps + ) + for source, rows in enumerate(overlaps) + if source != rank + ) + for rank in range(world) + ] + ) + owners = _minimum_assignment(costs) + result.extend((item, owners[rank]) for rank, pair in enumerate(pairs) for item in pair) + return result + + +def zigzag_thd_layout( + global_cu_seqlens: torch.Tensor | Sequence[int], *, world_size: int, block_size: int = 4 +) -> tuple[THDRankLayout, ...]: + """Conventional per-document early/late pairing, including short documents. + + Rank r owns pieces r and 2*CP-1-r. Empty pieces are omitted; adjacent + pieces remain distinct, matching the selection backend's execution spans. + This layout can have unequal or empty ranks and does not impose a row cap. + """ + config = THDPlannerConfig(block_size=block_size, min_split_tokens=block_size) + boundaries = _validate_inputs(global_cu_seqlens, world_size=world_size, config=config) + ranks = [[] for _ in range(world_size)] + for sequence, (start, end) in enumerate(zip(boundaries, boundaries[1:])): + base, remainder = divmod((end - start) // block_size, 2 * world_size) + cuts = [start + (i * base + min(i, remainder)) * block_size for i in range(2 * world_size + 1)] + for rank in range(world_size): + for piece in (rank, 2 * world_size - 1 - rank): + a, z = cuts[piece : piece + 2] + if a < z: + ranks[rank].append(THDSpan(sequence, a, z, start, end, rank)) + return tuple(THDRankLayout(r, tuple(spans)) for r, spans in enumerate(ranks)) + + +def _zigzag_lower_bound(boundaries, source, world, block_size, model): + """Average compute plus exact outgoing traffic, without building THD spans. + + Consecutive zigzag pieces assigned to the same node form interval runs. + Intersect each source span with its rank's two pieces and its node's runs: + this gives exact self, intra-node and inter-node row counts. The number of + runs is small for contiguous node ranks; arbitrary explicit topologies are + supported too. Average compute and outgoing-only transport are lower bounds + on their respective critical times. Packing and incoming/peer bottlenecks + are omitted, so this cannot overprice the concrete full-zigzag candidate. + """ + piece_nodes = [model.rank_node_ids[min(i, 2 * world - 1 - i)] for i in range(2 * world)] + node_runs = defaultdict(list) + begin = 0 + for node, pieces in groupby(piece_nodes): + end = begin + sum(1 for _ in pieces) + node_runs[node].append((begin, end)) + begin = end + geometry, total_work, spans = [], 0, 0 + for a, z in zip(boundaries, boundaries[1:]): + blocks = (z - a) // block_size + geometry.append(divmod(blocks, 2 * world)) + total_work += _candidate_work(0, z - a, block_size) + spans += min(blocks, 2 * world) + intra, inter = [0] * world, [0] * world + for item, rank in source: + base, remainder = geometry[item.sequence] + + def overlap(left, right): + a = item.sequence_start + (left * base + min(left, remainder)) * block_size + z = item.sequence_start + (right * base + min(right, remainder)) * block_size + return max(0, min(item.token_end, z) - max(item.token_start, a)) + + own_rows = overlap(rank, rank + 1) + overlap(2 * world - 1 - rank, 2 * world - rank) + node_rows = sum(overlap(a, z) for a, z in node_runs[model.rank_node_ids[rank]]) + intra[rank] += node_rows - own_rows + inter[rank] += item.tokens - node_rows + transfer = 0.0 + for local, remote in zip(intra, inter): + a, b = local / model.intra_node_gbps, remote / model.inter_node_gbps + transfer = max(transfer, (max(a, b) if model.overlap_intra_inter else a + b) * model.bytes_per_query / 1e6) + if model.inter_node_aggregate_gbps is not None: + node_out = defaultdict(int) + for rank, rows in enumerate(inter): + node_out[model.rank_node_ids[rank]] += rows + transfer = max(transfer, max(node_out.values()) * model.bytes_per_query / (model.inter_node_aggregate_gbps * 1e6)) + if any(intra) or any(inter): + # Each used link kind requires at least one distinct remote peer. + peers = max((a > 0) + (b > 0) for a, b in zip(intra, inter)) + transfer += (model.exchange_us + model.peer_startup_us * peers) / 1e3 + compute = (model.candidate_ns * total_work + model.query_ns * boundaries[-1]) / (world * 1e6) + compute += model.span_us * spans / (world * 1e3) + return compute + transfer + + +def _zigzag_meets_floor(boundaries, world, block_size, floor, source_fragments): + """Check grain feasibility before pricing or materializing full zigzag. + + Whole documents with sufficiently large pieces need no piece walk. For + finer pieces, retain exactly the same original-source-fragment exception + used by the other candidates. This rejects no otherwise legal layout. + """ + if not floor: + return True + for sequence, (start, end) in enumerate(zip(boundaries, boundaries[1:])): + blocks = (end - start) // block_size + base, remainder = divmod(blocks, 2 * world) + if max(1, base) * block_size >= floor: + continue + left = start + for piece in range(min(blocks, 2 * world)): + right = left + (base + (piece < remainder)) * block_size + if right - left < floor and (sequence, left, right) not in source_fragments: + return False + left = right + return True + + +def _source_selection_candidate(source, world, model): + """Price known identity ownership without reconstructing its transport. + + Source items already have exact causal work and global token order. Reuse + both for baseline metrics instead of recomputing candidate prefixes and + source intersections several times on the common no-migration path. + """ + groups = [[] for _ in range(world)] + tokens, causal, boundaries, compute = ([0] * world for _ in range(4)) + previous = None + for item, rank in source: + groups[rank].append( + THDSpan(item.sequence, item.token_start, item.token_end, item.sequence_start, item.sequence_end, rank) + ) + tokens[rank] += item.tokens + causal[rank] += item.causal_work + compute[rank] += model.candidate_ns * item.causal_work / 1e6 + model.query_ns * item.tokens / 1e6 + model.span_us / 1e3 + if previous is not None and previous[0].sequence == item.sequence and previous[1] != rank: + boundaries[previous[1]] += 1 + boundaries[rank] += 1 + previous = item, rank + metrics = THDLayoutMetrics( + tuple(tokens), + tuple(causal), + tuple(boundaries), + max(tokens) / (sum(tokens) / world), + max(causal) / (sum(causal) / world) if sum(causal) else 1.0, + max(compute), + ) + return tuple(THDRankLayout(r, tuple(g)) for r, g in enumerate(groups)), metrics + + +def plan_thd_selection( + global_cu_seqlens: torch.Tensor | Sequence[int], + *, + world_size: int, + max_tokens_per_rank: int, + cost_model: THDSelectionCostModel, + config: THDPlannerConfig | None = None, +) -> THDLayoutPlan: + """Plan selection-only routing with an explicit per-rank row capacity. + + Compare contiguous, complete per-document zigzag and locality-preserving + long-document candidates in the same cost model. Only the latter pin + short source spans and split documents at least twice ``min_split_tokens``. + Target row counts + may differ: attention and KDA must retain source ownership, and selection + IDs/scores must be restored through the inverse exchange before use. + + The capacity includes pinned rows and must fit every contiguous source + shard. It bounds query-shaped buffers, not total backend workspace; the + caller must also budget selection scratch and gathered keys. This eager + CPU planner does no device probing, timing or implicit caching. + """ + config = THDPlannerConfig() if config is None else config + if not isinstance(config, THDPlannerConfig): + raise TypeError("config must be a THDPlannerConfig") + if not isinstance(cost_model, THDSelectionCostModel): + raise TypeError("cost_model must be a THDSelectionCostModel") + boundaries = _validate_inputs(global_cu_seqlens, world_size=world_size, config=config) + if len(cost_model.rank_node_ids) != world_size: + raise ValueError("cost-model node map must have one entry per process-group rank") + if isinstance(max_tokens_per_rank, bool) or not isinstance(max_tokens_per_rank, int): + raise TypeError("max_tokens_per_rank must be an integer") + offsets = tuple(x * config.block_size for x in balanced_shards(boundaries[-1] // config.block_size, world_size)) + if max_tokens_per_rank % config.block_size or max_tokens_per_rank < max(b - a for a, b in zip(offsets, offsets[1:])): + raise ValueError("max_tokens_per_rank must be block aligned and fit every source shard") + base = _contiguous_items(boundaries, world_size, config.block_size) + source_fragments = ( + {(item.sequence, item.token_start, item.token_end) for item, _ in base} if config.min_chunk_tokens else set() + ) + movable = {i for i, (a, z) in enumerate(zip(boundaries, boundaries[1:])) if z - a >= 2 * config.min_split_tokens} + pinned = [(item, rank) for item, rank in base if item.sequence not in movable] + pinned_spans = () + source_ranks, source_metrics = _source_selection_candidate(base, world_size, cost_model) + candidates = [("contiguous", source_ranks, source_metrics.score)] + + def add(assignments, policy): + # Pinned and movable documents are disjoint. Reusing pinned spans + # avoids rebuilding hundreds of short-document objects per grid. + spans = sorted((*pinned_spans, *_coalesce(assignments)), key=lambda s: (s.sequence, s.token_start)) + # Capacity repair can split an otherwise useful item into tiny pieces. + # Reject the whole candidate instead of silently weakening the floor. + # Existing short source fragments are unavoidable and remain legal. + if policy != "contiguous" and any( + span.tokens < config.min_chunk_tokens and (span.sequence, span.token_start, span.token_end) not in source_fragments + for span in spans + ): + return + grouped = [[] for _ in range(world_size)] + for span in spans: + grouped[span.owner].append(span) + ranks = tuple(THDRankLayout(r, tuple(group)) for r, group in enumerate(grouped)) + if any(rank.local_tokens > max_tokens_per_rank for rank in ranks): + return + score = cost_model.estimate_ms(ranks, block_size=config.block_size) + candidates.append((policy, ranks, score)) + + baseline_ms = candidates[0][2] + lower_bound = baseline_ms + # This bound allows arbitrary row movement, including short documents. + # It may prune full zigzag too, but the long-document threshold may not. + if baseline_ms > cost_model.min_gain_ms: + lower_bound = ( + cost_model.candidate_ns * sum(item.causal_work for item, _ in base) / 1e6 + + cost_model.query_ns * boundaries[-1] / 1e6 + + cost_model.span_us * (len(boundaries) - 1) / 1e3 + ) / world_size + lower_bound = _transfer_lower_bound(base, world_size, config.block_size, cost_model, lower_bound, baseline_ms) + if movable and lower_bound < cost_model._migration_threshold_ms(baseline_ms): + pinned_loads, pinned_compute = [0] * world_size, [0.0] * world_size + for item, owner in pinned: + pinned_loads[owner] += item.tokens + pinned_compute[owner] += ( + cost_model.candidate_ns * item.causal_work / 1e6 + + cost_model.query_ns * item.tokens / 1e6 + + cost_model.span_us / 1e3 + ) + # These source-local spans remain in every long-document candidate. + # Nonnegative work and transport cannot reduce their critical compute + # time. This bound applies only to the pinned family, not full zigzag. + if max(pinned_compute) < cost_model._migration_threshold_ms(baseline_ms): + pinned_spans = _coalesce(pinned) + add( + _paired_selection_items(boundaries, movable, world_size, config.block_size, offsets, cost_model), + "selection-paired-local", + ) + seen = set() + for grid in range(1, config.max_chunks_per_rank + 1): + items = tuple(_adaptive_items(boundaries, world_size, config, chunks_per_rank=grid, sequences=movable)) + if items in seen: + continue + seen.add(items) + add( + _selection_assign( + items, + pinned_loads=pinned_loads, + pinned_compute=pinned_compute, + world=world_size, + capacity=max_tokens_per_rank, + offsets=offsets, + model=cost_model, + block_size=config.block_size, + ), + f"selection-local-{grid}x", + ) + # Score the cheap local candidates first. They tighten the incumbent, + # allowing an admissible bound to reject a fragmented full zigzag without + # allocating O(documents * CP) span objects. No candidate-specific heuristic + # threshold is used: any full zigzag that can win is still materialized. + threshold = min(cost_model._migration_threshold_ms(baseline_ms), min(c[2] for c in candidates)) + if ( + lower_bound < threshold + and _zigzag_meets_floor(boundaries, world_size, config.block_size, config.min_chunk_tokens, source_fragments) + and _zigzag_lower_bound(boundaries, base, world_size, config.block_size, cost_model) < threshold + ): + zigzag = zigzag_thd_layout(boundaries, world_size=world_size, block_size=config.block_size) + if all(rank.local_tokens <= max_tokens_per_rank for rank in zigzag) and all( + span.tokens >= config.min_chunk_tokens or (span.sequence, span.token_start, span.token_end) in source_fragments + for rank in zigzag + for span in rank.spans + ): + candidates.append(("zigzag", zigzag, cost_model.estimate_ms(zigzag, block_size=config.block_size))) + selected = min(candidates, key=lambda c: (c[2], c[0] != "contiguous", c[0])) + if selected[2] >= cost_model._migration_threshold_ms(baseline_ms): + selected = candidates[0] + policy, ranks, score = selected + metrics = ( + source_metrics + if policy == "contiguous" + else _metrics( + [span for rank in ranks for span in rank.spans], world_size=world_size, block_size=config.block_size, config=config + ) + ) + metrics = replace(metrics, score=score) + return THDLayoutPlan( + world_size, + config.block_size, + policy, + ranks, + metrics, + tuple((name, value) for name, _, value in candidates), + "ms", ) @@ -588,7 +1440,10 @@ def plan_thd_layout( "THDLayoutMetrics", "THDLayoutPlan", "THDPlannerConfig", + "THDSelectionCostModel", "THDRankLayout", "THDSpan", "plan_thd_layout", + "plan_thd_selection", + "zigzag_thd_layout", ] diff --git a/cula/hybrid/__init__.py b/cula/hybrid/__init__.py index 708d6636..998458c9 100644 --- a/cula/hybrid/__init__.py +++ b/cula/hybrid/__init__.py @@ -3,6 +3,7 @@ """cuDNN-backed hybrid-attention integration primitives.""" +from cula.hybrid.csa_cp import CsaCPSelectionPlan, CsaCPSelectionResult, CsaCPSelectionWorkspace, cudnn_csa_select_cp from cula.hybrid.distributed_pool import cudnn_index_pool_cp from cula.hybrid.dsa import ( DEFAULT_CANDIDATE_BUDGET_BYTES, @@ -18,6 +19,9 @@ from cula.hybrid.plan import IndexPoolPlan, IndexPoolShardPlan __all__ = [ + "CsaCPSelectionPlan", + "CsaCPSelectionResult", + "CsaCPSelectionWorkspace", "DEFAULT_CANDIDATE_BUDGET_BYTES", "MAX_BALANCED_CANDIDATE_BUDGET_BYTES", "MEMORY_EFFICIENT_CANDIDATE_BUDGET_BYTES", @@ -28,6 +32,7 @@ "IndexPoolPlan", "IndexPoolShardPlan", "cudnn_dsa_select", + "cudnn_csa_select_cp", "cudnn_index_pool", "cudnn_index_pool_cp", ] diff --git a/cula/hybrid/csa_cp.py b/cula/hybrid/csa_cp.py new file mode 100644 index 00000000..99df789e --- /dev/null +++ b/cula/hybrid/csa_cp.py @@ -0,0 +1,283 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepared CSA indexer selection with adaptive CP routing and inverse mapping.""" + +from __future__ import annotations + +import math +from bisect import bisect_left, bisect_right +from collections.abc import Sequence +from dataclasses import dataclass + +import torch +import torch.distributed as dist + +from cula._metadata import _range_indices, as_boundaries, balanced_shards +from cula.distributed.redistribute import THDRedistributionPlan, THDRedistributor +from cula.distributed.thd_planner import ( + THDPlannerConfig, + THDRankLayout, + THDSelectionCostModel, + THDSpan, + plan_thd_selection, + zigzag_thd_layout, +) +from cula.hybrid.dsa import DsaSelectionPlan, DsaSelectionWorkspace, cudnn_dsa_select + + +@dataclass(frozen=True, slots=True) +class CsaCPSelectionPlan: + """Host-only selection plan for one rank of a packed THD batch. + + Source tokens are contiguous, balanced in complete pooling blocks. All + ranks must use identical global boundaries, policy, capacity and costs. + Documents must be block-aligned; selection destinations may be empty. + No CUDA metadata is read, hardware probed, or process group accessed here. + """ + + global_cu_seqlens: tuple[int, ...] + source_offsets: tuple[int, ...] + ranks: tuple[THDRankLayout, ...] + policy: str + evaluated_scores: tuple[tuple[str, float], ...] + top_k: int + block_size: int + exchange: THDRedistributionPlan + selection: DsaSelectionPlan | None + key_ranges: tuple[tuple[int, int], ...] + cost_model: THDSelectionCostModel | None + + @classmethod + def from_cu_seqlens( + cls, + global_cu_seqlens: torch.Tensor | Sequence[int], + *, + rank: int, + world_size: int, + top_k: int, + policy: str = "planner", + cost_model: THDSelectionCostModel | None = None, + max_tokens_per_rank: int | None = None, + config: THDPlannerConfig | None = None, + candidate_budget_bytes: int | None = None, + ) -> CsaCPSelectionPlan: + """Choose a layout, then prepare native selection-window metadata. + + ``planner`` requires explicit calibrated costs. Fixed ``contiguous`` + and complete per-document ``zigzag`` policies need no cost model. + The default destination row cap is twice the largest source shard; + it is separate from the native compact-logit scratch budget. + """ + for name, value, minimum in (("world_size", world_size, 1), ("rank", rank, 0), ("top_k", top_k, 1)): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{name} must be an integer >= {minimum}") + if rank >= world_size or top_k > 2048: + raise ValueError("rank must be below world_size and top_k must be <= 2048") + config = THDPlannerConfig() if config is None else config + if not isinstance(config, THDPlannerConfig): + raise TypeError("config must be a THDPlannerConfig") + block = config.block_size + if isinstance(block, bool) or not isinstance(block, int) or block < 1: + raise ValueError("config.block_size must be a positive integer") + if policy not in ("contiguous", "zigzag", "planner"): + raise ValueError("policy must be contiguous, zigzag, or planner") + if cost_model is not None and not isinstance(cost_model, THDSelectionCostModel): + raise TypeError("cost_model must be a THDSelectionCostModel") + if cost_model is not None and len(cost_model.rank_node_ids) != world_size: + raise ValueError("cost-model node map must match the process-group size") + if policy == "planner" and cost_model is None: + raise ValueError("planner requires an explicit calibrated cost_model") + if candidate_budget_bytes is not None and ( + isinstance(candidate_budget_bytes, bool) + or not isinstance(candidate_budget_bytes, int) + or candidate_budget_bytes < 4 + ): + raise ValueError("candidate_budget_bytes must hold at least one FP32 element") + cu = as_boundaries(global_cu_seqlens, name="global_cu_seqlens") + if cu[-1] >= 2**31 or any(x % block for x in cu): + raise ValueError("document boundaries must be block-aligned and fit int32") + offsets = tuple(x * block for x in balanced_shards(cu[-1] // block, world_size)) + source_cap = max(z - a for a, z in zip(offsets, offsets[1:])) + cap = 2 * source_cap if max_tokens_per_rank is None else max_tokens_per_rank + if isinstance(cap, bool) or not isinstance(cap, int) or cap < source_cap or cap % block: + raise ValueError("max_tokens_per_rank must be block-aligned and fit every source shard") + scores = () + if policy == "planner": + planned = plan_thd_selection( + cu, world_size=world_size, max_tokens_per_rank=cap, cost_model=cost_model, config=config + ) + ranks, policy, scores = planned.ranks, planned.policy, planned.evaluated_scores + elif policy == "zigzag": + ranks = zigzag_thd_layout(cu, world_size=world_size, block_size=block) + else: + ranks = tuple( + THDRankLayout( + owner, + tuple( + THDSpan(seq, max(left, a), min(right, z), a, z, owner) + for seq in range(bisect_right(cu, left) - 1, bisect_left(cu, right)) + for a, z in (cu[seq : seq + 2],) + ), + ) + for owner, (left, right) in enumerate(zip(offsets, offsets[1:])) + ) + if any(r.local_tokens > cap for r in ranks): + raise ValueError("selected layout exceeds max_tokens_per_rank") + local = ranks[rank] + ranges = tuple((s.sequence_start // block, s.sequence_end // block) for s in local.spans) + prefix = [0] + for a, z in ranges: + prefix.append(prefix[-1] + z - a) + native = None + if local.local_tokens: + native = DsaSelectionPlan.from_cu_seqlens( + local.local_cu_seqlens, + prefix, + top_k=top_k, + block_size=block, + q_causal_offsets=local.causal_offsets, + candidate_budget_bytes=candidate_budget_bytes, + ) + return cls( + cu, + offsets, + ranks, + policy, + scores, + top_k, + block, + THDRedistributionPlan.build(ranks, offsets, rank), + native, + ranges, + cost_model, + ) + + +@dataclass(frozen=True, slots=True) +class CsaCPSelectionWorkspace: + """Device metadata, transport and reusable selection scratch. + + Prepare/use on the same CUDA stream, or explicitly synchronize between + streams. Use a separate workspace for overlapping invocations; the native + scratch is mutable. Keep the process group alive through every invocation. + This eager, allocating transport has no CUDA Graph capture contract. + """ + + plan: CsaCPSelectionPlan + exchange: THDRedistributor + selection: DsaSelectionWorkspace | None + key_slice: slice + key_indices: torch.Tensor | None + + @classmethod + def from_plan(cls, plan: CsaCPSelectionPlan, device: torch.device | str, *, group=None) -> CsaCPSelectionWorkspace: + if not isinstance(plan, CsaCPSelectionPlan): + raise TypeError("plan must be a CsaCPSelectionPlan") + device = torch.device(device) + if device.type != "cuda": + raise ValueError("CSA CP selection requires a CUDA device") + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + exchange = THDRedistributor(plan.exchange, device, group=group) + native = None if plan.selection is None else DsaSelectionWorkspace.from_plan(plan.selection, device) + ranges, indices = plan.key_ranges, None + key_slice = slice(0, 0) + if ranges: + if all(a[1] == b[0] for a, b in zip(ranges, ranges[1:])): + key_slice = slice(ranges[0][0], ranges[-1][1]) + else: + indices = _range_indices(ranges, device) + return cls(plan, exchange, native, key_slice, indices) + + +@dataclass(frozen=True, slots=True) +class CsaCPSelectionResult: + """Source-ordered document-local int32 block IDs and FP32 raw scores. + + Outputs own their storage and survive workspace reuse. Invalid slots are + -1 IDs; slot order is unspecified. Selection is discrete and has no autograd + connection: downstream indexer training uses its explicit backward/loss. + """ + + block_ids: torch.Tensor + scores: torch.Tensor + + +@torch.no_grad() +def cudnn_csa_select_cp( + qi: torch.Tensor, + global_kbar: torch.Tensor, + weights: torch.Tensor, + *, + workspace: CsaCPSelectionWorkspace, + sm_scale: float = 1.0, + deterministic: bool = False, +) -> CsaCPSelectionResult: + """Route local queries, select pooled blocks, and restore original rows. + + ``qi`` is contiguous BF16 [source_rows, H, D], ``weights`` is BF16/FP32 + [source_rows, H], and ``global_kbar`` is contiguous BF16 [total_blocks, 1, D] + in original packed-document order, already gathered by the caller. Pooling, + attention, projections and their gradients stay independently composable. + No precision casts, head padding, or device-to-host metadata copies occur. + + All ranks must enter in the same order with matching tensor trailing + dimensions/dtypes and scalar options, including empty destination ranks. + Tensor validation is local, like the underlying PyTorch collectives. + """ + if not isinstance(workspace, CsaCPSelectionWorkspace): + raise TypeError("workspace must be a CsaCPSelectionWorkspace") + if not isinstance(deterministic, bool): + raise TypeError("deterministic must be a bool") + if isinstance(sm_scale, bool) or not isinstance(sm_scale, int | float) or not math.isfinite(sm_scale): + raise ValueError("sm_scale must be a finite host scalar") + plan, exchange = workspace.plan, workspace.exchange + if dist.get_rank(exchange.group) != plan.exchange.rank or dist.get_world_size(exchange.group) != plan.exchange.world_size: + raise ValueError("workspace process group no longer matches the plan") + if qi.ndim != 3 or qi.shape[0] != plan.exchange.source_rows or min(qi.shape[1:]) < 1: + raise ValueError("qi must have shape [source_rows, H, D] with positive H/D") + if weights.shape != qi.shape[:2] or global_kbar.shape != (plan.global_cu_seqlens[-1] // plan.block_size, 1, qi.shape[2]): + raise ValueError("weights/global_kbar shapes do not match the source query contract") + if ( + qi.dtype != torch.bfloat16 + or global_kbar.dtype != torch.bfloat16 + or weights.dtype not in (torch.bfloat16, torch.float32) + ): + raise TypeError("qi/global_kbar must be BF16 and weights BF16 or FP32; no implicit quantization") + for tensor in (qi, global_kbar, weights): + if tensor.device != exchange.device or not tensor.is_contiguous(): + raise ValueError("inputs must be contiguous and on the workspace device") + if plan.cost_model is not None: + payload = qi.shape[1] * qi.shape[2] * qi.element_size() + qi.shape[1] * weights.element_size() + plan.top_k * 8 + if payload != plan.cost_model.bytes_per_query: + raise ValueError("cost_model.bytes_per_query must match Q/weights plus returning int32 IDs/FP32 scores") + routed_q, routed_weights = exchange.distribute(qi), exchange.distribute(weights) + if plan.selection is None: + ids = torch.empty((0, plan.top_k), dtype=torch.int32, device=exchange.device) + scores = torch.empty((0, plan.top_k), dtype=torch.float32, device=exchange.device) + else: + keys = ( + global_kbar[workspace.key_slice] + if workspace.key_indices is None + else global_kbar.index_select(0, workspace.key_indices) + ) + selected = cudnn_dsa_select( + routed_q, + keys, + routed_weights, + plan=plan.selection, + workspace=workspace.selection, + sm_scale=sm_scale, + deterministic=deterministic, + ) + ids, scores = selected.block_ids, selected.scores + ids, scores = exchange.restore(ids), exchange.restore(scores) + # The identity transport returns aliases, whereas a real exchange owns its + # outputs. Provide the same retained-result lifetime for every policy. + if plan.exchange.identity: + ids, scores = ids.clone(), scores.clone() + return CsaCPSelectionResult(ids, scores) + + +__all__ = ["CsaCPSelectionPlan", "CsaCPSelectionWorkspace", "CsaCPSelectionResult", "cudnn_csa_select_cp"] diff --git a/cula/hybrid/dsa.py b/cula/hybrid/dsa.py index f3f0cf74..def025e5 100644 --- a/cula/hybrid/dsa.py +++ b/cula/hybrid/dsa.py @@ -246,16 +246,22 @@ def from_cu_seqlens( row = 0 while row < query_length: - low, high = row + 1, query_length - while low < high: - middle = (low + high + 1) // 2 - needed = _candidate_floats(position + row, middle - row, block_size) - if needed <= budget_floats: - low = middle - else: - high = middle - 1 - row_end = low + # Most short documents fit whole. Check the exact remaining + # work once before searching a split; retain the same maximal + # prefix when a long document exceeds the scratch budget. + row_end = query_length candidates = _candidate_floats(position + row, row_end - row, block_size) + if candidates > budget_floats: + low, high = row + 1, query_length - 1 + while low < high: + middle = (low + high + 1) // 2 + needed = _candidate_floats(position + row, middle - row, block_size) + if needed <= budget_floats: + low = middle + else: + high = middle - 1 + row_end = low + candidates = _candidate_floats(position + row, row_end - row, block_size) atoms.append((sequence, row, row_end, key_length, position + row, candidates, query_length)) row = row_end @@ -265,6 +271,7 @@ def from_cu_seqlens( sequence, row_start, row_end, key_length, position, candidates, query_length = atoms[atom] whole = row_start == 0 and row_end == query_length group = [atoms[atom]] + group_candidates = candidates atom += 1 while whole and atom < len(atoms): next_atom = atoms[atom] @@ -273,10 +280,11 @@ def from_cu_seqlens( next_sequence != group[-1][0] + 1 or next_start != 0 or next_end != next_length - or sum(item[5] for item in group) + next_candidates > budget_floats + or group_candidates + next_candidates > budget_floats ): break group.append(next_atom) + group_candidates += next_candidates atom += 1 first_sequence = group[0][0] diff --git a/docs/CSA_CP_SELECTION.md b/docs/CSA_CP_SELECTION.md new file mode 100644 index 00000000..9f5973b1 --- /dev/null +++ b/docs/CSA_CP_SELECTION.md @@ -0,0 +1,146 @@ +# CSA context-parallel selection API + +`cudnn_csa_select_cp` packages THD planning, query/weight redistribution, +cuDNN block selection, and the inverse exchange. It returns document-local +block IDs and FP32 scores in the caller's original token order. Downstream +attention implementations can use the planner without copying benchmark code. + +The interface stops at selection. Projection weights, pooling transforms, +sparse attention, and the indexer loss remain independently composable. +`cudnn_index_pool_cp` can pool before gathering; the caller supplies global +pooled keys after its normalization/position transform. Attention and KDA +keep source ownership. + +## Integration + +```python +import torch.distributed as dist +from cula.distributed import THDPlannerConfig, THDSelectionCostModel +from cula.hybrid import ( + CsaCPSelectionPlan, + CsaCPSelectionWorkspace, + cudnn_csa_select_cp, +) + +# Identical host-visible boundaries on all ranks, prepared outside forward. +plan = CsaCPSelectionPlan.from_cu_seqlens( + global_cu_seqlens_cpu, + rank=dist.get_rank(cp_group), + world_size=dist.get_world_size(cp_group), + top_k=512, + policy="planner", + cost_model=THDSelectionCostModel(**calibrated_coefficients), + config=THDPlannerConfig(min_chunk_tokens=1024), + candidate_budget_bytes=512 << 20, +) +workspace = CsaCPSelectionWorkspace.from_plan(plan, qi.device, group=cp_group) + +# Reuse for the same packed layout. Nondeterministic selection is the default. +selected = cudnn_csa_select_cp( + qi, # BF16 [source_rows, index_heads, index_dim] + global_kbar[:, None], # BF16 [global_blocks, 1, index_dim] + index_weights, # BF16 or FP32 [source_rows, index_heads] + workspace=workspace, +) +# selected.block_ids: int32 [source_rows, 512], document-local, -1 padding +# selected.scores: FP32 [source_rows, 512], paired raw scores +``` + +`plan.policy` and `plan.evaluated_scores` expose the chosen candidate and modeled +costs. Fixed `contiguous` and complete per-document `zigzag` policies need no +cost model. There is no implicit hardware detection, calibration, cache, or +device-to-host metadata readback. Planning remains on the host; workspace +materialization uploads compact interval metadata and builds the large integer +index/permutation arrays on the target device. This avoids constructing and +copying token-sized CPU arrays. Device operations use explicit output sizes. + +## Layout, lifetime and numerical contract + +- Documents are block-aligned, with pooling block size four by default. Source + ownership divides the packed stream into block-balanced contiguous shards. + `plan.source_offsets` gives the exact cuts, which can have unequal lengths. + There must be at least one block per source rank. +- All ranks use identical global metadata, costs and constraints. Rank IDs and + topology maps are relative to the supplied group; subgroups are supported. + Every rank enters the same exchanges, including empty selection destinations. +- Inputs are contiguous on the workspace device. Trailing dimensions, dtypes, + scale and deterministic mode agree across peers. No casts or head padding + are implicit; native backend head/dimension restrictions still apply. +- `cost_model.bytes_per_query` must match outgoing Q/weights and returning + int32 IDs/FP32 scores: `H*D*2 + H*weights.element_size() + top_k*8`. + This checks payload shape, not applicability of fitted kernel/bandwidth costs. +- IDs stay local to the original document. Rows are restored to source order; + output slot order is unspecified. Keep IDs and scores paired when sorting. + Add document block offsets only for a backend requiring global IDs. +- Results own their storage and survive workspace reuse. Identity routing + copies native outputs to provide the same lifetime as a real exchange. +- Selection is discrete and runs without autograd. Indexer training uses its + explicit backward/loss interface, as in the complete CSA benchmark. +- Scratch is mutable. Prepare/use on one stream, or establish an explicit + stream dependency. Overlapping invocations require separate workspaces. + Keep the process group alive. The eager transport allocates and does not + promise CUDA Graph capture/replay support. + +## Granularity and smaller contexts + +One planner handles 128K, 256K and 1M contexts. For a single long document, +causal selection work falls roughly quadratically with context length, while +moved bytes fall roughly linearly. Startup, span launches and preparation can +dominate shorter contexts. + +`min_split_tokens` controls document eligibility, not resulting piece size. +`min_chunk_tokens` separately constrains newly created selection spans: + +1. Adaptive grids cap their piece count to meet the floor. +2. Candidates violating it after placement/capacity repair are rejected. +3. Existing source fragments and complete short documents remain legal. +4. Contiguous remains feasible even if the requested grain is too large. +5. The planner applies the constraint to its full-zigzag candidate; explicitly + forcing `policy="zigzag"` preserves the baseline layout. + +The floor defaults to zero for backward compatibility. The example explicitly +uses 1024 tokens, a tunable constraint rather than a universally optimal value. +For 128K at CP64, six pieces per rank average about 341 tokens; a 1024-token +floor caps the adaptive partition at 128 pieces. This is a CPU planning +property, not a measured CP64 performance result. + +Destination capacity defaults to twice the largest source shard. It limits +query rows, not total memory. Candidate scratch has a separate budget; global +keys, routed payloads, retained results and key packing consume additional +memory. Explicit full zigzag can exceed the default cap for tiny documents; +raise `max_tokens_per_rank` deliberately to benchmark that layout. + +Calibrate for the actual shape, topology and message sizes. Historical 1M +profiles' 20/40 ms `min_gain_ms` values are one-use preparation budgets, not +universal defaults. Reusing them at shorter contexts can reject useful layouts. +Measure preparation, transport and native selection separately, then validate +their sum on held-out distributions. Use explicit calibrated parameters rather +than case-name or context-length-specific layout rules. + +## Validation + +- `tests/test_csa_cp_selection.py`: host metadata parity, invalid inputs, + capacities, result lifetime and CP64 grain constraints. +- `benchmarks/check_csa_cp_selection.py`: native selection in two disjoint + subgroups, unequal source sizes, empty destinations, exact source mapping, + and workspace reuse. +- `benchmarks/bench_csa_cp_windows.py`: legacy contiguous versus the public + contiguous/zigzag/planner paths, with rotating measurement order and full + output/seven-gradient comparisons. Small fixtures add independent FP32 + attention/indexer/pooling checks. + +The performance scope is complete CSA operator F/B, including pooling, +selection, sparse attention, teacher/KL backward, layouts and communication. +It excludes projections, KDA, MoE, optimizer and checkpoint recomputation. +Comparison fixtures coexist during timing: diagnostic allocator peaks include +other variants and are not standalone per-variant memory measurements. +Workspace bytes are reported separately. First-use compilation is outside +warmed timing and is not interpreted as a planner speedup. Fresh preparation +is rebuilt after warmup, with rotated and reversed policy order. Its full +wall time includes workspace allocation, H2D metadata and a completion sync; +there is no implicit cache or plan-reuse credit. Host-time outliers are retained. + +Measured B300 CP8 results and the preparation limitations are documented in +[the 128K/256K study](CSA_CP_WINDOW_RESULTS.md) and the expanded +[128K/256K/1M three-way comparison](CSA_CP_THREE_WAY_RESULTS.md), including +22- and 64-document packs. diff --git a/docs/CSA_CP_THREE_WAY_RESULTS.md b/docs/CSA_CP_THREE_WAY_RESULTS.md new file mode 100644 index 00000000..75a50087 --- /dev/null +++ b/docs/CSA_CP_THREE_WAY_RESULTS.md @@ -0,0 +1,301 @@ +# CSA CP: three policies across 128K, 256K and 1M + +This is the complete rerun after reducing host preparation work. On B300 CP8, +adaptive selection routing stays within **0.339%** of the faster fixed +contiguous/zigzag policy in steady execution across these 24 cases. All selected +layouts match the previous revision. The largest gains occur with unequal long +documents at 1M; uniform packs generally select contiguous. This is measured +coverage, not a guarantee over arbitrary THD inputs. + +The workload is **one complete CSA operator forward/backward**: ratio-4 pooling, +pooled-key and attention-KV gathers, routed selection and inverse exchange, +sparse attention, teacher/KL and all activation/parameter gradients. Runtime +layout conversions and communication are included. KDA, projections, MoE, +optimizer and checkpoint recomputation are excluded. These are not timings of +a connected seven-KDA/one-CSA training block. + +## Measurement and workloads + +- Eight B300 GPUs, CSA H64/D576/V512, indexer H64/D128, Top-K512. +- BF16 Q/K/weights; FP32 scores, nonlinearities and communication reductions. + TF32 and BF16 reduced-precision GEMM intermediate reductions are disabled. + Native nondeterministic mode is unchanged. +- PyTorch 2.12.1+cu130, CUDA runtime 13.0, CuTeDSL 4.8.1, cuDNN 9.20.0.48. + cuDNN Frontend source commit `855ccd77ba9efed22441fd454f7cc0206a068e94`. + Frontend wheel version metadata does not identify the imported source tree. +- Two warmups, seven timed execution samples and eight fresh preparation samples. + Policy order rotates and reverses. Each reported duration is the median of + the per-sample maximum across all ranks. Compilation, synthetic input creation + and correctness reference work are outside timing. Host-time outliers remain. +- The existing compute/transport profile is used with explicit `min_gain_ms=0`, + `max_chunks_per_rank=6`, `min_chunk_tokens=1024`. This is a reuse-oriented + throughput experiment, not a newly calibrated profile or changed default. +- Public contiguous, full per-document zigzag and planner share the same source + ownership. Zigzag/planner redistribute selection Q/weights and restore results; + attention retains the original ownership. A fourth legacy-contiguous fixture + is the correctness reference and is not a table column. + +Here K means 1,024 tokens and M means 1,048,576 tokens. All lengths are divisible +by four, and each pack sums exactly to the context size T. + +| Distribution | Document lengths | +|---|---| +| 1 long document | T | +| 64 uniform | 64 x T/64 | +| 64 near-uniform | Alternating T/64 - T/1024 and T/64 + T/1024, 32 pairs | +| 20 short + 2 long | 20 x 1,024, followed by 2 x (T - 20,480)/2 | +| 1,024 uniform | 1,024 x T/1,024 | +| 512 near-uniform | Alternating T/512 - 4 and T/512 + 4, 256 pairs | +| 998 short + 2 long | 998 x 64 and 2 x (T - 63,872)/2; long docs after 199 shorts and at the end | +| 1,000 long-tail | 800 x 64, 199 x 256 and one remaining long document | + +All tables report milliseconds; lower is better. Percentage columns are +`100 * (planner / baseline - 1)`; negative means less time. Subpercent differences +should be treated cautiously, rather than as strong evidence of a speedup. + +## 128K steady execution + +| Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | +|---|---:|---:|---:|---:|---:| +| 1 long document | 47.69 | 46.27 | 45.83 | -3.91% | -0.97% | +| 64 uniform | 20.00 | 20.67 | 19.99 | -0.04% | -3.29% | +| 64 near-uniform | 20.03 | 20.76 | 20.02 | -0.03% | -3.53% | +| 20 short + 2 long | 35.41 | 35.36 | 35.48 | +0.19% | +0.34% | +| 1,024 uniform | 9.32 | 11.10 | 9.28 | -0.37% | -16.34% | +| 512 near-uniform | 10.00 | 11.20 | 10.01 | +0.12% | -10.60% | +| 998 short + 2 long | 33.10 | 38.75 | 33.09 | -0.05% | -14.63% | +| 1,000 long-tail | 32.33 | 37.88 | 32.33 | +0.01% | -14.66% | + +## 256K steady execution + +| Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | +|---|---:|---:|---:|---:|---:| +| 1 long document | 115.74 | 105.47 | 104.23 | -9.94% | -1.17% | +| 64 uniform | 50.32 | 51.76 | 50.36 | +0.08% | -2.70% | +| 64 near-uniform | 50.35 | 51.71 | 50.30 | -0.09% | -2.72% | +| 20 short + 2 long | 90.67 | 88.23 | 87.59 | -3.40% | -0.72% | +| 1,024 uniform | 18.96 | 22.55 | 18.94 | -0.15% | -16.01% | +| 512 near-uniform | 21.74 | 24.21 | 21.77 | +0.14% | -10.06% | +| 998 short + 2 long | 83.71 | 95.86 | 82.07 | -1.96% | -14.39% | +| 1,000 long-tail | 98.91 | 115.31 | 92.09 | -6.89% | -20.14% | + +## 1M steady execution + +| Distribution | Contiguous | Zigzag | Planner | P time vs C | P time vs Z | +|---|---:|---:|---:|---:|---:| +| 1 long document | 852.26 | 637.36 | 637.78 | -25.17% | +0.07% | +| 64 uniform | 239.24 | 244.40 | 239.44 | +0.09% | -2.03% | +| 64 near-uniform | 239.18 | 244.56 | 239.23 | +0.02% | -2.18% | +| 20 short + 2 long | 578.44 | 492.49 | 490.87 | -15.14% | -0.33% | +| 1,024 uniform | 106.93 | 120.95 | 106.89 | -0.04% | -11.63% | +| 512 near-uniform | 153.33 | 162.94 | 153.63 | +0.19% | -5.71% | +| 998 short + 2 long | 567.96 | 550.56 | 494.71 | -12.90% | -10.14% | +| 1,000 long-tail | 800.96 | 715.78 | 591.82 | -26.11% | -17.32% | + +## Preparation plus one use + +The following is **median fresh preparation + median execution**, not a directly +timed cold end-to-end sample. Preparation includes common metadata, host planning, +workspace/index construction, H2D copies and a completion sync. It is rebuilt, +not replaced by a cache lookup. First compilation is still excluded. + +### 128K + +| Distribution | Contiguous | Zigzag | Planner | +|---|---:|---:|---:| +| 1 long document | 48.41 | 47.17 | 53.27 | +| 64 uniform | 20.90 | 24.40 | 21.10 | +| 64 near-uniform | 21.07 | 25.31 | 21.31 | +| 20 short + 2 long | 36.17 | 37.22 | 43.18 | +| 1,024 uniform | 15.78 | 54.64 | 17.80 | +| 512 near-uniform | 13.56 | 33.89 | 14.72 | +| 998 short + 2 long | 40.23 | 82.95 | 42.34 | +| 1,000 long-tail | 39.66 | 83.09 | 41.45 | + +### 256K + +| Distribution | Contiguous | Zigzag | Planner | +|---|---:|---:|---:| +| 1 long document | 116.81 | 106.57 | 111.94 | +| 64 uniform | 51.31 | 55.48 | 51.57 | +| 64 near-uniform | 51.34 | 55.48 | 51.51 | +| 20 short + 2 long | 91.79 | 90.53 | 96.11 | +| 1,024 uniform | 26.19 | 67.67 | 28.22 | +| 512 near-uniform | 25.42 | 47.63 | 26.58 | +| 998 short + 2 long | 92.58 | 191.44 | 106.15 | +| 1,000 long-tail | 107.78 | 263.45 | 115.51 | + +### 1M + +| Distribution | Contiguous | Zigzag | Planner | +|---|---:|---:|---:| +| 1 long document | 854.50 | 639.80 | 647.29 | +| 64 uniform | 240.69 | 248.73 | 241.09 | +| 64 near-uniform | 240.69 | 248.94 | 240.94 | +| 20 short + 2 long | 580.61 | 495.68 | 500.99 | +| 1,024 uniform | 114.18 | 168.40 | 116.17 | +| 512 near-uniform | 158.24 | 187.40 | 159.63 | +| 998 short + 2 long | 577.91 | 599.07 | 521.10 | +| 1,000 long-tail | 813.63 | 817.01 | 620.98 | + +## What the comparison shows + +The aggregate below sums one execution of each of the eight distributions at +each context length. It assumes equal case frequency, not a production workload +mix. Values are percentage time reductions relative to each fixed policy; +negative reductions mean a slowdown. + +| Context | Steady vs contiguous | Steady vs zigzag | Prepare + one use vs contiguous | Prepare + one use vs zigzag | +|---|---:|---:|---:|---:| +| 128K | 0.89% | 7.19% | -8.23% | 34.35% | +| 256K | 4.35% | 8.60% | -4.33% | 33.10% | +| 1M | 16.50% | 6.77% | 14.87% | 10.48% | + +The same planner and coefficients handle all three lengths. Single long documents +benefit from balancing causal work; many equal documents are already balanced +under contiguous ownership. Mixed packs benefit from redistributing expensive +long-document work while keeping cheap short documents local. At 1M, the +998-short/two-long and 1,000-document long-tail cases reduce steady time by +**10.14% and 17.32% versus full zigzag**, respectively. The 64-document cases +choose contiguous; the 22-document 1M case performs close to zigzag. + +Preparation remains a limitation for one-use layouts. At 128K/256K, aggregate +preparation-plus-one-step is still slower than contiguous. At 1M, the single +long document and 22-document pack remain slightly slower than zigzag under +one-use accounting. The largest one-use gap to the faster fixed policy is +**19.40%** (128k-mixed-22), compared with 50.77% in the previous run. +These are separately measured cohorts; the controlled host A/B below isolates +the preparation change. Neither cohort establishes universally optimal one-use +selection or a <3% preparation guarantee. Use a calibrated preparation budget +and account for expected layout reuse. + +## Equivalent preparation optimization + +The update removes redundant metadata work and prunes candidates with a proved +model-level bound; it introduces no retained cache: + +1. Compute pinned per-rank loads once per invocation and copy the small arrays + into each search grid. Visit only movable documents in adaptive grids. +2. Skip long-document candidates when immutable pinned work alone exceeds the + migration threshold. Full zigzag remains independently eligible. +3. Price a source-local span directly, preserving source/receive permutation + detection and floating-point accumulation order. +4. Check full-zigzag grain feasibility before transport estimation, retaining + the original-source-fragment exception. No feasible candidate is discarded. +5. Check exact whole-document/remainder work before the DSA binary split search. + Oversized intervals still use the same maximal budget-fitting prefix. + +Candidate definitions, coefficients, tie breaks, selected layouts, native window +boundaries, scratch sizes and GPU execution are unchanged. The diagnostic list +of evaluated scores can omit candidates rejected by the new lower bound. +All extra state is local to preparation; no mutable global cache, collective, +cast or D2H read is introduced. + +The following host-only A/B uses rank 7's complete `CsaCPSelectionPlan` constructor, +20 samples per variant, alternating before/after order after two warmups. It +excludes GPU workspace preparation. The baseline imports source from `ea71df3`; +the otherwise unchanged caller binds that revision's planner and DSA builder. +The raw report covers all 24 planner shapes plus fixed-policy controls. + +| Host-only case | Before (ms) | After (ms) | Time reduction | +|---|---:|---:|---:| +| 128k-single-planner | 6.177 | 6.052 | +2.0% | +| 256k-single-planner | 6.190 | 6.082 | +1.8% | +| 1024k-uniform-64-planner | 0.397 | 0.361 | +9.1% | +| 256k-mixed-22-planner | 6.795 | 6.507 | +4.2% | +| 128k-uniform-1024-planner | 4.494 | 4.157 | +7.5% | +| 128k-uniform-1024-zigzag | 38.705 | 37.026 | +4.3% | +| 128k-mixed-1000-planner | 21.876 | 3.862 | +82.3% | +| 1024k-long-tail-1000-planner | 24.956 | 16.904 | +32.3% | + +Full fresh preparation, including GPU metadata/workspace work, is shown next. +These before/after medians come from separate complete-CSA runs. Small values +have visible run-to-run variation; the table does not imply every case improved. +Host-time spikes remain in the raw samples. For example, the new 1M long-tail planner preparation ranges from 28.45 to 150.10 ms. + +| Context | Distribution | Before preparation | After preparation | Time reduction | +|---|---|---:|---:|---:| +| 128K | 1 long document | 7.55 | 7.45 | +1.4% | +| 128K | 64 uniform | 1.24 | 1.11 | +10.8% | +| 128K | 20 short + 2 long | 7.61 | 7.70 | -1.3% | +| 128K | 1,024 uniform | 8.97 | 8.52 | +5.1% | +| 128K | 998 short + 2 long | 28.31 | 9.26 | +67.3% | +| 128K | 1,000 long-tail | 12.13 | 9.12 | +24.8% | +| 256K | 1 long document | 7.79 | 7.71 | +1.0% | +| 256K | 64 uniform | 1.28 | 1.21 | +5.0% | +| 256K | 20 short + 2 long | 8.77 | 8.52 | +2.9% | +| 256K | 1,024 uniform | 8.97 | 9.28 | -3.5% | +| 256K | 998 short + 2 long | 32.67 | 24.09 | +26.3% | +| 256K | 1,000 long-tail | 31.81 | 23.42 | +26.4% | +| 1M | 1 long document | 9.32 | 9.51 | -2.0% | +| 1M | 64 uniform | 1.72 | 1.65 | +4.3% | +| 1M | 20 short + 2 long | 9.80 | 10.12 | -3.3% | +| 1M | 1,024 uniform | 9.63 | 9.29 | +3.6% | +| 1M | 998 short + 2 long | 34.91 | 26.39 | +24.4% | +| 1M | 1,000 long-tail | 38.68 | 29.16 | +24.6% | + +## Precision, review and provenance + +- **518 focused CPU tests pass**, including exact budget-boundary checks with + unaligned causal offsets, local-reorder pricing and grain-check equivalence. +- **660 planner comparisons** preserve the selected layout, metrics and score + exactly versus `ea71df3`. Every retained candidate score is identical; omitted + candidates are verified to miss the migration margin: 55 shapes + (31 historical + 24 window cases) x CP8/16/64 x zero/original headroom x + zero/1,024-token floors. CP64 is host metadata validation, not a GPU benchmark. +- **300 randomized DSA plans and 69,281 windows** match the baseline exactly. +- **128 native subgroup checks pass** for BF16/FP32 weights, unequal sources, + empty destinations and retained outputs; canonical IDs and scores match + unrouted selection exactly in these fixtures. +- All **24 shapes x four paths x eight ranks** pass full-tensor output and + seven-gradient comparisons against legacy contiguous, plus exact row-marker + forward/inverse/autograd checks. Three small fixtures under all four paths + pass independent FP32 attention/teacher/KL/pooling checks. Those FP32 checks + use the selected key set and do not independently establish discrete ranking. + +Maximum relative RMS over all public paths/ranks/cases: + +| Tensor | Maximum relative RMS | +|---|---:| +| Output | 0.21341% | +| dQ | 0.06697% | +| dKV | 0.10307% | +| Indexer dQ | 0.09174% | +| Indexer dWeights | 0.06641% | +| Pool dKV | 0.13371% | +| Pool dScore | 0.13834% | +| Pool dAPE | 0.00254% | + +Maximum auxiliary loss absolute difference is **1.16e-09**. No math, +dtype or native precision setting changed. These error envelopes include native +nondeterminism; small differences from the previous run are not a new precision +improvement or a bitwise-equivalence claim. + +The tested source is `489779ba35e2c5f51eccce7c7ff95ed4cdd640d0`. All 125 source +files in the final timing manifest match the reviewed checkout. +Every selected policy matches the previous run, and every measured full-zigzag +baseline fits the planner's row cap. The new measurements supersede the primary +tables in this document. The [previous 24-case evidence](results/csa_cp_three_way_20261116.json) +and [original 128K/256K study](CSA_CP_WINDOW_RESULTS.md) remain available. + +[Current machine-readable results](results/csa_cp_preparation_20261116.json) +retain critical timing/preparation samples, selected policies, precision envelopes, +source-record hashes, cohort source hashes and the counterbalanced host A/B. +The allocator diagnostics in the underlying harness include four coexisting +fixtures; they are not isolated per-policy memory claims. + +## Reproduction + +```bash +python -m torch.distributed.run --standalone --nproc-per-node=8 \ + -m benchmarks.bench_csa_cp_windows \ + --profile benchmarks/profiles/thd_selection_b300_cp8.json \ + --output-dir results/csa-three-way --tokens 131072 262144 1048576 \ + --cases single uniform-64 near-uniform-64 mixed-22 \ + uniform-1024 near-uniform-512 mixed-1000 long-tail-1000 \ + --min-gain-ms 0 --min-chunk-tokens 1024 --max-chunks-per-rank 6 \ + --warmups 2 --repeats 7 --prepare-repeats 8 +``` + +See the [public API contract](CSA_CP_SELECTION.md) for explicit plan/workspace +lifetimes, source ownership and downstream integration. diff --git a/docs/CSA_CP_WINDOW_RESULTS.md b/docs/CSA_CP_WINDOW_RESULTS.md new file mode 100644 index 00000000..c7b3ceee --- /dev/null +++ b/docs/CSA_CP_WINDOW_RESULTS.md @@ -0,0 +1,145 @@ +# CSA CP selection at 128K and 256K + +The [expanded three-way comparison](CSA_CP_THREE_WAY_RESULTS.md) covers 1M +contexts and 22-/64-document packs, for 24 cases, and has been rerun after the +equivalent host-preparation optimization. +The historical ten-case statistics below remain scoped to this original study. + +This tests the public `cudnn_csa_select_cp` API on eight B300 GPUs. The workload +is **one complete CSA operator forward/backward**, including ratio-4 pooling, +pooled-key and attention-KV gathers, selection routing/restoration, sparse +attention, teacher/KL, and all activation/parameter gradients. It excludes +KDA, projections, MoE, optimizer and checkpoint recomputation. These numbers +must not be substituted for a connected seven-KDA/one-CSA training block. + +## Protocol + +- CSA H64/D576/V512; indexer H64/D128; Top-K512; BF16 Q/K/weights; FP32 scores, + nonlinearities and communication reductions. TF32 and BF16 reduced-precision + GEMM intermediate reductions are disabled. Native nondeterministic mode. +- PyTorch 2.12.1+cu130, CUDA runtime 13.0, CuTeDSL 4.8.1. cuDNN Frontend source + `855ccd77ba9efed22441fd454f7cc0206a068e94`: all 3,285 checked Python/CUDA/C++ + source files match that commit. The installed frontend distribution version + in the JSON is wheel metadata, not the identity of the imported source tree. +- Five distributions per context: one document, 1,024 equal documents, 512 + slightly unequal documents, 998 short plus two long documents, and a + 1,000-document short/medium/long-tail mixture. `bench_csa_cp_windows.cases` + specifies exact lengths without a tokenizer or private dataset. +- Two warmups, seven execution samples and eight fresh preparation samples. + Policy order is rotated and reversed. Tables use the median of each sample's + maximum-rank duration. Preparation includes CPU planning, workspace/index + construction, H2D copies and a completion sync. It is rebuilt, not cached. + RNG generation, JIT compilation and correctness oracles are outside timing. +- Throughput experiment: existing compute/transport coefficients, explicit + `min_gain_ms=0`, `max_chunks_per_rank=6`, and `min_chunk_tokens=1024`. + This tests reuse-oriented planning; **zero is not a recommendation for a + fresh layout used once**. It does not replace the shipped 1M profile. + +## Complete CSA results + +C = public contiguous; Z = public complete per-document zigzag; P = public +planner. All times are milliseconds. The last column compares **preparation +plus one step** against the faster C/Z under that same accounting. + +| Case | C step | Z step | P step | C preparation | P preparation | P one-use gap | +|---|---:|---:|---:|---:|---:|---:| +| 128k-long-tail-1000 | 32.37 | 37.90 | 32.35 | 7.58 | 12.13 | +11.34% | +| 128k-mixed-1000 | 33.07 | 38.75 | 33.07 | 7.64 | 28.31 | +50.77% | +| 128k-near-uniform-512 | 10.01 | 11.21 | 10.03 | 3.74 | 4.96 | +8.92% | +| 128k-single | 47.60 | 46.23 | 45.82 | 0.73 | 7.55 | +13.21% | +| 128k-uniform-1024 | 9.29 | 11.10 | 9.30 | 6.78 | 8.97 | +13.70% | +| 256k-long-tail-1000 | 99.36 | 115.25 | 92.20 | 9.70 | 31.81 | +13.72% | +| 256k-mixed-1000 | 83.64 | 96.05 | 82.15 | 9.59 | 32.67 | +23.16% | +| 256k-near-uniform-512 | 21.79 | 24.21 | 21.76 | 3.94 | 5.02 | +4.10% | +| 256k-single | 115.29 | 105.58 | 104.15 | 1.12 | 7.79 | +4.89% | +| 256k-uniform-1024 | 18.96 | 22.56 | 18.94 | 6.91 | 8.97 | +7.89% | + +The worst measured steady-state gap against faster C/Z is **+0.142%**. Single +long documents improve over contiguous by about **3.7% at 128K** and **9.7% at +256K**. The 256K long-tail case improves about **7.2%**. These are observed +candidate comparisons, not a proof of optimality over arbitrary THD inputs. + +**Fresh preparation is still a limitation.** Searching can cost more than the +execution it saves, particularly when a mixed layout ultimately returns to +contiguous. The complete preparation-plus-one-step result is not uniformly +near the best fixed policy, and this standalone CSA study does not establish +an average <3% preparation bound. Host-time spikes remain in the raw critical +samples; no samples or garbage-collection time were removed. For example, the +256K long-tail planner preparation has a 31.81 ms median and a 132.93 ms maximum. +Do not infer a stable preparation budget from a single sample. + +## Sensitivity and next cost-model work + +Two additional experiments are retained in the machine-readable results: + +- Keeping the historical **20 ms** absolute headroom selected contiguous in + all ten cases, missing useful single-long-document routing. +- Reducing search to **two chunks/rank with 2 ms headroom** retained the single + long-document gains, but the 256K two-long/998-short mixture selected a route + **2.84% slower** than contiguous. Full search found a faster route. Preparation + for that exploratory run has only three samples and is not directly comparable + to the final eight-sample protocol. + +These are sensitivity experiments, **not independent short-context coefficient +calibration**. Neither a smaller fixed headroom nor fewer search grids is a +universal fix. A deployment should budget expected reuse and measure its message +sizes and kernel granularity. Further improvement should price the preparation +of each candidate and the value of additional search, rather than adding a +separate algorithm or case-name lookup for each context length. The interface +keeps costs, capacity, search granularity and minimum fragment size explicit. + +The fragment floor prevents increasingly fine splitting at large CP. At +128K/CP64, a 1,024-token floor caps an adaptive partition at 128 pieces rather +than the requested 384. Original short source fragments remain legal; a +candidate violating the floor after capacity repair is rejected. **CP64 here +is CPU metadata validation only**, not a 64-GPU performance measurement. + +## Correctness and review + +- **435 focused CPU tests pass**, including API validation, transport inverses, + empty targets, capacities, metadata parity at 128K/256K/1M and CP64 floors. +- **124 historical plans are exactly unchanged** at the default zero floor: + 31 distributions × CP8/CP16 × zero/original headroom, compared field-for-field + against the previous PR head. This is host-plan parity, not a new H20 run. +- **128 native subgroup checks pass**, covering BF16 and FP32 weights, unequal + source lengths, empty destinations, exact IDs/scores versus unrouted native + selection, and retained results across workspace reuse. Maximum score + difference is zero in these fixtures. +- All ten final context/distribution cases pass full-tensor comparison of the + output and seven gradients against the legacy contiguous implementation. + Largest relative RMS is **0.214%**; maximum auxiliary loss absolute difference + is **4.66e-10**. The native nondeterministic computation is not bitwise stable; + this does not change its dtype or numerical approximation contract. +- Three small fixtures under all four paths pass independent FP32 attention, + teacher/KL and pooling-gradient checks. These references use the selected key + set; they do not independently prove native discrete ranking. +- The 125 source files in the final timing manifest match the reviewed checkout. + Review checked group participation, unique inverse-permutation writes, source + ownership, empty destinations, retained outputs, explicit stream dependencies, + dtype validation and absence of D2H metadata reads. No handwritten GPU kernel + or native precision mode is changed. + +Allocator peaks in the raw harness include four coexisting comparison fixtures; +they are **not standalone per-policy memory claims**. The JSON separately +records candidate-workspace bytes for each rank. + +[Machine-readable results, error envelopes, samples and source hashes](results/csa_cp_windows_20261116.json) + +## Reproduction + +```bash +python -m torch.distributed.run --standalone --nproc-per-node=8 \ + -m benchmarks.bench_csa_cp_windows \ + --profile benchmarks/profiles/thd_selection_b300_cp8.json \ + --output-dir results/csa-windows --tokens 131072 262144 \ + --cases single uniform-1024 near-uniform-512 mixed-1000 long-tail-1000 \ + --min-gain-ms 0 --min-chunk-tokens 1024 --max-chunks-per-rank 6 \ + --warmups 2 --repeats 7 --prepare-repeats 8 + +python -m torch.distributed.run --standalone --nproc-per-node=8 \ + -m benchmarks.check_csa_cp_selection --weights-dtype fp32 +``` + +See the [integration contract](CSA_CP_SELECTION.md) for plan/workspace lifetimes, +source ownership and explicit preparation. Calibrate for the actual deployment; +this experiment does not install a universal small-context profile. diff --git a/docs/HYBRID_CUDNN_RUNTIME.md b/docs/HYBRID_CUDNN_RUNTIME.md index 1c479461..1f7bbf56 100644 --- a/docs/HYBRID_CUDNN_RUNTIME.md +++ b/docs/HYBRID_CUDNN_RUNTIME.md @@ -1,5 +1,8 @@ # cuDNN hybrid-attention runtime +For downstream context-parallel indexers, the [CSA CP selection API](CSA_CP_SELECTION.md) +combines adaptive planning, distributed selection and exact source-order restoration. + cuLA's hybrid layer is a layout-aware integration boundary for composing cuDNN Frontend attention kernels. It is intentionally not another graph API: cuDNN owns the kernels and their numerical contracts; cuLA owns static packed diff --git a/docs/KIMI_CP_RUNTIME.md b/docs/KIMI_CP_RUNTIME.md index 0f3f795c..52861a7b 100644 --- a/docs/KIMI_CP_RUNTIME.md +++ b/docs/KIMI_CP_RUNTIME.md @@ -50,11 +50,22 @@ balanced partition or explicit global shard offsets. It derives: - clipped local `cu_seqlens`; - first and last local sequence IDs; - the first and last boundary token spans; -- the number of preceding and following ranks in those sequence chains. +- the number of preceding and following ranks in those sequence chains; +- whether any document crosses a shard boundary anywhere in the group. Plan construction is CPU-only and must happen once per packed microbatch, not once per layer. The same plan is reusable by every KDA layer. +When every internal shard boundary is a document boundary, +`requires_boundary_exchange` is false on **every rank**. The runtime then uses +native local KDA/autograd without boundary summaries, state probes or transport. +The operands, dtypes and batch-invariance option are unchanged. A locally +isolated rank must still join the boundary exchanges if another document crosses +ranks elsewhere in the group. Manually constructed plans default to the +conservative exchanging path. The paired benchmark +`benchmarks/bench_kda_independent_shards.py` checks all outputs and five gradients, +including weak decay and an isolated-rank/crossing-peer control. + Only the first and last local spans can cross rank boundaries under the contiguous-shard contract. Forward therefore summarizes only the last local span; backward summarizes only the first. This is the boundary-only strategy @@ -73,12 +84,17 @@ Forward: Backward performs the symmetric reverse chain: -1. Produce `(G, M^T)` and reusable backward context for the first local span. +1. Produce `(G, M^T)` for the first local span. 2. All-gather the transition in its stored-domain orientation. 3. Select following ranks belonging to the last local sequence. 4. Pass them as `following_state_grad` and `following_transition`. 5. Let cuDNN compose the reverse chain and compute local tensor and gate gradients. +Native backward context contains per-token intermediates. It is reused only +when the summary covers the complete local shard. A multi-document shard's +prefix-only summary cannot supply context for the full apply operation; apply +prepares its own intermediates in that case. + The state payload and transition remain FP32. cuLA exposes no merge-precision switch; the cuDNN apply API owns the fixed high-precision state chain. The public cuDNN Frontend 1.32.2 wheel does not contain this API. Until a compatible diff --git a/docs/THD_1M_STATISTICS_CASES.md b/docs/THD_1M_STATISTICS_CASES.md new file mode 100644 index 00000000..0790927c --- /dev/null +++ b/docs/THD_1M_STATISTICS_CASES.md @@ -0,0 +1,114 @@ +# Statistics-inspired 1M THD workload catalog + +The motivating histogram describes **document counts by character length**. Its largest visible bucket is `(10,100]`; exact percentages, +the full long tail and tokenizer information were not supplied. This catalog +uses explicit synthetic priors to explore that shape. It is not a reconstruction +of the real token distribution or a claim about hidden chart values. + +There are **18 new cases**, in addition to the previous five baseline cases. +Every case contains exactly **1,048,576 tokens**, with positive document lengths +divisible by four for the current complete-CSA CP harness. All source CP16 +shards consequently have 65,536 tokens. The generator uses only Python's +standard library and does not initialize CUDA or distributed execution. + +## Distribution assumptions + +Weights below are **sample-count priors**, not token shares or measurements +read from the chart. The primary `mix80` prior approximates the chart's visible +shape; `tiny90` and `mix70` test sensitivity to its dominant short bucket. +The generator samples uniformly over aligned lengths within each bucket. + +| Synthetic prior | (10,100] | (100,1K] | (1K,2K] | (2K,4K] | (4K,8K] | (8K,16K] | +|---|---:|---:|---:|---:|---:|---:| +| tiny90 | 90% | 5% | 2% | 2% | 0.8% | 0.2% | +| mix80 | 80% | 7% | 5% | 5% | 2.5% | 0.5% | +| mix70 | 70% | 12% | 8% | 6% | 3% | 1% | + +K=1024 is a synthetic convention. Default bucket endpoints act as token-length +proxies. Two additional cases divide those endpoints by 2 and 4 before drawing +aligned token lengths; these are character/token **sensitivity assumptions**, +not measured conversions. Actual tokenization is required to calibrate them. +For these scaled cases, the reported `<=100 token` fraction is not the same +quantity as the source chart's `(10,100] character` fraction. + +Sampling stops at the desired mixture token budget. Only the final draw may +be shortened to fill exactly; the manifest records the number of tokens removed. +It never fills a residual with an invented giant document. Explicit long stress +documents are then added without truncation. Fixed seed `20261112` and a +separate deterministic permutation make the complete ordered length list +reproducible. Different seeds provide additional replicas. + +## Twelve primary cases + +All numbers below are the **generated default-seed values**, not expected +probabilities. Prefix every short name with `1m-stat-` to obtain the benchmark +case id. Randomized cases use a reproducible shuffled order. + +| Case | Documents | <=100-token document share | <=100-token token share | Largest document | Explicit long-token share | +|---|---:|---:|---:|---:|---:| +| tiny64 | 16,384 | 100.00% | 100.00% | 64 | 0% | +| tiny90 | 4,100 | 89.68% | 19.75% | 14,148 | 0% | +| mix80 | 1,857 | 79.43% | 7.88% | 15,064 | 0% | +| mix70 | 1,403 | 69.57% | 5.27% | 15,932 | 0% | +| mix80-div2 | 3,860 | 80.00% | 8.37% | 7,532 | 0% | +| mix80-div4 | 7,783 | 82.04% | 9.53% | 4,052 | 0% | +| mix80-1x128k | 1,639 | 79.44% | 6.94% | 131,072 | 12.5% | +| mix80-1x256k | 1,450 | 79.03% | 6.10% | 262,144 | 25% | +| mix80-1x512k | 1,066 | 79.55% | 4.60% | 524,288 | 50% | +| mix80-2x256k | 1,067 | 79.48% | 4.60% | 262,144 | 50% | +| mix80-4x128k | 1,069 | 79.33% | 4.60% | 131,072 | 50% | +| mix80-8x64k | 1,073 | 79.03% | 4.60% | 65,536 | 50% | + +The distinction between document share and token share is material: `mix80` +contains nearly 80% tiny documents, but those documents contribute only 7.88% +of tokens. In `mix80-2x256k`, just two of 1,067 documents contribute 50% of all +tokens. These cases expose both many-document overhead and long-prefix causal +work, which the earlier 22-document mixed case could not represent together. + +The four 50%-long-token cases share **the identical 524,288-token mixture +background**, while splitting the remaining 524,288 tokens as one, two, four +or eight long documents. This controls total long-token volume and tests +whether the planner handles changing document count and causal lengths. +The 64K–512K spikes are deliberate stress additions; their frequency is not +inferred from the chart. + +`tiny64` stresses many short segments, boundaries, padding and scheduling. +`mix70`/`mix80`/`tiny90` vary document-count concentration. `div2`/`div4` increase +document density at a fixed token budget. Long-spike cases stress causal load, +communication and scratch memory without making document-count histograms +look predominantly long. + +## Six order-only controls + +For **both** `mix80` and `mix80-2x256k`, add these suffixes: + +| Suffix | Ordering | Purpose | +|---|---|---| +| `-ascending` | shortest to longest | Cluster longer documents at the end of packed input | +| `-descending` | longest to shortest | Move long-document ownership toward earlier source ranks | +| `-alternating` | shortest, longest, second shortest, second longest, ... | Interleave the same histogram | + +Each variant retains exactly the same length multiset and total as its +shuffled counterpart. Thus performance differences can be associated with +packing order/rank ownership rather than a different length histogram. The +control fixes lengths, not document-content tensors across order variants: +the current GPU harness still creates synthetic operands from rank-local RNG. + +## Reproducible suites + +The generator `benchmarks/thd_1m_cases.py` writes exact lengths, seeds, +assumptions, SHA256 digests and count/token histograms. The matrix runner +`benchmarks/bench_csa_layout_matrix.py` accepts `--list-cases` without CUDA. + +- `baseline`: five single, uniform, near-uniform and long-tail cases. +- `counted`: eleven 256--1024-document cases, including order-only controls. +- `heldout`: five lognormal, jittered, tiny-tail and trimodal cases for a supplied seed. +- `statistics`: the eighteen synthetic histogram cases described above. +- `validation`: the five baseline cases, eleven counted cases at seed 20261112, + and five held-out cases at each of seeds 20261114, 20261115 and 20261116. + This fixes the exact **31-case** matrix in the [measured report](THD_SELECTION_RESULTS.md). + +The statistics suite is an additional stress catalog; it is not part of that +31-case GPU result. All suites use synthetic tensor operands, not training data. +A planner does not receive case names or histogram priors: only CPU sequence +boundaries, its configuration and explicit deployment coefficients. diff --git a/docs/THD_LAYOUT_PLANNER.md b/docs/THD_LAYOUT_PLANNER.md index bb46035c..fbf2fd35 100644 --- a/docs/THD_LAYOUT_PLANNER.md +++ b/docs/THD_LAYOUT_PLANNER.md @@ -1,117 +1,214 @@ -# Cost-driven THD context-parallel layout planner - -## Why the alternating prototype is not the production policy - -Alternating the shard direction of adjacent long samples exactly balances the -original `20 x 1K + 2 x 514048` experiment, but it encodes properties of that -batch into the policy. It is weak for a single dominant sequence, an odd -number of long sequences, and power-law mixtures. - -`plan_thd_layout` instead consumes arbitrary CPU `global_cu_seqlens` and -balances two independent resources: - -- token-linear KDA and index-pooling work; -- position-dependent causal candidate work in CSA/DSA. - -Short samples remain whole when exact token capacity permits it. Long samples -are considered on several planning grids from `1 x CP` through `6 x CP`. Equal-token pieces are assigned by a -deterministic multi-resource largest-processing-time scheduler, then adjacent -pieces with the same owner are coalesced. This naturally produces an -early/late zigzag for one long sequence and cross-sample compensation when -several long sequences are available. - -After compute scheduling, a capacity-repair pass moves whole spans where -possible and otherwise splits the cheapest span boundary. Every rank receives -its exact balanced block capacity (at most one block difference when the global -block count is not divisible by CP), so the plan remains compatible with -fixed-shape KDA and pooled all-gather collectives. - -## Objective and fallback guarantee +# THD context-parallel planning + +cuLA separates **where selection runs** from **where recurrent state and +attention activations live**. `plan_thd_selection` balances causal indexer work +while KDA, pooling, attention forward/backward and indexer backward retain their +original contiguous CP shards. Only indexer Q/weights move; document-local +block IDs and FP32 scores return to the source query owners before attention. +The [redistributor](THD_REDISTRIBUTION.md) implements both directions and their +first-order autograd, without casting or reducing row values. + +This avoids applying full per-document zigzag to every short document or moving +wide MLA attention activations just to balance selection. It is an eager +integration; it does not promise CUDA Graph capture or whole-model E2E speedups. +See the [31-distribution measurements](THD_SELECTION_RESULTS.md) for scope, +preparation costs, precision, memory and measured limitations. + +## Public interfaces + +- `plan_thd_selection`: calibrated, capacity-bounded selection ownership; target + row counts can differ or be zero. Keep attention and KDA in source ownership. +- `zigzag_thd_layout`: conventional per-document early/late pairing, shared by + the planner and benchmark baseline. Rank r receives pieces r and 2*CP-1-r + from each document, including short documents. Empty pieces are omitted; + adjacent pieces remain distinct execution spans. +- `plan_thd_layout`: existing balanced-token policy, for consumers that need + exact balanced block capacities. Its default analytic objective is preserved. + This API alone does not make shuffled KDA spans executable by a contiguous + Kimi CP state chain. + +All planning uses CPU sequence boundaries and explicit caller configuration. +There is no device probing, online timing, hidden cache, process collective or +case-name lookup. Peers must use identical global metadata, coefficients and +configuration. Plan once at microbatch ingress and reuse where appropriate. +Planning is deterministic; kernel `deterministic=False` remains the throughput +option and is independent of the planner's stable tie-breaking. + +```python +from cula.distributed import plan_thd_selection, THDRedistributionPlan, THDRedistributor + +layout = plan_thd_selection( + global_cu_seqlens, + world_size=cp_world_size, + max_tokens_per_rank=query_capacity, + cost_model=calibrated_model, +) +route = THDRedistributionPlan.build(layout.ranks, source_offsets, cp_rank) +exchange = THDRedistributor(route, local_index_q.device, group=cp_group) +selection_q = exchange.distribute(local_index_q) +selection_weights = exchange.distribute(local_weights) +# Selection must use layout.for_rank(cp_rank)'s original document origins, +# causal offsets and document-local block numbering, with the required keys. +selected = select_for_spans(selection_q, selection_weights, layout.for_rank(cp_rank)) +local_ids = exchange.restore(selected.block_ids) +local_scores = exchange.restore(selected.scores) +``` -For rank `r`, let `T_r` be its tokens and `C_r` its exact analytic causal -candidate count. The default robust objective is +The model assumes balanced, block-aligned contiguous source shards. Pass those +same offsets to the redistributor. `query_capacity` must be block aligned and +fit the largest source shard. It bounds query-shaped buffers, **not** gathered +keys or backend scratch. The full-CSA benchmark uses twice the source row count; +callers must budget actual backend workspace separately. Documents must be +block aligned for these planners; the current CSA harness uses ratio four. + +## Candidate pool and fallback + +Selection compares all feasible candidates under one cost model: + +1. Contiguous source ownership. +2. Complete per-document zigzag, including documents below the split threshold. +3. Early/late pairs within long documents, relabeled onto ranks with a + minimum-cost assignment to reduce aggregate traffic. Short source spans stay + local in this candidate family. +4. Greedy long-span placement on grids from 1 through `max_chunks_per_rank` + pieces per rank. Descending modeled work is placed using current compute, + incoming/outgoing traffic and row capacity; a span splits only when no bin + can hold it. Adjacent same-owner spans coalesce. + +Only candidates 3 and 4 pin documents shorter than twice `min_split_tokens` +(default 16K). Full zigzag is independently considered and obeys the same row +cap. A zigzag that concentrates tiny documents beyond that cap is infeasible. +The minimum-cost assignment is a stable O(CP^3) primal-dual algorithm. It +minimizes aggregate traffic, so the result is still scored by the full +critical-path model rather than assumed to be best. + +Migration requires both `min_gain_fraction` (default 5%) and `min_gain_ms` +(default zero) over contiguous. `min_gain_ms` is an explicit minimum predicted +saving per selection use. A caller can price one-time preparation divided by +an expected reuse count; cuLA neither guesses that count nor hides preparation +in steady-state timing. `evaluated_scores` contains only materialized/scored +candidates. The selected modeled cost cannot exceed contiguous; this is **not** +a guarantee of minimum measured GPU latency over arbitrary distributions. + +## Compute and communication model + +For each rank, compute cost in nanoseconds is ```text -max(max(T_r) / mean(T_r), max(C_r) / mean(C_r)) - + 0.02 * max(cross_rank_boundary_count_r) +candidate_ns * exact_causal_candidates + + query_ns * query_rows + 1000 * span_us * execution_spans ``` -The boundary term prevents fine planning grids from winning on an immaterial -compute difference while introducing extra KDA state chains. The weights and -maximum grid are explicit configuration rather than hidden thresholds. - -The planner always evaluates the ordinary contiguous CP partition with the -same objective and returns the lowest-scoring candidate. Therefore its result -is never worse than contiguous under the declared model. This is a model-level -guarantee; GPU profiling is still needed to calibrate how candidate work, -token work, and boundary communication overlap on a target system. - -## Representative CP4 distributions - -The CPU benchmark is `benchmarks/large_cp_emulation/bench_thd_planner.py`. -The following table uses 1M total tokens and the default cost model. - -| THD distribution | selected grid | token max/avg | causal max/avg | selected score | contiguous score | remote boundaries | -|---|---:|---:|---:|---:|---:|---:| -| 1024 equal short samples | contiguous | 1.000 | 1.000 | 1.000 | 1.000 | 0 | -| one 1M sequence | 2 x CP | 1.000 | 1.000 | 1.080 | 1.790 | 6 | -| 20 x 1K + 2 x 514048 | 2 x CP | 1.000 | 1.000 | 1.040 | 1.560 | 4 | -| 256 x 1K + 1 x 768K | 3 x CP | 1.000 | 1.037 | 1.157 | 2.261 | 10 | -| 32 x 4K + 128K + 256K + 512K | 4 x CP | 1.000 | 1.046 | 1.166 | 2.302 | 10 | -| power-law 1K through 511K | 4 x CP | 1.000 | 1.028 | 1.188 | 2.149 | 15 | - -The important result is not that one grid wins everywhere. Low-fragmentation -`2 x CP` is selected for the original and single-long cases, while irregular -mixtures spend additional boundaries only when they materially improve both -token and causal balance. - -At logical CP64, the planning grid scales below the fixed 16K short-sample -threshold for long sequences. A one-million-token sequence therefore still -selects `2 x CP`, with exact token and causal balance. The original two-long -mixture selects `5 x CP` and reduces modeled causal max/average from 2.007 to -1.072; the power-law mixture also selects `5 x CP` and reduces it from 2.770 -to 1.034. The threshold is used only to preserve genuinely short samples, not -to prevent a dominant long sequence from forming sub-rank zigzag pieces. - -## Four-GPU DSA validation - -The layout-native DSA benchmark executes real deterministic cuDNN index -pooling and DSA selection, including pooled-key all-gather and global block-ID -remap. Each number below is the median critical-path GPU time after two warmups -and three measured iterations. All cases contain 1,048,576 global tokens. -These published measurements used four index heads; current benchmark output -records `index_heads` explicitly because the default model shape is now 64. - -| THD distribution | contiguous | planned | speedup | planned causal max/avg | -|---|---:|---:|---:|---:| -| one 1M sequence | 418.51 ms | 239.16 ms | 42.9% | 1.000 | -| 20 x 1K + 2 x 514048 | 166.77 ms | 109.05 ms | 34.6% | 1.000 | -| 256 x 1K + 1 x 768K | 291.80 ms | 134.35 ms | 54.0% | 1.037 | - -For the original mixture, planned per-rank selection time is approximately -108 ms and remap plus collective time is 0.55--0.96 ms. Under contiguous CP, -the light ranks spend 107--120 ms waiting at the collective while the heavy -ranks finish selection. For the single-long sequence, contiguous candidate -work grows from 8.59 to 60.13 billion across ranks; the planned layout gives -every rank exactly 34.36 billion candidates. This confirms that the speedup is -removal of causal load bubbles rather than a precision or kernel change. - -## Plan contract and integration boundary - -`THDLayoutPlan` records the global spans, owner rank, local `cu_seqlens`, -causal offsets, per-resource loads, and all candidate scores. Planning is -CPU-only and deterministic. It must run once at microbatch ingress, before the -model layer stack, rather than permuting tokens inside every KDA or CSA layer. - -The current KDA runtime still assumes one contiguous global token shard. A -separate lowering step is required before `THDLayoutPlan` becomes its default: - -1. derive every sequence's ordered `(sequence, span ordinal)` chain; -2. merge forward and backward KDA state in sequence order, not physical-rank - order; -3. produce CSA global block IDs and key-remap metadata from the same spans; -4. apply the token permutation once and retain it across all model layers. - -The benchmark adapter already lets DSA consume the planned layout. Keeping the -planner independent from KDA transport is intentional: layout policy and the -collective implementation can be validated and evolved separately. +The candidate count is an integer prefix-sum formula for the complete causal +blocks visible at each query. It includes original-document causal offsets, +not just each fragment's local length. Critical compute is the maximum over +ranks. + +Transport sums actual rows over source/destination edges. `bytes_per_query` +includes the outgoing Q/weights and returning IDs/scores in their real dtypes. +The critical transfer term takes the maximum of: + +- incoming and outgoing per-rank time at explicit intra/inter-node rates; +- the busiest aggregated peer edge, if `peer_gbps` is supplied; +- shared node injection/receive time, if `inter_node_aggregate_gbps` is supplied. + +Add the busiest rank's nonempty peer count times `peer_startup_us`, actual +permutation-copy work times `packing_ns_per_query`, and `exchange_us`. Multiple +fragments on one edge do not pay multiple peer startups. Naturally ordered +source/receive ranges skip `index_select` and carry zero copy cost on that side. +Global identity skips all transport terms. + +Ordered forward exchange, compute and inverse exchange costs add. The optional +`overlap_intra_inter` lets local and inter-node transfer resources overlap +inside collectives; it does not assume compute/communication overlap. Enable +it only after measuring mixed-link traffic. Coefficients are effective eager +implementation costs, not physical link specifications. The payload model is +shape-specific; different head counts, top-K, topology or software require +calibration and held-out validation. + +## Admissible pruning and preparation complexity + +Three lower bounds avoid building candidates that cannot meet the margin: + +- Every partition has the same token/candidate sums and at least one span per + document. Their mean is a compute lower bound. To reduce source rank i's + compute below C, some work must leave it; no moved row can remove more work + than its largest causal position allows. Optimistic outgoing bandwidth prices + these minimum rows. Minimize `C + max(outgoing_bound_i(C))` at the endpoints + and intersections of affine functions. With concurrent intra/inter links, + their capacities add in this optimistic bound. +- Full zigzag has a stronger bound: mean compute plus exact outgoing intra/inter + traffic. Consecutive pieces on the same node form interval runs. Intersecting + source spans with those runs and the owner's two pieces avoids constructing + O(documents * CP) objects. This bound omits incoming and peer bottlenecks and + permutation copies. Local candidates run first to tighten its incumbent. +- The long-document candidate family retains pinned source spans. Its critical + compute time cannot fall below the largest pinned per-rank compute load; + added nonnegative work and transport cannot lower it. If that bound already + misses the migration margin, skip the entire long-document search. Full + zigzag is checked independently because it can move the short documents too. + +Bounds are tested against exhaustive small assignments and concrete zigzag +layouts on contiguous, interleaved and irregular topologies. They cannot exclude +a strictly better modeled zigzag; they make no statement about model error. + +Repeated planning grids are deduplicated. Pinned spans are coalesced once; +their initial per-rank loads are computed once and copied for each grid. Adaptive +grids visit only movable documents. Owner groups are built in one pass, and +diagnostics are computed only for the +selected selection plan. Balanced-token scheduling uses a rank heap for +identical-cost pieces and indexes capacity repair by donor and piece shape. +Source-local spans have a direct intersection path; other source intersections +use binary search. A geometry-only grain check rejects infeasible full zigzag +before transport pricing, preserving the original-source-fragment exception. +These changes preserve the scores of evaluated candidates, choices and stable +tie breaks. Diagnostics can omit candidates proven unable to meet the margin. + +DSA window construction checks whether all remaining rows fit before searching +for a split. It uses exact integer work, including unaligned causal offsets; +long documents still receive the same maximal budget-fitting prefixes. An +integer candidate total avoids rescanning all preceding short documents; +window boundaries, scratch requirements and numerical execution are unchanged. +All reuse is local to one preparation call; there is no retained layout cache +or new device-to-host metadata read. + +## Calibrating and measuring + +`benchmarks/calibrate_thd_selection.py` fits nonnegative candidate/query/span +costs from unmodeled selection runs and retains source JSON digests. Pipeline +exchange measurements use differences of maximum cumulative phase endpoints, +so KDA/selection arrival waits are not counted again as network work. + +`benchmarks/bench_thd_transport.py` measures the actual Q/weight and inverse +ID/score exchanges with exact marker checks. Sparse peer pairs, node-local, +inter-node and dense exchanges exercise different message sizes. The 32768-row +and striped fixtures are held out. `--validation-only` measures additional +mixed-link fractions. `benchmarks/fit_thd_transport.py` fits nonnegative resource +costs without using held-out rows in its objective. Ordered fixtures do not +identify copy bandwidth: measure permutation copies separately and supply that +coefficient in the input profile. Unidentifiable node limits must not be +interpreted as measured physical bandwidth. + +The [example profiles](../benchmarks/profiles/README.md) are optional deployment +inputs, not library defaults. The runner's `--suite validation` fixes the 31 +measured THD shapes. Use `summarize_csa_layout_matrix.py` to check complete +output/gradient evidence, compatible configurations, all-rank samples and +preparation accounting before producing a comparison. Preparation includes +common metadata, layout/index/workspace construction and KDA transport buffers. +Report it separately and added to one iteration; amortize only when reuse is real. + +## KDA integration + +The [Kimi CP runtime](KIMI_CP_RUNTIME.md) requires contiguous source ownership. +When every internal shard boundary is a document boundary, all ranks agree that +no recurrence crosses ranks. Native local KDA/autograd then skips summaries and +boundary collectives, with unchanged operands, dtypes and batch-invariance. +One isolated rank cannot bypass exchanges required by other ranks. Manually +constructed plans conservatively default to requiring exchange. + +On the native summary path, backward context is reused only if the summarized +first span covers the complete local shard. A prefix-only context does not +contain the full THD shard's per-token intermediates; full apply prepares its +own intermediates instead. The paired independent-shard benchmark checks the +fast path against forced boundary execution, including weak decay and a +crossing-document control. diff --git a/docs/THD_REDISTRIBUTION.md b/docs/THD_REDISTRIBUTION.md new file mode 100644 index 00000000..ecce6f86 --- /dev/null +++ b/docs/THD_REDISTRIBUTION.md @@ -0,0 +1,99 @@ +# Executing and reversing a THD query layout + +`plan_thd_layout` and `plan_thd_selection` choose global token spans. `THDRedistributionPlan` and +`THDRedistributor`, exported from `cula.distributed`, execute that assignment +from existing contiguous CP shards and restore the original ownership/order. +The planner's modeled score is not an end-to-end latency guarantee; see the +[real CP16/CP8 measurements](THD_SELECTION_RESULTS.md). + +## Contract + +Each process-group rank starts with rows in +`[source_offsets[rank], source_offsets[rank + 1])`. Offsets start at zero and +are nondecreasing; source and target row counts need not be equal. The target +order is the concatenation of that rank's `THDRankLayout.spans`, in the supplied +order. A plan validates full coverage, span ownership, sequence containment, +and absence of gaps/overlap. Source offsets and layouts must be identical on +every peer; rank numbers are **process-group ranks**, not global ranks. + +Preparation builds CPU interval metadata and device row permutations once. +Execution packs rows as necessary, performs a real variable-split +`all_to_all_single`, and unpacks into target span order. `restore` reverses both +the communication splits and the permutations. There is no cast, reduction, +padding, or replicated full-tensor gather in this transport. An identity +layout skips the collective only when **all ranks** are unchanged, and returns +the input tensor itself. Callers must account for this alias before modifying +a returned tensor in place. + +All peers must call the exchanges in the same order with matching trailing +shapes and dtypes. Input tensors must have contiguous rows on the explicitly +selected device; `"cuda"` resolves to the current device during preparation. +An explicit device such as `torch.device('cuda', local_rank)` is also supported. +The caller owns any necessary conversion to contiguous storage. The process +group must outlive every backward using the redistributor. Backward participation +and collective order must also agree across ranks; rank-local conditional losses +must not cause peers to skip a required inverse exchange. + +Prepare on the execution stream, or establish an explicit stream dependency +before using the device indices. Keep collective ordering consistent across +peers. Permutation indices are read-only after preparation; each nonidentity +exchange allocates its own packed/received outputs rather than overwriting +transport storage retained by an earlier autograd invocation. + +## Integration + +```python +from cula.distributed import ( + THDRedistributionPlan, + THDRedistributor, + plan_thd_layout, +) + +# global_cu_seqlens and source_offsets are the same CPU metadata on all peers. +layout = plan_thd_layout(global_cu_seqlens, world_size=cp_world_size) +route = THDRedistributionPlan.build(layout.ranks, source_offsets, cp_rank) +exchange = THDRedistributor(route, local_q.device, group=cp_group) + +target_q = exchange.distribute(local_q) +# The attention call must use this rank's span metadata, original document +# origins, and causal offsets. KV ownership is a separate contract. +target_output = attention_for_spans(target_q, layout.for_rank(cp_rank)) +local_output = exchange.restore(target_output) +``` + +`distribute` backward invokes the inverse exchange; `restore` backward invokes +the forward exchange. First-order autograd is supported; higher-order gradients +are explicitly disabled. The full CSA benchmark also exercises the explicit +inverse APIs because native backward wrappers return gradient tensors directly. + +This does not turn arbitrary attention into distributed attention: global KV +visibility, document isolation, causal masking, KV-gradient reduction and +replicated-parameter reduction remain the integration's responsibility. In +particular, do not pass a shuffled set of recurrent KDA spans into a contiguous +Kimi CP boundary plan. The CP16 experiment keeps KDA and KV/pooling ownership +contiguous, redistributes CSA query-side operands, and restores CSA outputs and +query-side gradients at the layer boundary. + +For the selection-only strategy, use `plan_thd_selection` with calibrated costs +and an explicit row capacity. Route indexer Q/weights to its spans and inverse-route +the selected IDs/scores, then run attention in original source ownership. The +[planner contract](THD_LAYOUT_PLANNER.md#capacity-bounded-selection-planner) +describes this policy and its preparation/reuse budget. + +## Execution and validation scope + +This is an eager PyTorch/NCCL implementation with output/packing allocations. +It does not promise CUDA Graph capture, reusable tensor workspaces, asynchronous +overlap, or a topology-aware communication schedule. Tensor lifetimes can be +material at MLA widths and should be included in memory measurements. + +CPU protocol tests cover unequal counts, reversed target span order, sender +packing, receiver unpacking, inverse counts and identity skipping. Real CP16 +tests on two H20 nodes verify token/channel markers and both autograd directions +for all three layouts, including all 31 measured 1M distributions. Full-tensor CSA +output/gradient comparisons are in the linked report. The CSA harness requires +ratio-4-aligned document/source boundaries. In its default selection scope, +empty selection ranks skip native plan construction and selection kernels but +still participate in all exchanges. Attention stays in nonempty source shards. +The optional query-wide scope still requires nonempty attention query ranks; +these tests do not establish support for unaligned document boundaries. diff --git a/docs/THD_SELECTION_RESULTS.md b/docs/THD_SELECTION_RESULTS.md new file mode 100644 index 00000000..276be400 --- /dev/null +++ b/docs/THD_SELECTION_RESULTS.md @@ -0,0 +1,182 @@ +# Real-CP THD selection results (2026-11-15) + +This is a **1 KDA + 1 complete CSA operator-chain** forward/backward study. +KDA and CSA use independent synthetic inputs; projections, residuals, MLP, +optimizer and the full training model are excluded. It is not a seven-KDA/one-CSA +model measurement. The 31 shapes contain exactly 1,048,576 tokens each. + +## Setup and accounting + +- H20-3e: two nodes, eight GPUs per node, real CP16. B300: one node, real CP8. +- cuDNN Frontend commit `855ccd77ba9efed22441fd454f7cc0206a068e94`, backend 9.20.0.48, CuTeDSL 4.8.1. +- KDA: H32, K=V=128, batch invariant enabled; CSA MLA-absorb: H64, D576, V512; indexer: H64/D128, Top-K512, ratio-4 pooling. +- Non-deterministic selection/pooling/sparse backward. The transport never changes dtypes; pool gradients retain the per-loss-rank ordering before FP32 accumulation. +- Three warmups and ten timed iterations per policy/shape. Time is the median of each iteration's maximum-rank duration. +- Timed work includes all runtime layouts and activation/gradient communication, pooling, Top-K, sparse attention, detached teacher/KL and all backwards. +- Preparation separately includes common metadata, planning, index/workspace materialization and KDA transport allocation. Fresh-layout cost adds this **once**, without amortization. RNG generation, JIT warmup and numerical oracles are excluded. +- Memory is peak allocated/reserved for the entire measured chain, **including inputs, outputs and gradients**, not workspace-only memory. + +## Results + +C = contiguous; Z = complete per-document zigzag; P = calibrated selection planner. +All table times are milliseconds. The machine-readable [results](results/thd_selection_20261115.json) +include per-policy preparation, memory, errors and original record hashes. + +| Case | Docs | H20 C | H20 Z | H20 P | B300 C | B300 Z | B300 P | +|---|---:|---:|---:|---:|---:|---:|---:| +| 20s2l | 22 | 1770.25 | 1361.15 | 1326.95 | 606.72 | 525.02 | 523.09 | +| count-mixed-1000-1long | 1000 | 1738.30 | 1107.86 | 1078.85 | 564.92 | 480.15 | 466.27 | +| count-mixed-256-1long | 256 | 1933.60 | 1223.23 | 1197.76 | 646.42 | 509.91 | 494.01 | +| count-mixed-512-2long | 512 | 1235.56 | 942.93 | 932.91 | 441.96 | 421.93 | 407.34 | +| count-mixed-512-2long-ascending | 512 | 1240.10 | 947.32 | 938.54 | 466.90 | 434.59 | 421.41 | +| count-mixed-512-2long-descending | 512 | 1241.66 | 936.86 | 933.32 | 466.69 | 430.64 | 412.11 | +| count-mixed-512-4long | 512 | 972.04 | 880.24 | 874.13 | 338.36 | 338.60 | 338.46 | +| count-near-uniform-512 | 512 | 452.80 | 489.87 | 452.81 | 174.68 | 185.55 | 174.55 | +| count-tiny-mix-1000 | 1000 | 804.45 | 819.81 | 803.92 | 277.50 | 288.30 | 277.98 | +| count-tiny-mix-512 | 512 | 685.93 | 716.08 | 685.85 | 252.41 | 260.91 | 252.61 | +| count-uniform-1024 | 1024 | 304.09 | 345.73 | 303.89 | 119.41 | 133.69 | 119.48 | +| count-uniform-256 | 256 | 549.62 | 584.04 | 549.45 | 213.00 | 220.55 | 213.09 | +| jittered-768-s14 | 768 | 358.52 | 397.64 | 358.48 | 139.60 | 152.69 | 139.54 | +| jittered-768-s15 | 768 | 358.78 | 398.11 | 358.70 | 139.20 | 152.08 | 138.97 | +| jittered-768-s16 | 768 | 359.07 | 398.91 | 359.14 | 139.43 | 152.10 | 139.20 | +| lognormal-1000-s14 | 1000 | 685.00 | 700.99 | 684.54 | 242.74 | 255.30 | 242.80 | +| lognormal-1000-s15 | 1000 | 763.70 | 760.71 | 763.37 | 238.08 | 248.49 | 238.69 | +| lognormal-1000-s16 | 1000 | 736.00 | 735.10 | 736.39 | 243.51 | 255.28 | 244.25 | +| lognormal-257-s14 | 257 | 666.27 | 694.94 | 666.14 | 241.62 | 249.37 | 241.79 | +| lognormal-257-s15 | 257 | 786.76 | 774.97 | 786.91 | 265.90 | 270.33 | 266.58 | +| lognormal-257-s16 | 257 | 792.44 | 789.95 | 792.44 | 260.86 | 265.92 | 261.18 | +| tiny-tail-768-s14 | 768 | 1157.98 | 905.56 | 887.67 | 436.30 | 406.08 | 397.62 | +| tiny-tail-768-s15 | 768 | 1348.06 | 970.18 | 959.86 | 498.77 | 423.77 | 416.50 | +| tiny-tail-768-s16 | 768 | 1590.68 | 1054.16 | 1043.18 | 559.72 | 466.85 | 460.22 | +| trimodal-513-s14 | 513 | 1590.94 | 1143.29 | 1123.55 | 528.07 | 482.18 | 471.44 | +| trimodal-513-s15 | 513 | 1565.77 | 1154.85 | 1138.55 | 550.40 | 479.26 | 475.16 | +| trimodal-513-s16 | 513 | 1917.07 | 1225.09 | 1195.03 | 606.03 | 509.75 | 498.81 | +| near-uniform | 64 | 657.86 | 691.02 | 657.87 | 261.57 | 266.84 | 261.59 | +| power-law | 7 | 1788.44 | 1200.61 | 1195.98 | 612.98 | 493.15 | 492.85 | +| single | 1 | 2908.84 | 1934.28 | 1926.76 | 879.36 | 667.50 | 664.84 | +| uniform | 64 | 653.51 | 690.32 | 657.16 | 261.74 | 267.41 | 261.72 | + +| Metric | H20 CP16 | B300 CP8 | +|---|---:|---:| +| Largest P slowdown vs faster C/Z, steady | 1.541% | 0.304% | +| Largest P slowdown vs faster C/Z, prepare + one iteration | 4.038% | 5.188% | +| Best C/P speedup | 1.614x | 1.323x | +| Mean P preparation | 34.118 ms | 18.972 ms | +| Mean extra P vs C preparation / mean P steady time | 3.038% | 2.518% | +| All P preparation / all P steady time | 4.011% | 5.647% | +| contiguous max allocated GiB | 44.91 | 72.95 | +| zigzag max allocated GiB | 46.20 | 73.43 | +| planner max allocated GiB | 46.02 | 75.91 | + +These results do **not** show universal optimality or a strict <3% preparation +guarantee. The complete planner preparation is larger than its net incremental +cost over contiguous, and the fresh-layout metric is more conservative than +steady-state throughput. Preparation was sampled once per case, so host jitter +is visible. Profile margins are explicit: 5% plus 40 ms on H20 and 20 ms on B300. + +## Precision and calibration + +All 186 policy/shape/platform results passed finite-value checks. Forward, inverse +and both autograd row-marker checks were bitwise exact. Every policy used the +same source input fingerprints and compared all 14 output/gradient tensors +against a lossless CPU copy of contiguous results. KDA output and all five KDA +gradients had zero difference. Worst CSA relative RMS was 0.145868% on H20 and +0.161238% on B300; maximum auxiliary KL absolute differences were 3.2014e-10 +and 1.7462e-10. These are auxiliary indexer losses, not training loss results. + +Four additional small FP32 attention/teacher/KL/pooling-gradient fixtures per +platform were tested under all three policies. The reference uses the selected +key set and checks causality, document isolation, uniqueness and tail coverage; +it does not independently prove the selector's discrete ranking. + +Compute coefficients used seven calibration shapes and their C/Z runs. +The other 24 matrix cases were not used for compute fitting. Independent +communication sweeps held out sizes/topologies: absolute percentage error +averaged 7.14% on H20 and 10.53% on B300. Four additional mixed-link H20 +checks had 2.01--3.73% error. These are effective-cost models, not physical +network simulators. No final E2E result is used as a case-specific lookup. + +## Reproduction + +Install the compatible cuDNN build and cuLA's `cudnn-hybrid` extra first. Under +CUDA 13, preserve the `[cu13]` extra when pinning `nvidia-cutlass-dsl`. Use the +appropriate NCCL/rendezvous configuration for the machine; rank-to-node mapping +must match the explicit profile. For the single-node B300 configuration: + +```bash +OMP_NUM_THREADS=4 torchrun --standalone --nproc-per-node=8 -m benchmarks.bench_csa_layout_matrix \ + --suite validation --heads 64 --warmups 3 --repeats 10 \ + --planner-strategy selection \ + --planner-cost-model benchmarks/profiles/thd_selection_b300_cp8.json \ + --output-dir results/b300-cp8 +python -m benchmarks.summarize_csa_layout_matrix results/b300-cp8 --output results/b300-summary.json +``` + +For H20, launch on two eight-GPU nodes with `--nnodes=2`, distinct node ranks +and shared rendezvous; select `thd_selection_h20_cp16.json`. The runner first +executes small FP32 checks, then all 31 cases, keeping contiguous first and +alternating the other two policies. `--list-cases` lists shapes without CUDA. + +## Original submission source review (2026-11-15) + +The full matrix above precedes the final CPU metadata cleanup. This cleanup +rejects noninteger transport intervals, removes an obsolete benchmark import, +fixes preparation accounting in the public summarizer and replaces a quadratic +DSA window candidate sum with an integer running total. Planner decisions and +numerical kernels are unchanged. That submission has separate regression +validation recorded below; the historical matrix is not relabeled as a new run. + +Local CPU preparation microbenchmark (10 measured samples after one warmup): + +| Documents | Old window preparation ms | New ms | Metadata | +|---|---:|---:|---| +| 22 | 0.087 | 0.083 | field-for-field equal | +| 1000 | 11.527 | 2.693 | field-for-field equal | +| 4096 | 153.647 | 7.648 | field-for-field equal | +| 16384 | 2404.650 | 25.638 | field-for-field equal | + +These CPU-only measurements are not GPU or whole-chain speedups. The change +removes repeated scans of already-grouped documents; it does not change scratch +capacity, math precision or kernel launch boundaries. +## Original submission GPU regression + +On both H20 CP16 and B300 CP8, the original submission source passed four small FP32 fixtures +and three representative 1M shapes (single long, 1000 documents with one long, +and 1000-document tiny mixture), each under C/Z/P: **42 complete CSA rows**. +The 1M rerun used three warmups and five measurements. A separate six-case KDA +A/B on each platform compared native document-local execution with the forced +boundary route, including weak decay and isolated ranks with a crossing peer: +**24 A/B rows**, with zero difference in output and all five gradients. + +All 181 tested source/config files matched the original submission. The 62 original +planner decisions still match every measured rank's policy, row count, span +count and remote rows. All **2,232** DSA rank plans across the full 31 x 3 x +(CP16 + CP8) matrix have field-for-field identical window metadata before and +after the integer running-total optimization. The [regression record](results/thd_selection_review_20261115.json) +includes source hashes, correctness maxima and per-case timings. + +337 focused CPU tests and both repository pre-commit hooks passed. Review +covered the complete proposed diff and its distributed/hybrid call paths: +coverage and ownership, collective participation, inverse autograd, storage +lifetimes, context reuse, cost-model bounds and benchmark accounting. No +handwritten GPU kernel, precision mode or math approximation is changed. +This is not a claim of a complete race audit of every unchanged kernel in cuLA. + + +## Post-review validation and timing protocol + +The subsequent review fixes add group-wide source-alignment rejection, finite +calibration-input validation, strict JSON failure reports, and the SM90 +independent-shard option preflight. The independent-shard KDA benchmark now +warms both routes and alternates baseline/candidate order for every measured +pair. Each raw sample records its pair index, actual execution order and +position; new reports identify this as `paired-alternating-v1`. Numerical +output/gradient comparison runs separately after both routes' timed samples. + +The historical JSON result files and tables above are unchanged. In particular, +the original KDA A/B timings ran all baseline samples before all candidate +samples; they are not measurements under the new paired protocol. The recorded +source hashes describe those historical runs, not the later review fixes. +No fresh GPU, NCCL or performance rerun accompanies these fixes. Their +validation consists of focused CPU regressions and repository-wide Ruff +lint/format checks; a new paired GPU run is needed for updated timing claims. diff --git a/docs/results/csa_cp_preparation_20261116.json b/docs/results/csa_cp_preparation_20261116.json new file mode 100644 index 00000000..bccabb52 --- /dev/null +++ b/docs/results/csa_cp_preparation_20261116.json @@ -0,0 +1,6770 @@ +{ + "scope": "Complete CSA operator forward/backward, including layouts and communication; no KDA/projections/MoE", + "platform": "B300 CP8", + "baseline_commit": "ea71df3b3bfad69141d1ee3ddd884ac9ccdb27ce", + "native_source_commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "960a403b1e804b087653fd5c8032781fd2d1249f37a6a7b0f2b80c3fca1896f6", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "853f7045ead4d230d4bd5b0a211d60861c840c6190c6502bfdcde310cf866bc2", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0", + "cudnn_backend": "9.20.0.48", + "cudnn_frontend_version_scope": "Wheel metadata; imported source identified by native_source_commit" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "arguments": { + "tokens": [ + 131072, + 262144, + 1048576 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "cases": { + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "7ca4febcee1eaf89e99486a3e54fddeeb01614dd16c887ac63a35df7e1fd86e5", + "policies": { + "contiguous": { + "step_ms": 47.692832946777344, + "prepare_ms": 0.7185455178841949, + "one_use_ms": 48.41137846466154, + "prepare_min_ms": 0.699331983923912, + "prepare_max_ms": 0.7794059347361326, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015779392555523083, + "dq": 0.000579412002925132, + "dkv": 0.0006017076308924175, + "d_index_q": 0.000795245110121862, + "d_weights": 0.0006388088783563453, + "d_pool_kv": 0.0007155220754445791, + "d_pool_score": 0.0007320226024776074, + "d_ape": 2.5403258387015062e-05 + }, + "max_loss_abs_diff": 9.89530235528946e-10, + "critical_step_samples_ms": [ + 47.38896179199219, + 47.49897766113281, + 47.80780792236328, + 47.66294479370117, + 47.76643371582031, + 47.85734558105469, + 47.692832946777344 + ], + "critical_prepare_samples_ms": [ + 0.7794059347361326, + 0.7474300218746066, + 0.7116730557754636, + 0.7364189950749278, + 0.7166480645537376, + 0.7204429712146521, + 0.6996119627729058, + 0.699331983923912 + ], + "baseline_prepare_ms": 0.7295145187526941 + }, + "zigzag": { + "step_ms": 46.274879455566406, + "prepare_ms": 0.8905025315470994, + "one_use_ms": 47.165381987113506, + "prepare_min_ms": 0.8781669894233346, + "prepare_max_ms": 2.297741943039, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015800177020671097, + "dq": 0.0005803295677451422, + "dkv": 0.000603814033831282, + "d_index_q": 0.0005765121709522783, + "d_weights": 0.0005104610024849347, + "d_pool_kv": 0.0007192810999619293, + "d_pool_score": 0.0007225948121424813, + "d_ape": 2.3842562812573465e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-10, + "critical_step_samples_ms": [ + 45.98963165283203, + 46.42329788208008, + 46.58995056152344, + 46.043582916259766, + 46.320030212402344, + 46.1190071105957, + 46.274879455566406 + ], + "critical_prepare_samples_ms": [ + 2.297741943039, + 0.9367750026285648, + 0.8781669894233346, + 0.8949730545282364, + 0.8860320085659623, + 0.9126270888373256, + 0.8803660748526454, + 0.8839479414746165 + ], + "baseline_prepare_ms": 0.9122525225393474 + }, + "planner": { + "step_ms": 45.82748794555664, + "prepare_ms": 7.445039984304458, + "one_use_ms": 53.2725279298611, + "prepare_min_ms": 7.354012923315167, + "prepare_max_ms": 7.553054951131344, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.001576013197553258, + "dq": 0.000578531419351737, + "dkv": 0.0005801127675590207, + "d_index_q": 0.0004534482180659951, + "d_weights": 0.00043037340000837636, + "d_pool_kv": 0.0007108593320852681, + "d_pool_score": 0.0007300664640860723, + "d_ape": 2.353131665386829e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-09, + "critical_step_samples_ms": [ + 45.80588912963867, + 46.03481674194336, + 45.82748794555664, + 45.93056106567383, + 45.71779251098633, + 46.30985641479492, + 45.72528076171875 + ], + "critical_prepare_samples_ms": [ + 7.553054951131344, + 7.538643083535135, + 7.417501998133957, + 7.354012923315167, + 7.4830540688708425, + 7.437848136760294, + 7.372890948317945, + 7.452231831848621 + ], + "baseline_prepare_ms": 7.54898099694401 + } + } + }, + "128k-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "23706ee0a647a3cb27010c2ed87cc4d8b06a80125c3b8c22123ae1a312a4a571", + "policies": { + "contiguous": { + "step_ms": 19.99852752685547, + "prepare_ms": 0.8981814607977867, + "one_use_ms": 20.896708987653255, + "prepare_min_ms": 0.8844570256769657, + "prepare_max_ms": 0.9815751109272242, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.5358557388634415e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.850226281255351e-05, + "d_pool_score": 6.240650825233437e-05, + "d_ape": 1.9435079133677194e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 19.997407913208008, + 19.980575561523438, + 20.005632400512695, + 20.014080047607422, + 19.99852752685547, + 19.980127334594727, + 20.010080337524414 + ], + "critical_prepare_samples_ms": [ + 0.9815751109272242, + 0.9244229877367616, + 0.8844570256769657, + 0.9146069642156363, + 0.884861103259027, + 0.8991919457912445, + 0.8856870699673891, + 0.8971709758043289 + ], + "baseline_prepare_ms": 0.9577220189385116 + }, + "zigzag": { + "step_ms": 20.669599533081055, + "prepare_ms": 3.7295810761861503, + "one_use_ms": 24.399180609267205, + "prepare_min_ms": 3.5765160573646426, + "prepare_max_ms": 4.66969795525074, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.199779774104606e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.823004119093153e-05, + "d_pool_score": 6.126991867025295e-05, + "d_ape": 1.8107487625478647e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.700927734375, + 20.724992752075195, + 20.669599533081055, + 20.676511764526367, + 20.666271209716797, + 20.65430450439453, + 20.660608291625977 + ], + "critical_prepare_samples_ms": [ + 3.811003058217466, + 4.470668034628034, + 3.6063099978491664, + 3.6481590941548347, + 4.66969795525074, + 3.6183729534968734, + 4.324503941461444, + 3.5765160573646426 + ], + "baseline_prepare_ms": 4.178662959020585 + }, + "planner": { + "step_ms": 19.990047454833984, + "prepare_ms": 1.1078330571763217, + "one_use_ms": 21.097880512010306, + "prepare_min_ms": 1.0634340578690171, + "prepare_max_ms": 1.1886698193848133, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.403263577176253e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 8.073828813471261e-05, + "d_pool_score": 7.995610208711404e-05, + "d_ape": 2.2451904402386835e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 19.990047454833984, + 20.148128509521484, + 19.977664947509766, + 20.00320053100586, + 19.976255416870117, + 20.004608154296875, + 19.986560821533203 + ], + "critical_prepare_samples_ms": [ + 1.1886698193848133, + 1.1112691136077046, + 1.1124929878860712, + 1.0784630430862308, + 1.130640972405672, + 1.1040159733965993, + 1.1043970007449389, + 1.0634340578690171 + ], + "baseline_prepare_ms": 1.2417834950610995 + } + } + }, + "128k-near-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "7a9bb46a066badd62c28275ab7d42939639e7bca37e919dc022bff73cf1a20fe", + "policies": { + "contiguous": { + "step_ms": 20.031328201293945, + "prepare_ms": 1.0363015462644398, + "one_use_ms": 21.067629747558385, + "prepare_min_ms": 0.9917590068653226, + "prepare_max_ms": 1.106114941649139, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016380789447513027, + "dq": 8.525440840760677e-05, + "dkv": 0.00013011699218693176, + "d_index_q": 2.42101830699352e-05, + "d_weights": 3.241630010282288e-05, + "d_pool_kv": 7.695981858658009e-05, + "d_pool_score": 9.499932585619792e-05, + "d_ape": 3.1762099590003316e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.028352737426758, + 19.99331283569336, + 20.05708885192871, + 20.031328201293945, + 20.022687911987305, + 20.038463592529297, + 20.053375244140625 + ], + "critical_prepare_samples_ms": [ + 1.106114941649139, + 1.0442269267514348, + 1.0163340484723449, + 1.0369500378146768, + 1.0006509255617857, + 1.0416280711069703, + 0.9917590068653226, + 1.0356530547142029 + ], + "baseline_prepare_ms": 0.9788154857233167 + }, + "zigzag": { + "step_ms": 20.757631301879883, + "prepare_ms": 4.550810961518437, + "one_use_ms": 25.30844226339832, + "prepare_min_ms": 4.380041034892201, + "prepare_max_ms": 6.088564987294376, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.00016408046524629192, + "dq": 8.506906905685405e-05, + "dkv": 0.00013016442209025538, + "d_index_q": 2.4266660285821442e-05, + "d_weights": 3.5896669106976304e-05, + "d_pool_kv": 9.384115935688863e-05, + "d_pool_score": 0.00012792608827345438, + "d_ape": 3.490957941984386e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.83123207092285, + 20.77631950378418, + 20.74355125427246, + 20.726688385009766, + 20.844192504882812, + 20.74188804626465, + 20.757631301879883 + ], + "critical_prepare_samples_ms": [ + 4.700917052105069, + 4.677826887927949, + 4.382802057079971, + 4.414985072799027, + 4.380041034892201, + 4.423795035108924, + 5.361750023439527, + 6.088564987294376 + ], + "baseline_prepare_ms": 4.162916971836239 + }, + "planner": { + "step_ms": 20.02444839477539, + "prepare_ms": 1.2869200436398387, + "one_use_ms": 21.31136843841523, + "prepare_min_ms": 1.2415660312399268, + "prepare_max_ms": 1.3624459970742464, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016313288531341212, + "dq": 8.540299959803365e-05, + "dkv": 0.00012140824119413083, + "d_index_q": 2.4035200300153297e-05, + "d_weights": 2.2592142747911436e-05, + "d_pool_kv": 7.511559012165556e-05, + "d_pool_score": 7.687588823617633e-05, + "d_ape": 2.910722067354833e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.129823684692383, + 20.039775848388672, + 20.014432907104492, + 20.02444839477539, + 20.013120651245117, + 20.06505584716797, + 20.01033592224121 + ], + "critical_prepare_samples_ms": [ + 1.3624459970742464, + 1.2910119257867336, + 1.301727956160903, + 1.2415660312399268, + 1.2881830334663391, + 1.2434010859578848, + 1.2856570538133383, + 1.2555728899315 + ], + "baseline_prepare_ms": 1.181907020509243 + } + } + }, + "128k-mixed-22": { + "documents": 22, + "tokens": 131072, + "source_record_sha256": "6ecd138552df3ddc716d8f2b5546c243019202c335e763cb31803e5a23d38a46", + "policies": { + "contiguous": { + "step_ms": 35.410911560058594, + "prepare_ms": 0.754816981498152, + "one_use_ms": 36.165728541556746, + "prepare_min_ms": 0.7329720538109541, + "prepare_max_ms": 1.6807179199531674, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0019090964634188628, + "dq": 0.0005760018331729727, + "dkv": 0.0006459942222257381, + "d_index_q": 0.0009173576570437036, + "d_weights": 0.0005682475663426924, + "d_pool_kv": 0.000817398720386681, + "d_pool_score": 0.0008227400563089687, + "d_ape": 1.4598269933704832e-05 + }, + "max_loss_abs_diff": 5.238689482212067e-10, + "critical_step_samples_ms": [ + 35.033023834228516, + 34.84620666503906, + 35.41068649291992, + 35.410911560058594, + 35.445377349853516, + 35.47119903564453, + 35.475006103515625 + ], + "critical_prepare_samples_ms": [ + 0.823180889710784, + 0.756045919843018, + 1.6807179199531674, + 0.7380660390481353, + 0.8030008757486939, + 0.753588043153286, + 0.7329720538109541, + 0.7516909390687943 + ], + "baseline_prepare_ms": 0.8113369694910944 + }, + "zigzag": { + "step_ms": 35.35753631591797, + "prepare_ms": 1.8669585115276277, + "one_use_ms": 37.2244948274456, + "prepare_min_ms": 1.8487819470465183, + "prepare_max_ms": 3.108761040493846, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0019284075136349816, + "dq": 0.0005817180180578517, + "dkv": 0.00046197630358230487, + "d_index_q": 0.0003621120560024531, + "d_weights": 0.00034954515220108934, + "d_pool_kv": 0.0006609639282456203, + "d_pool_score": 0.0006636759480983085, + "d_ape": 1.4089509206796339e-05 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 35.02534484863281, + 35.17510223388672, + 36.00688171386719, + 35.21503829956055, + 35.84960174560547, + 35.35753631591797, + 36.07881546020508 + ], + "critical_prepare_samples_ms": [ + 3.108761040493846, + 1.942998031154275, + 1.861648983322084, + 1.8494660034775734, + 1.8487819470465183, + 1.8964800983667374, + 1.8524520564824343, + 1.8722680397331715 + ], + "baseline_prepare_ms": 1.9275369704701006 + }, + "planner": { + "step_ms": 35.47750473022461, + "prepare_ms": 7.7040985925123096, + "one_use_ms": 43.18160332273692, + "prepare_min_ms": 7.597577874548733, + "prepare_max_ms": 9.156922926194966, + "selected_policy": "selection-local-3x", + "max_relative_rms": { + "output": 0.0019138774083845046, + "dq": 0.000577212651210394, + "dkv": 0.00046300417813060113, + "d_index_q": 0.00036281654146900024, + "d_weights": 0.00034608194650246384, + "d_pool_kv": 0.0006540298845914364, + "d_pool_score": 0.0006691398806773541, + "d_ape": 1.3458997363557325e-05 + }, + "max_loss_abs_diff": 4.0745362639427185e-10, + "critical_step_samples_ms": [ + 34.82156753540039, + 35.76873779296875, + 35.48476791381836, + 35.47750473022461, + 35.731231689453125, + 35.40195083618164, + 35.03801727294922 + ], + "critical_prepare_samples_ms": [ + 9.156922926194966, + 7.684374926611781, + 7.653841050341725, + 7.862776052206755, + 7.597577874548733, + 7.689073099754751, + 7.847677916288376, + 7.719124085269868 + ], + "baseline_prepare_ms": 7.6061730505898595 + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "e42faf3fe40f13f60990140dc3fa5dac55345f029f7783599374a9ac2e46649d", + "policies": { + "contiguous": { + "step_ms": 9.31715202331543, + "prepare_ms": 6.460049480665475, + "one_use_ms": 15.777201503980905, + "prepare_min_ms": 6.399138015694916, + "prepare_max_ms": 6.61184499040246, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8425583640526028e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.07692854412947e-05, + "d_pool_score": 4.358570392543705e-05, + "d_ape": 2.1843910536361686e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.288512229919434, + 9.269824028015137, + 9.289088249206543, + 9.31715202331543, + 9.563712120056152, + 9.406111717224121, + 9.327136039733887 + ], + "critical_prepare_samples_ms": [ + 6.61184499040246, + 6.471202941611409, + 6.399138015694916, + 6.513276020996273, + 6.425823085010052, + 6.459023919887841, + 6.449210923165083, + 6.4610750414431095 + ], + "baseline_prepare_ms": 6.7794660571962595 + }, + "zigzag": { + "step_ms": 11.095487594604492, + "prepare_ms": 43.545027961954474, + "one_use_ms": 54.64051555655897, + "prepare_min_ms": 43.15978987142444, + "prepare_max_ms": 159.60040991194546, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.823962386686128e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.014173056955901e-05, + "d_pool_score": 3.808940089626602e-05, + "d_ape": 2.3177467841882244e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.1212797164917, + 11.13651180267334, + 11.091487884521484, + 11.085920333862305, + 11.095487594604492, + 11.101375579833984, + 11.079903602600098 + ], + "critical_prepare_samples_ms": [ + 46.472262940369546, + 43.510618968866765, + 43.37981704156846, + 159.60040991194546, + 44.112925068475306, + 43.441104935482144, + 43.15978987142444, + 43.57943695504218 + ], + "baseline_prepare_ms": 47.22392506664619 + }, + "planner": { + "step_ms": 9.282464027404785, + "prepare_ms": 8.517480513546616, + "one_use_ms": 17.7999445409514, + "prepare_min_ms": 8.404429885558784, + "prepare_max_ms": 8.715566829778254, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8184966417286295e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.250769702550404e-05, + "d_pool_score": 3.6555913008951535e-05, + "d_ape": 2.173760194321718e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.282464027404785, + 9.294624328613281, + 9.262335777282715, + 9.278528213500977, + 9.307040214538574, + 9.284832000732422, + 9.274815559387207 + ], + "critical_prepare_samples_ms": [ + 8.67331400513649, + 8.40490497648716, + 8.536944980733097, + 8.404429885558784, + 8.61146894749254, + 8.442464983090758, + 8.498016046360135, + 8.715566829778254 + ], + "baseline_prepare_ms": 8.97316396003589 + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "eba0eefb74fef716f6c396b10698ddd865c6ba04c6b69c37170224765a3ca7af", + "policies": { + "contiguous": { + "step_ms": 9.996671676635742, + "prepare_ms": 3.5602174466475844, + "one_use_ms": 13.556889123283327, + "prepare_min_ms": 3.4920680336654186, + "prepare_max_ms": 3.646206925623119, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2234539065804016e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.212013067353867e-05, + "d_pool_score": 6.974185018420703e-05, + "d_ape": 2.2844736073180494e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.996671676635742, + 9.997088432312012, + 9.978528022766113, + 10.020832061767578, + 9.99020767211914, + 9.993856430053711, + 10.014687538146973 + ], + "critical_prepare_samples_ms": [ + 3.5972590558230877, + 3.5885649267584085, + 3.5170429619029164, + 3.4920680336654186, + 3.4954349976032972, + 3.593871952034533, + 3.5318699665367603, + 3.646206925623119 + ], + "baseline_prepare_ms": 3.743164998013526 + }, + "zigzag": { + "step_ms": 11.19603157043457, + "prepare_ms": 22.693975479342043, + "one_use_ms": 33.890007049776614, + "prepare_min_ms": 22.371524944901466, + "prepare_max_ms": 136.65311911609024, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.0477249966524713e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.326994000792746e-05, + "d_pool_score": 6.248366916465234e-05, + "d_ape": 2.302149020581522e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.188480377197266, + 11.215295791625977, + 11.511136054992676, + 11.195136070251465, + 11.212479591369629, + 11.19603157043457, + 11.186783790588379 + ], + "critical_prepare_samples_ms": [ + 24.338410003110766, + 33.537326962687075, + 22.391692851670086, + 22.684970987029374, + 22.371524944901466, + 22.702979971654713, + 22.49394287355244, + 136.65311911609024 + ], + "baseline_prepare_ms": 24.085226003080606 + }, + "planner": { + "step_ms": 10.009087562561035, + "prepare_ms": 4.715179966297001, + "one_use_ms": 14.724267528858036, + "prepare_min_ms": 4.600052139721811, + "prepare_max_ms": 4.814712912775576, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.962048572951886e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.034841683489502e-05, + "d_pool_score": 6.365301246893881e-05, + "d_ape": 2.1817777204410688e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.988896369934082, + 10.01091194152832, + 9.999456405639648, + 10.009087562561035, + 10.284640312194824, + 10.013312339782715, + 9.988832473754883 + ], + "critical_prepare_samples_ms": [ + 4.814712912775576, + 4.696596995927393, + 4.743945901282132, + 4.629171919077635, + 4.722174955531955, + 4.600052139721811, + 4.7081849770620465, + 4.768294980749488 + ], + "baseline_prepare_ms": 4.955698561388999 + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "dbc1324660bfdc95476a1f0d82d936cf16edd8a9cc6990afa6a37e17401f5dc6", + "policies": { + "contiguous": { + "step_ms": 33.102272033691406, + "prepare_ms": 7.131611928343773, + "one_use_ms": 40.23388396203518, + "prepare_min_ms": 7.029787986539304, + "prepare_max_ms": 7.579102064482868, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0021210770683415432, + "dq": 0.0006697244454984808, + "dkv": 0.0006752772580638095, + "d_index_q": 0.00041775240877173306, + "d_weights": 0.0003214792609200598, + "d_pool_kv": 0.000576733611039875, + "d_pool_score": 0.0005744006365142611, + "d_ape": 6.907255374051929e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 33.08230209350586, + 33.068511962890625, + 33.102272033691406, + 33.116512298583984, + 33.155582427978516, + 33.54044723510742, + 33.07088088989258 + ], + "critical_prepare_samples_ms": [ + 7.579102064482868, + 7.029787986539304, + 7.084714015945792, + 7.242373889312148, + 7.1294138906523585, + 7.133809966035187, + 7.39094999153167, + 7.094503031112254 + ], + "baseline_prepare_ms": 7.641692471224815 + }, + "zigzag": { + "step_ms": 38.75347137451172, + "prepare_ms": 44.199544994626194, + "one_use_ms": 82.95301636913791, + "prepare_min_ms": 43.52790501434356, + "prepare_max_ms": 159.79488100856543, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0021340651638746743, + "dq": 0.0006454264679630054, + "dkv": 0.0005374193486070684, + "d_index_q": 0.000355993913482489, + "d_weights": 0.00032167200796056337, + "d_pool_kv": 0.0005637152047024604, + "d_pool_score": 0.0005797494053691072, + "d_ape": 6.916447167492779e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.73811340332031, + 38.75721740722656, + 38.72956848144531, + 38.75347137451172, + 38.723487854003906, + 38.79964828491211, + 38.764190673828125 + ], + "critical_prepare_samples_ms": [ + 45.98132590763271, + 43.77324297092855, + 43.52790501434356, + 159.79488100856543, + 44.65641709975898, + 44.314848026260734, + 43.64072694443166, + 44.084241962991655 + ], + "baseline_prepare_ms": 45.72551843011752 + }, + "planner": { + "step_ms": 33.08563232421875, + "prepare_ms": 9.258774924091995, + "one_use_ms": 42.344407248310745, + "prepare_min_ms": 9.213590994477272, + "prepare_max_ms": 9.48783487547189, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002089306735283265, + "dq": 0.0006319712339703342, + "dkv": 0.0005306332292867902, + "d_index_q": 0.0003543899517389669, + "d_weights": 0.00031634301834457875, + "d_pool_kv": 0.0005684620661380382, + "d_pool_score": 0.0005770038318201088, + "d_ape": 6.819908090487931e-06 + }, + "max_loss_abs_diff": 4.0745362639427185e-10, + "critical_step_samples_ms": [ + 33.08563232421875, + 33.09142303466797, + 33.06816101074219, + 33.11497497558594, + 33.05699157714844, + 33.14908981323242, + 33.06867218017578 + ], + "critical_prepare_samples_ms": [ + 9.332454996183515, + 9.246370987966657, + 9.213590994477272, + 9.24178201239556, + 9.244293090887368, + 9.430562960915267, + 9.271178860217333, + 9.48783487547189 + ], + "baseline_prepare_ms": 28.312876995187253 + } + } + }, + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "64a03500c3f1fe411b7cadd8009104ede8ca9b139cd0732acb13828bd86add34", + "policies": { + "contiguous": { + "step_ms": 32.327518463134766, + "prepare_ms": 7.331589062232524, + "one_use_ms": 39.65910752536729, + "prepare_min_ms": 7.055153022520244, + "prepare_max_ms": 9.122443967498839, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015056029919507346, + "dq": 0.00045665164417140727, + "dkv": 0.0004702181088295793, + "d_index_q": 0.0003533783738426617, + "d_weights": 0.000315067727554275, + "d_pool_kv": 0.0005386844431895517, + "d_pool_score": 0.0005634558855019262, + "d_ape": 3.905854431531449e-06 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 32.32707214355469, + 32.327518463134766, + 32.334590911865234, + 32.341793060302734, + 32.315391540527344, + 32.321632385253906, + 32.36006546020508 + ], + "critical_prepare_samples_ms": [ + 7.307856110855937, + 9.122443967498839, + 8.694347110576928, + 7.367579033598304, + 7.055153022520244, + 7.122335839085281, + 7.355322013609111, + 7.146980031393468 + ], + "baseline_prepare_ms": 7.578457531053573 + }, + "zigzag": { + "step_ms": 37.883487701416016, + "prepare_ms": 45.2110730111599, + "one_use_ms": 83.09456071257591, + "prepare_min_ms": 43.57520886696875, + "prepare_max_ms": 169.3743329960853, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016271139134036983, + "dq": 0.0004914893001159871, + "dkv": 0.0004848398943906415, + "d_index_q": 0.0003562240851567133, + "d_weights": 0.0003187078936891727, + "d_pool_kv": 0.0005284217741949873, + "d_pool_score": 0.000541357985562331, + "d_ape": 3.691615037628746e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 37.89449691772461, + 37.91689682006836, + 37.87980651855469, + 37.855712890625, + 37.8691520690918, + 37.9150390625, + 37.883487701416016 + ], + "critical_prepare_samples_ms": [ + 45.877177035436034, + 45.723085990175605, + 50.25763192679733, + 169.3743329960853, + 44.69906003214419, + 44.112405041232705, + 43.57520886696875, + 44.34900498017669 + ], + "baseline_prepare_ms": 45.84232851630077 + }, + "planner": { + "step_ms": 32.330718994140625, + "prepare_ms": 9.115683031268418, + "one_use_ms": 41.44640202540904, + "prepare_min_ms": 9.055349975824356, + "prepare_max_ms": 9.575794916599989, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015061092586627954, + "dq": 0.00045634085695053377, + "dkv": 0.00046791157809618843, + "d_index_q": 0.000356679693690852, + "d_weights": 0.0003092211654689727, + "d_pool_kv": 0.0005367827264826421, + "d_pool_score": 0.0005700652608558374, + "d_ape": 3.810351760870522e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.33180618286133, + 32.330718994140625, + 32.31001663208008, + 32.39107131958008, + 32.3276481628418, + 32.335872650146484, + 32.297183990478516 + ], + "critical_prepare_samples_ms": [ + 9.234660072252154, + 9.0619339607656, + 9.575794916599989, + 9.059371892362833, + 9.055349975824356, + 9.288421017117798, + 9.15169098880142, + 9.079675073735416 + ], + "baseline_prepare_ms": 12.12866249261424 + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "68b017a16ef4cf9fea1c3c9c0cc1a272786364ef4943afcf9a72f5eb3c9f90b7", + "policies": { + "contiguous": { + "step_ms": 115.74121856689453, + "prepare_ms": 1.0700724669732153, + "one_use_ms": 116.81129103386775, + "prepare_min_ms": 1.0450789704918861, + "prepare_max_ms": 1.2837169924750924, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.001482133304629509, + "dq": 0.0005028985179263687, + "dkv": 0.0010282109682620694, + "d_index_q": 0.0007703566859007169, + "d_weights": 0.0006286309166410099, + "d_pool_kv": 0.0013354595252499009, + "d_pool_score": 0.0013699629151741153, + "d_ape": 2.1869040480447828e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 114.95225524902344, + 115.74121856689453, + 116.17721557617188, + 114.34761810302734, + 116.25465393066406, + 114.593505859375, + 116.58633422851562 + ], + "critical_prepare_samples_ms": [ + 1.2837169924750924, + 1.1237040162086487, + 1.0450789704918861, + 1.0568589204922318, + 1.0688218753784895, + 1.0853169951587915, + 1.058198045939207, + 1.0713230585679412 + ], + "baseline_prepare_ms": 1.1161149595864117 + }, + "zigzag": { + "step_ms": 105.46825408935547, + "prepare_ms": 1.10343296546489, + "one_use_ms": 106.57168705482036, + "prepare_min_ms": 1.090729027055204, + "prepare_max_ms": 2.6004710234701633, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014829370691384724, + "dq": 0.000502969985804172, + "dkv": 0.0010306683280579473, + "d_index_q": 0.0007713429886426259, + "d_weights": 0.0006304765275187952, + "d_pool_kv": 0.0013371215867479295, + "d_pool_score": 0.00138344136125153, + "d_ape": 2.0698491606366296e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 105.06502532958984, + 105.46825408935547, + 104.91980743408203, + 105.63510131835938, + 105.70121765136719, + 105.19264221191406, + 105.93225860595703 + ], + "critical_prepare_samples_ms": [ + 2.6004710234701633, + 1.137568149715662, + 1.1008289875462651, + 1.1032898910343647, + 1.1035760398954153, + 1.1218799045309424, + 1.090729027055204, + 1.0995609918609262 + ], + "baseline_prepare_ms": 1.1357140028849244 + }, + "planner": { + "step_ms": 104.23197174072266, + "prepare_ms": 7.709605968557298, + "one_use_ms": 111.94157770927995, + "prepare_min_ms": 7.618593983352184, + "prepare_max_ms": 8.737536030821502, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014859596400049872, + "dq": 0.0005041720349634445, + "dkv": 0.001030506433322175, + "d_index_q": 0.0006257742363324554, + "d_weights": 0.0005596577742386472, + "d_pool_kv": 0.001329566057037352, + "d_pool_score": 0.00137685351760298, + "d_ape": 2.1145177390575036e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 104.40544128417969, + 104.23197174072266, + 104.6523208618164, + 104.536865234375, + 104.16582489013672, + 104.04156494140625, + 104.11017608642578 + ], + "critical_prepare_samples_ms": [ + 7.696714950725436, + 8.737536030821502, + 7.659266120754182, + 7.647061021998525, + 8.64289712626487, + 7.72249698638916, + 7.618593983352184, + 8.357836049981415 + ], + "baseline_prepare_ms": 7.791060954332352 + } + } + }, + "256k-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "6f1d4a0da49698ef15f82a432e981a02d9d084b70fe81e2b56d8f189b043cd6a", + "policies": { + "contiguous": { + "step_ms": 50.320064544677734, + "prepare_ms": 0.9923935285769403, + "one_use_ms": 51.312458073254675, + "prepare_min_ms": 0.9678329806774855, + "prepare_max_ms": 1.1316180462017655, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008982351679445784, + "dq": 0.0004175412751329343, + "dkv": 0.0002881395432601503, + "d_index_q": 0.0001082730122096085, + "d_weights": 9.702894207094345e-05, + "d_pool_kv": 0.0001972150021053803, + "d_pool_score": 0.00019493588963177602, + "d_ape": 6.910623983780962e-06 + }, + "max_loss_abs_diff": 3.4924596548080444e-10, + "critical_step_samples_ms": [ + 50.30031967163086, + 50.302433013916016, + 50.362239837646484, + 50.36908721923828, + 50.320064544677734, + 50.36316680908203, + 50.31760025024414 + ], + "critical_prepare_samples_ms": [ + 1.1316180462017655, + 1.0181260295212269, + 0.9678329806774855, + 0.9887899504974484, + 0.9702970273792744, + 0.9985698852688074, + 0.9720890084281564, + 0.9959971066564322 + ], + "baseline_prepare_ms": 1.0602579568512738 + }, + "zigzag": { + "step_ms": 51.75715255737305, + "prepare_ms": 3.72084544505924, + "one_use_ms": 55.47799800243229, + "prepare_min_ms": 3.68827604688704, + "prepare_max_ms": 4.462504060938954, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009079115543848496, + "dq": 0.0004210909515919965, + "dkv": 0.0002940172927688538, + "d_index_q": 0.00010857831208366282, + "d_weights": 9.552466137385803e-05, + "d_pool_kv": 0.00020868563840840786, + "d_pool_score": 0.0002198801712107116, + "d_ape": 7.540515901910232e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 51.67875289916992, + 51.771583557128906, + 51.7336311340332, + 51.81840133666992, + 51.714080810546875, + 51.75715255737305, + 51.81683349609375 + ], + "critical_prepare_samples_ms": [ + 4.01491392403841, + 3.7799880374222994, + 3.68827604688704, + 3.7137470208108425, + 3.7259499076753855, + 3.7157409824430943, + 4.462504060938954, + 3.702106885612011 + ], + "baseline_prepare_ms": 4.144389531575143 + }, + "planner": { + "step_ms": 50.36211013793945, + "prepare_ms": 1.2112134136259556, + "one_use_ms": 51.57332355156541, + "prepare_min_ms": 1.1628499487414956, + "prepare_max_ms": 1.3246370945125818, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008988738864725319, + "dq": 0.00041697524787721483, + "dkv": 0.0002882229310364065, + "d_index_q": 0.00010873610457156427, + "d_weights": 9.523323253939226e-05, + "d_pool_kv": 0.00021093823450393177, + "d_pool_score": 0.00021293712669284863, + "d_ape": 6.762282048261405e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 50.2674560546875, + 50.269630432128906, + 50.581024169921875, + 50.36211013793945, + 50.84463882446289, + 50.260257720947266, + 50.700191497802734 + ], + "critical_prepare_samples_ms": [ + 1.3246370945125818, + 1.218253979459405, + 1.2112829135730863, + 1.1899759992957115, + 1.211143913678825, + 1.1628499487414956, + 1.2666999828070402, + 1.1778250336647034 + ], + "baseline_prepare_ms": 1.275484450161457 + } + } + }, + "256k-near-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "921d5ba93c6562588668d8e4fe34833a7208e52c5c1a0b8b6c5b728539491a9d", + "policies": { + "contiguous": { + "step_ms": 50.34755325317383, + "prepare_ms": 0.9968500817194581, + "one_use_ms": 51.344403334893286, + "prepare_min_ms": 0.9588320972397923, + "prepare_max_ms": 1.0964140528813004, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009002720083858091, + "dq": 0.00041658039734501545, + "dkv": 0.00029032119524166356, + "d_index_q": 0.0001083702031927515, + "d_weights": 9.270341749291116e-05, + "d_pool_kv": 0.00021243719360570197, + "d_pool_score": 0.00021657144439094105, + "d_ape": 7.183122931513331e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.26278305053711, + 50.286529541015625, + 50.332576751708984, + 50.415008544921875, + 50.4021110534668, + 50.41408157348633, + 50.34755325317383 + ], + "critical_prepare_samples_ms": [ + 1.0964140528813004, + 1.005454920232296, + 0.968775013461709, + 1.0026779491454363, + 0.9609600529074669, + 0.9930110536515713, + 0.9588320972397923, + 1.000689109787345 + ], + "baseline_prepare_ms": 1.0578265646472573 + }, + "zigzag": { + "step_ms": 51.7072639465332, + "prepare_ms": 3.7724130088463426, + "one_use_ms": 55.479676955379546, + "prepare_min_ms": 3.713970072567463, + "prepare_max_ms": 4.540979978628457, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009040166709284812, + "dq": 0.0004216716611809067, + "dkv": 0.00028856434640319855, + "d_index_q": 0.00010881543305361698, + "d_weights": 9.496546039412519e-05, + "d_pool_kv": 0.0002074227586177013, + "d_pool_score": 0.00020241674597203904, + "d_ape": 6.808701999575195e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 51.7072639465332, + 51.767616271972656, + 51.7031364440918, + 51.71673583984375, + 51.699039459228516, + 51.76015853881836, + 51.689247131347656 + ], + "critical_prepare_samples_ms": [ + 4.038910963572562, + 3.892948036082089, + 3.727796138264239, + 3.766917041502893, + 3.743300912901759, + 3.713970072567463, + 4.540979978628457, + 3.777908976189792 + ], + "baseline_prepare_ms": 4.253186983987689 + }, + "planner": { + "step_ms": 50.30198287963867, + "prepare_ms": 1.2103734770789742, + "one_use_ms": 51.512356356717646, + "prepare_min_ms": 1.165413879789412, + "prepare_max_ms": 1.283300924114883, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009011720652930045, + "dq": 0.00041697590209048004, + "dkv": 0.0002890001975034279, + "d_index_q": 0.00010860096710106817, + "d_weights": 9.628555678061784e-05, + "d_pool_kv": 0.0002099978400113182, + "d_pool_score": 0.00021443311522890485, + "d_ape": 6.98920313885421e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.25263977050781, + 50.36908721923828, + 50.4403190612793, + 50.30198287963867, + 50.267486572265625, + 50.27724838256836, + 50.336639404296875 + ], + "critical_prepare_samples_ms": [ + 1.283300924114883, + 1.2185070663690567, + 1.224570907652378, + 1.1921209516003728, + 1.2027559569105506, + 1.165413879789412, + 1.217990997247398, + 1.1812089942395687 + ], + "baseline_prepare_ms": 1.2725520064122975 + } + } + }, + "256k-mixed-22": { + "documents": 22, + "tokens": 262144, + "source_record_sha256": "fe159c639833d743955457884a4611ca25e984548b1f3208042b4a6bb301638c", + "policies": { + "contiguous": { + "step_ms": 90.67167663574219, + "prepare_ms": 1.1164660099893808, + "one_use_ms": 91.78814264573157, + "prepare_min_ms": 1.0949659626930952, + "prepare_max_ms": 1.2457289267331362, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00143689993907933, + "dq": 0.0004899801690773162, + "dkv": 0.0007826892603974143, + "d_index_q": 0.000796723694054483, + "d_weights": 0.0006164320797316369, + "d_pool_kv": 0.0012264280539640705, + "d_pool_score": 0.0012747699641356362, + "d_ape": 1.3222093515720844e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 90.87088012695312, + 90.7339859008789, + 90.64134216308594, + 89.80009460449219, + 90.7357406616211, + 90.02569580078125, + 90.67167663574219 + ], + "critical_prepare_samples_ms": [ + 1.2457289267331362, + 1.1465109419077635, + 1.0949659626930952, + 1.1196030536666512, + 1.100623863749206, + 1.1375189060345292, + 1.0985910193994641, + 1.1133289663121104 + ], + "baseline_prepare_ms": 1.150823023635894 + }, + "zigzag": { + "step_ms": 88.22876739501953, + "prepare_ms": 2.2970764548517764, + "one_use_ms": 90.52584384987131, + "prepare_min_ms": 2.2732780780643225, + "prepare_max_ms": 15.867101028561592, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.00144284104523375, + "dq": 0.0004920673948700904, + "dkv": 0.0006368864218746345, + "d_index_q": 0.0005277277670036296, + "d_weights": 0.00044285628176593295, + "d_pool_kv": 0.0006697211734598175, + "d_pool_score": 0.0006819690109157976, + "d_ape": 1.273986864151461e-05 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 88.02159881591797, + 88.35001373291016, + 88.56687927246094, + 87.62576293945312, + 88.37532806396484, + 88.22876739501953, + 87.9927978515625 + ], + "critical_prepare_samples_ms": [ + 10.660310974344611, + 15.867101028561592, + 2.304245950654149, + 2.2795250406488776, + 2.2899069590494037, + 2.288761897943914, + 2.2732780780643225, + 2.6616440154612064 + ], + "baseline_prepare_ms": 2.339682076126337 + }, + "planner": { + "step_ms": 87.58985900878906, + "prepare_ms": 8.515741501469165, + "one_use_ms": 96.10560051025823, + "prepare_min_ms": 8.127353037707508, + "prepare_max_ms": 9.741527959704399, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0014447715952959578, + "dq": 0.0004925536881693634, + "dkv": 0.0007865823928776758, + "d_index_q": 0.0008311647195845252, + "d_weights": 0.0006640931260018509, + "d_pool_kv": 0.001227268093921778, + "d_pool_score": 0.0012784182647735045, + "d_ape": 1.3639331200697894e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 87.6021728515625, + 87.31673431396484, + 87.58985900878906, + 87.67081451416016, + 87.4035873413086, + 87.91382598876953, + 87.31298828125 + ], + "critical_prepare_samples_ms": [ + 8.778966031968594, + 8.252516970969737, + 9.164304006844759, + 8.212398970499635, + 9.741527959704399, + 8.127353037707508, + 9.139055968262255, + 8.186427876353264 + ], + "baseline_prepare_ms": 8.772762026637793 + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "c3e7815a901d92f31141e0604e18d7bc895fd5f085746cc903da1efe1d6ff7d1", + "policies": { + "contiguous": { + "step_ms": 18.964704513549805, + "prepare_ms": 7.2258434956893325, + "one_use_ms": 26.190548009239137, + "prepare_min_ms": 6.673496915027499, + "prepare_max_ms": 10.859394096769392, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.126639010558396e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.658018032939693e-05, + "d_pool_score": 4.361095036650771e-05, + "d_ape": 1.4700628800457994e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 18.96460723876953, + 18.953855514526367, + 18.956928253173828, + 18.976831436157227, + 18.967615127563477, + 18.964704513549805, + 19.011167526245117 + ], + "critical_prepare_samples_ms": [ + 10.859394096769392, + 7.232020958326757, + 7.296605035662651, + 6.673496915027499, + 7.219666033051908, + 7.159358123317361, + 7.325196056626737, + 7.035364978946745 + ], + "baseline_prepare_ms": 6.907838920596987 + }, + "zigzag": { + "step_ms": 22.546016693115234, + "prepare_ms": 45.124090975150466, + "one_use_ms": 67.6701076682657, + "prepare_min_ms": 43.87128900270909, + "prepare_max_ms": 163.06352394167334, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3271739597648486e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.455468819835076e-05, + "d_pool_score": 5.371778108792677e-05, + "d_ape": 1.5063328225066925e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 22.546239852905273, + 22.5980167388916, + 22.535808563232422, + 22.534208297729492, + 22.546144485473633, + 22.53615951538086, + 22.546016693115234 + ], + "critical_prepare_samples_ms": [ + 45.488978968933225, + 44.75626104976982, + 43.87128900270909, + 163.06352394167334, + 44.75920298136771, + 45.767895062454045, + 44.65588403400034, + 45.569678070023656 + ], + "baseline_prepare_ms": 46.86611460056156 + }, + "planner": { + "step_ms": 18.937055587768555, + "prepare_ms": 9.278230951167643, + "one_use_ms": 28.215286538936198, + "prepare_min_ms": 9.084169985726476, + "prepare_max_ms": 9.426652104593813, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.496402962841108e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.0054426196458054e-05, + "d_pool_score": 4.135736654968628e-05, + "d_ape": 1.2068528167060991e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 18.950944900512695, + 18.939456939697266, + 18.937055587768555, + 18.931711196899414, + 18.945024490356445, + 18.936767578125, + 18.926591873168945 + ], + "critical_prepare_samples_ms": [ + 9.400052018463612, + 9.135371074080467, + 9.426652104593813, + 9.084169985726476, + 9.326501982286572, + 9.229959920048714, + 9.1595669509843, + 9.419546113349497 + ], + "baseline_prepare_ms": 8.967155939899385 + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "7a6daf1992407b30c4ae176c4adb7995e06aa0300d429547621f5fe178c8a4b1", + "policies": { + "contiguous": { + "step_ms": 21.74198341369629, + "prepare_ms": 3.678025444969535, + "one_use_ms": 25.420008858665824, + "prepare_min_ms": 3.6021980922669172, + "prepare_max_ms": 3.9428939344361424, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.417560025712104e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 7.005921384441596e-05, + "d_pool_score": 5.8958733777925906e-05, + "d_ape": 1.631341031376204e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.741151809692383, + 21.74300765991211, + 21.733728408813477, + 21.758752822875977, + 21.73583984375, + 21.74198341369629, + 21.77382469177246 + ], + "critical_prepare_samples_ms": [ + 3.671723883599043, + 3.7506880471482873, + 3.645248943939805, + 3.6927630426362157, + 3.6021980922669172, + 3.684327006340027, + 3.6574859404936433, + 3.9428939344361424 + ], + "baseline_prepare_ms": 3.9414139464497566 + }, + "zigzag": { + "step_ms": 24.20911979675293, + "prepare_ms": 23.415951000060886, + "one_use_ms": 47.625070796813816, + "prepare_min_ms": 22.827126900665462, + "prepare_max_ms": 139.44511592853814, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.379937858947025e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.0763077851366874e-05, + "d_pool_score": 5.891486019054288e-05, + "d_ape": 1.6771080713138341e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 24.205888748168945, + 24.241024017333984, + 24.198047637939453, + 24.33344078063965, + 24.214048385620117, + 24.20911979675293, + 24.200191497802734 + ], + "critical_prepare_samples_ms": [ + 24.720999062992632, + 23.846879019401968, + 22.83283998258412, + 22.985022980719805, + 24.500835104845464, + 22.827126900665462, + 22.859422024339437, + 139.44511592853814 + ], + "baseline_prepare_ms": 24.67734646052122 + }, + "planner": { + "step_ms": 21.773439407348633, + "prepare_ms": 4.805412027053535, + "one_use_ms": 26.578851434402168, + "prepare_min_ms": 4.7372711123898625, + "prepare_max_ms": 4.933310090564191, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.865546995903407e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.477011134407135e-05, + "d_pool_score": 4.405915600329243e-05, + "d_ape": 1.4306253170553605e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.733535766601562, + 21.773439407348633, + 21.731647491455078, + 21.738143920898438, + 21.783647537231445, + 22.051231384277344, + 22.021024703979492 + ], + "critical_prepare_samples_ms": [ + 4.933310090564191, + 4.814392072148621, + 4.882890032604337, + 4.7437300672754645, + 4.796431981958449, + 4.772201064042747, + 4.873599042184651, + 4.7372711123898625 + ], + "baseline_prepare_ms": 5.023348960094154 + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "7446722df043d987c1f521ffc1939cab03422244a16d5f3d3765383e4fe0713d", + "policies": { + "contiguous": { + "step_ms": 83.70726776123047, + "prepare_ms": 8.869351528119296, + "one_use_ms": 92.57661928934976, + "prepare_min_ms": 8.712492999620736, + "prepare_max_ms": 10.470369947142899, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017232232291779785, + "dq": 0.0005228538978110929, + "dkv": 0.000691125497722852, + "d_index_q": 0.00036843245190847605, + "d_weights": 0.0003596773286925188, + "d_pool_kv": 0.0013194996004303847, + "d_pool_score": 0.0013134479113316487, + "d_ape": 8.920837294635517e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 84.19203186035156, + 83.22354888916016, + 83.92806243896484, + 83.70726776123047, + 84.4559326171875, + 83.31343841552734, + 83.37763214111328 + ], + "critical_prepare_samples_ms": [ + 10.470369947142899, + 9.009449044242501, + 8.902203058823943, + 8.816724992357194, + 8.712492999620736, + 8.713754126802087, + 9.297300945036113, + 8.836499997414649 + ], + "baseline_prepare_ms": 9.585648542270064 + }, + "zigzag": { + "step_ms": 95.86224365234375, + "prepare_ms": 95.58228251989931, + "one_use_ms": 191.44452617224306, + "prepare_min_ms": 46.32592911366373, + "prepare_max_ms": 165.51054792944342, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017361334114495666, + "dq": 0.0005265492561785378, + "dkv": 0.0004599309949688459, + "d_index_q": 0.0003699972474757121, + "d_weights": 0.00036300246633119064, + "d_pool_kv": 0.0006498637378581222, + "d_pool_score": 0.0006531249898699503, + "d_ape": 7.658629854388498e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 95.64492797851562, + 95.81046295166016, + 95.86739349365234, + 96.00326538085938, + 95.86224365234375, + 96.04275512695312, + 95.80735778808594 + ], + "critical_prepare_samples_ms": [ + 48.216517083346844, + 165.51054792944342, + 147.26835896726698, + 47.15038393624127, + 46.32592911366373, + 46.40162189025432, + 142.94804795645177, + 146.32742700632662 + ], + "baseline_prepare_ms": 92.28663396788761 + }, + "planner": { + "step_ms": 82.06880187988281, + "prepare_ms": 24.085616052616388, + "one_use_ms": 106.1544179324992, + "prepare_min_ms": 23.79455603659153, + "prepare_max_ms": 108.25915494933724, + "selected_policy": "selection-local-5x", + "max_relative_rms": { + "output": 0.001736570688522341, + "dq": 0.0005270035493433086, + "dkv": 0.0005767161784870217, + "d_index_q": 0.00039398350109750283, + "d_weights": 0.00036772447259371074, + "d_pool_kv": 0.0006438267350921343, + "d_pool_score": 0.0006413864917895826, + "d_ape": 8.028800212264691e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 82.06880187988281, + 81.95337677001953, + 81.74278259277344, + 82.35673522949219, + 81.7702407836914, + 82.2399673461914, + 82.25609588623047 + ], + "critical_prepare_samples_ms": [ + 23.79455603659153, + 24.032195913605392, + 24.03438393957913, + 23.953115101903677, + 24.136848165653646, + 24.561780970543623, + 108.25915494933724, + 24.656706023961306 + ], + "baseline_prepare_ms": 32.668465457390994 + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "dabbfa12c68f3a6cd8dbe7612e5f54ad194e004ecc568eeeeffa174212da0761", + "policies": { + "contiguous": { + "step_ms": 98.90694427490234, + "prepare_ms": 8.876458974555135, + "one_use_ms": 107.78340324945748, + "prepare_min_ms": 8.691001101396978, + "prepare_max_ms": 9.725504904054105, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011522479235681476, + "dq": 0.00042835242305097246, + "dkv": 0.0004943333278686983, + "d_index_q": 0.000657896894736068, + "d_weights": 0.0005964786262765446, + "d_pool_kv": 0.0006674154267850759, + "d_pool_score": 0.0006874466443201366, + "d_ape": 6.1667568045954095e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 98.90694427490234, + 98.74559783935547, + 100.46985626220703, + 99.39778900146484, + 98.54434967041016, + 98.55680084228516, + 99.67833709716797 + ], + "critical_prepare_samples_ms": [ + 9.50452999677509, + 8.7447389960289, + 8.844938012771308, + 8.691001101396978, + 8.871630998328328, + 8.881286950781941, + 9.725504904054105, + 9.013960952870548 + ], + "baseline_prepare_ms": 9.696168475784361 + }, + "zigzag": { + "step_ms": 115.310302734375, + "prepare_ms": 148.14059244235978, + "one_use_ms": 263.4508951767348, + "prepare_min_ms": 47.415707958862185, + "prepare_max_ms": 176.64630699437112, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0010866654170538991, + "dq": 0.0003696364920229251, + "dkv": 0.0003946430012396241, + "d_index_q": 0.0003810622253103829, + "d_weights": 0.0003747132595040899, + "d_pool_kv": 0.0006645926398509092, + "d_pool_score": 0.0006783532515969929, + "d_ape": 5.437880086557659e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 115.310302734375, + 115.21078491210938, + 115.49664306640625, + 115.5719985961914, + 115.37644958496094, + 115.2808609008789, + 115.2738265991211 + ], + "critical_prepare_samples_ms": [ + 140.08078002370894, + 156.66547801811248, + 176.64630699437112, + 47.415707958862185, + 48.025651019997895, + 144.5655939169228, + 159.11502100061625, + 151.71559096779674 + ], + "baseline_prepare_ms": 147.0409840112552 + }, + "planner": { + "step_ms": 92.09024047851562, + "prepare_ms": 23.416623007506132, + "one_use_ms": 115.50686348602176, + "prepare_min_ms": 22.514423937536776, + "prepare_max_ms": 127.23819306120276, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0010924404756515048, + "dq": 0.000392027321823514, + "dkv": 0.0004789729076346326, + "d_index_q": 0.000595231909155876, + "d_weights": 0.0005075082975035412, + "d_pool_kv": 0.0007611845879591085, + "d_pool_score": 0.0007714277449577657, + "d_ape": 6.021118465344542e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 91.97811126708984, + 92.4843521118164, + 91.98320007324219, + 92.22860717773438, + 91.90172576904297, + 92.09024047851562, + 92.3569564819336 + ], + "critical_prepare_samples_ms": [ + 22.948727943003178, + 109.54794602002949, + 22.841048892587423, + 23.339509032666683, + 22.514423937536776, + 24.088384001515806, + 127.23819306120276, + 23.49373698234558 + ], + "baseline_prepare_ms": 31.8146159988828 + } + } + }, + "1024k-single": { + "documents": 1, + "tokens": 1048576, + "source_record_sha256": "9119bf217085106d62d4a79a61271150ec5fd5efc62dc415a0070fc2ecfc741e", + "policies": { + "contiguous": { + "step_ms": 852.262451171875, + "prepare_ms": 2.238098415546119, + "one_use_ms": 854.5005495874211, + "prepare_min_ms": 2.167516970075667, + "prepare_max_ms": 2.5991599541157484, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011273030335427169, + "dq": 0.0003575866171757266, + "dkv": 0.00041255472186497554, + "d_index_q": 0.000522703724234038, + "d_weights": 0.000503204507333504, + "d_pool_kv": 0.0007366385730565614, + "d_pool_score": 0.000730156729966756, + "d_ape": 1.2261158117605297e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 855.7032470703125, + 852.262451171875, + 851.0695190429688, + 850.6251220703125, + 851.63916015625, + 852.8311157226562, + 854.872314453125 + ], + "critical_prepare_samples_ms": [ + 2.5991599541157484, + 2.248013042844832, + 2.268044976517558, + 2.2184940753504634, + 2.167516970075667, + 2.2399878362193704, + 2.188600949011743, + 2.236208994872868 + ], + "baseline_prepare_ms": 2.235381514765322 + }, + "zigzag": { + "step_ms": 637.3577880859375, + "prepare_ms": 2.4432449718005955, + "one_use_ms": 639.8010330577381, + "prepare_min_ms": 2.4204020155593753, + "prepare_max_ms": 5.492284893989563, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011418886057533793, + "dq": 0.00036775419972180114, + "dkv": 0.0004790727674961075, + "d_index_q": 0.0004997064155363637, + "d_weights": 0.0004932188618751324, + "d_pool_kv": 0.0007411197792606418, + "d_pool_score": 0.0007282443063629072, + "d_ape": 1.238206578062022e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 637.708740234375, + 642.3746948242188, + 637.3577880859375, + 637.919921875, + 635.5283813476562, + 637.1450805664062, + 633.81103515625 + ], + "critical_prepare_samples_ms": [ + 5.492284893989563, + 2.5590929435566068, + 2.472839900292456, + 2.4355159839615226, + 2.4204020155593753, + 2.4509739596396685, + 2.4335948983207345, + 2.4213059805333614 + ], + "baseline_prepare_ms": 2.2861745092086494 + }, + "planner": { + "step_ms": 637.7809448242188, + "prepare_ms": 9.511192969512194, + "one_use_ms": 647.292137793731, + "prepare_min_ms": 9.069906896911561, + "prepare_max_ms": 10.374767007306218, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.001161625101370349, + "dq": 0.0003849271614974025, + "dkv": 0.0005282062594106454, + "d_index_q": 0.0005244622397098521, + "d_weights": 0.0005299670263664599, + "d_pool_kv": 0.0007377781316978346, + "d_pool_score": 0.0007313667519857782, + "d_ape": 1.2469585835438161e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 638.3487548828125, + 637.7809448242188, + 636.5733642578125, + 637.4932250976562, + 637.3499755859375, + 639.65185546875, + 640.6060791015625 + ], + "critical_prepare_samples_ms": [ + 10.374767007306218, + 9.756346000358462, + 9.435725980438292, + 9.219342959113419, + 9.78031090926379, + 9.120933944359422, + 9.586659958586097, + 9.069906896911561 + ], + "baseline_prepare_ms": 9.320752928033471 + } + } + }, + "1024k-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "477bd6d1a7cf100b540a3a743b44248720e6f9ab40bee8715349b4361dad4122", + "policies": { + "contiguous": { + "step_ms": 239.2353973388672, + "prepare_ms": 1.450755458790809, + "one_use_ms": 240.686152797658, + "prepare_min_ms": 1.4129939954727888, + "prepare_max_ms": 1.5769170131534338, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015949253947342458, + "dq": 0.0005970787439556988, + "dkv": 0.0004837391776387549, + "d_index_q": 0.00028350067830152883, + "d_weights": 0.00020850715397855953, + "d_pool_kv": 0.0002889538103473619, + "d_pool_score": 0.0002844647514544151, + "d_ape": 5.9795103431945e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 239.2353973388672, + 238.79302978515625, + 238.77218627929688, + 239.280029296875, + 238.88275146484375, + 239.3966064453125, + 239.5908203125 + ], + "critical_prepare_samples_ms": [ + 1.5769170131534338, + 1.4974380610510707, + 1.4129939954727888, + 1.4513909118250012, + 1.456466969102621, + 1.4202899765223265, + 1.4501200057566166, + 1.43467599991709 + ], + "baseline_prepare_ms": 1.490010938141495 + }, + "zigzag": { + "step_ms": 244.40403747558594, + "prepare_ms": 4.328943963628262, + "one_use_ms": 248.7329814392142, + "prepare_min_ms": 4.188165999948978, + "prepare_max_ms": 6.3333159778267145, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016162390583326469, + "dq": 0.0006094600566733209, + "dkv": 0.0005030974253165897, + "d_index_q": 0.0002799287027435939, + "d_weights": 0.00021150429278782401, + "d_pool_kv": 0.0002866744762082726, + "d_pool_score": 0.000295534879448145, + "d_ape": 6.645333132807697e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 244.40403747558594, + 243.9402313232422, + 244.39878845214844, + 244.6631622314453, + 245.0597686767578, + 245.14906311035156, + 244.3597412109375 + ], + "critical_prepare_samples_ms": [ + 6.3333159778267145, + 4.481954965740442, + 4.322783905081451, + 4.29266900755465, + 4.335104022175074, + 4.287454066798091, + 5.151383113116026, + 4.188165999948978 + ], + "baseline_prepare_ms": 4.6627100091427565 + }, + "planner": { + "step_ms": 239.4423370361328, + "prepare_ms": 1.6502354992553592, + "one_use_ms": 241.09257253538817, + "prepare_min_ms": 1.596967107616365, + "prepare_max_ms": 1.7511360347270966, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015923902784638652, + "dq": 0.0005966125090240904, + "dkv": 0.00048673665217852476, + "d_index_q": 0.0002642586924764183, + "d_weights": 0.0002012227499861691, + "d_pool_kv": 0.00029146264458877296, + "d_pool_score": 0.000316266196867594, + "d_ape": 6.500418666259421e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 239.1868438720703, + 238.84658813476562, + 239.47996520996094, + 239.3558349609375, + 239.5425567626953, + 240.12112426757812, + 239.4423370361328 + ], + "critical_prepare_samples_ms": [ + 1.7511360347270966, + 1.6750609502196312, + 1.6717149410396814, + 1.6503350343555212, + 1.6501359641551971, + 1.6250868793576956, + 1.6428639646619558, + 1.596967107616365 + ], + "baseline_prepare_ms": 1.7240439774468541 + } + } + }, + "1024k-near-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "b51c859098119635daa6f893eb71b0c37c1fef528ca4601e030756c553e3543b", + "policies": { + "contiguous": { + "step_ms": 239.17958068847656, + "prepare_ms": 1.5109099331311882, + "one_use_ms": 240.69049062160775, + "prepare_min_ms": 1.4560830313712358, + "prepare_max_ms": 1.6104099340736866, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0016016710758490706, + "dq": 0.0005993639977494133, + "dkv": 0.0005071280037082532, + "d_index_q": 0.00024417279990879927, + "d_weights": 0.00019413726329423075, + "d_pool_kv": 0.00028954282003200476, + "d_pool_score": 0.0002955359390565756, + "d_ape": 6.6829402506119585e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 239.7038116455078, + 239.43247985839844, + 238.82850646972656, + 239.17958068847656, + 239.11888122558594, + 239.16099548339844, + 239.1847686767578 + ], + "critical_prepare_samples_ms": [ + 1.6104099340736866, + 1.5317690558731556, + 1.4775969320908189, + 1.5244908863678575, + 1.4560830313712358, + 1.5005789464339614, + 1.4756300952285528, + 1.521240919828415 + ], + "baseline_prepare_ms": 1.514466479420662 + }, + "zigzag": { + "step_ms": 244.55760192871094, + "prepare_ms": 4.382276034448296, + "one_use_ms": 248.93987796315923, + "prepare_min_ms": 4.281172063201666, + "prepare_max_ms": 6.378103978931904, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016200773797571998, + "dq": 0.0006071238591595291, + "dkv": 0.0005038878658893274, + "d_index_q": 0.000277629521087908, + "d_weights": 0.00021663570090953182, + "d_pool_kv": 0.0002824822719976606, + "d_pool_score": 0.00027970202182346534, + "d_ape": 5.951399279862816e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 244.45948791503906, + 244.55760192871094, + 245.24301147460938, + 244.60943603515625, + 244.06588745117188, + 244.22259521484375, + 244.9438018798828 + ], + "critical_prepare_samples_ms": [ + 6.378103978931904, + 4.513074993155897, + 4.392945091240108, + 4.371606977656484, + 5.423646070994437, + 4.291923018172383, + 4.289834876544774, + 4.281172063201666 + ], + "baseline_prepare_ms": 4.788588033989072 + }, + "planner": { + "step_ms": 239.22537231445312, + "prepare_ms": 1.7141494899988174, + "one_use_ms": 240.93952180445194, + "prepare_min_ms": 1.6789290821179748, + "prepare_max_ms": 1.81203696411103, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015931991971287927, + "dq": 0.000594438919428117, + "dkv": 0.00048420164599018655, + "d_index_q": 0.0002598202280658747, + "d_weights": 0.00019847531748186766, + "d_pool_kv": 0.0002823099923752181, + "d_pool_score": 0.0002799009540318669, + "d_ape": 6.337974334637259e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 238.89100646972656, + 239.2686767578125, + 238.71510314941406, + 239.22537231445312, + 238.6892547607422, + 239.8118133544922, + 239.24957275390625 + ], + "critical_prepare_samples_ms": [ + 1.81203696411103, + 1.7418409697711468, + 1.7184868920594454, + 1.7098120879381895, + 1.7221469897776842, + 1.6911160200834274, + 1.7061970429494977, + 1.6789290821179748 + ], + "baseline_prepare_ms": 1.75143702654168 + } + } + }, + "1024k-mixed-22": { + "documents": 22, + "tokens": 1048576, + "source_record_sha256": "9a79bc56f979185b724f8b5b55bd4292ce17d924f7ad524db1046fb348026159", + "policies": { + "contiguous": { + "step_ms": 578.4398193359375, + "prepare_ms": 2.1686649997718632, + "one_use_ms": 580.6084843357094, + "prepare_min_ms": 2.13688297662884, + "prepare_max_ms": 2.4644359946250916, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011041011872111484, + "dq": 0.00035084277748766797, + "dkv": 0.00048077283639952237, + "d_index_q": 0.0004962393008161014, + "d_weights": 0.0004618130449118246, + "d_pool_kv": 0.0006457713057874247, + "d_pool_score": 0.0006349061560406983, + "d_ape": 1.0350294386060346e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 580.9935913085938, + 577.6489868164062, + 578.0813598632812, + 580.2940673828125, + 578.3426513671875, + 578.4398193359375, + 579.4547729492188 + ], + "critical_prepare_samples_ms": [ + 2.4644359946250916, + 2.202583011239767, + 2.1521480521187186, + 2.176910056732595, + 2.152706030756235, + 2.13688297662884, + 2.1651480346918106, + 2.172181964851916 + ], + "baseline_prepare_ms": 2.234563056845218 + }, + "zigzag": { + "step_ms": 492.48590087890625, + "prepare_ms": 3.1932704732753336, + "one_use_ms": 495.6791713521816, + "prepare_min_ms": 3.0714080203324556, + "prepare_max_ms": 4.142447025515139, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011285694800997107, + "dq": 0.0003719019185829731, + "dkv": 0.0005662235654619531, + "d_index_q": 0.0005086592423664251, + "d_weights": 0.0004577104856603165, + "d_pool_kv": 0.0007116715193057737, + "d_pool_score": 0.0007034021456370472, + "d_ape": 1.0718926168143519e-05 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 492.2532653808594, + 493.8275146484375, + 492.48590087890625, + 494.0948181152344, + 491.0968933105469, + 491.9676513671875, + 496.3102722167969 + ], + "critical_prepare_samples_ms": [ + 4.142447025515139, + 3.217385965399444, + 3.1294310465455055, + 3.0714080203324556, + 3.084978088736534, + 3.169154981151223, + 3.448216011747718, + 3.476505051366985 + ], + "baseline_prepare_ms": 3.2312944531440735 + }, + "planner": { + "step_ms": 490.8729248046875, + "prepare_ms": 10.11992251733318, + "one_use_ms": 500.9928473220207, + "prepare_min_ms": 9.30530100595206, + "prepare_max_ms": 12.625832110643387, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0011161842289795045, + "dq": 0.0003611815947831294, + "dkv": 0.0005663782910108529, + "d_index_q": 0.0005064407713656789, + "d_weights": 0.0004754969744536212, + "d_pool_kv": 0.0007073881102403647, + "d_pool_score": 0.0006989918887090544, + "d_ape": 1.049051513112601e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 490.52728271484375, + 488.947998046875, + 494.4607849121094, + 491.1177062988281, + 493.6095886230469, + 490.8729248046875, + 490.8112487792969 + ], + "critical_prepare_samples_ms": [ + 10.942328022792935, + 9.30530100595206, + 10.56371396407485, + 10.191290057264268, + 9.348670137114823, + 12.625832110643387, + 10.048554977402091, + 9.677355061285198 + ], + "baseline_prepare_ms": 9.799090563319623 + } + } + }, + "1024k-uniform-1024": { + "documents": 1024, + "tokens": 1048576, + "source_record_sha256": "b444906588101601ed60331d4bddb587f91cd677f6cfb7b815248b53a1451470", + "policies": { + "contiguous": { + "step_ms": 106.93244934082031, + "prepare_ms": 7.248607522342354, + "one_use_ms": 114.18105686316267, + "prepare_min_ms": 6.926600937731564, + "prepare_max_ms": 7.728449068963528, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3262557940735125e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.029982612822922e-05, + "d_pool_score": 5.971250741145832e-05, + "d_ape": 8.469250666388928e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.99427032470703, + 106.90831756591797, + 107.24447631835938, + 106.77391815185547, + 107.3912353515625, + 106.73372650146484, + 106.93244934082031 + ], + "critical_prepare_samples_ms": [ + 7.304837927222252, + 7.447938900440931, + 7.5955630745738745, + 7.06713495310396, + 7.192377117462456, + 7.016728050075471, + 6.926600937731564, + 7.728449068963528 + ], + "baseline_prepare_ms": 7.491494528949261 + }, + "zigzag": { + "step_ms": 120.94866943359375, + "prepare_ms": 47.448766068555415, + "one_use_ms": 168.39743550214916, + "prepare_min_ms": 44.58731797058135, + "prepare_max_ms": 171.35898303240538, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.375492679918749e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.900628434802078e-05, + "d_pool_score": 5.1757344668054784e-05, + "d_ape": 7.775777513380578e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 120.98294067382812, + 120.94866943359375, + 121.13782501220703, + 120.91907501220703, + 121.1909408569336, + 120.86470031738281, + 120.89344024658203 + ], + "critical_prepare_samples_ms": [ + 50.39555404800922, + 48.49247704260051, + 48.66090486757457, + 171.35898303240538, + 44.880184112116694, + 45.30161793809384, + 44.58731797058135, + 46.40505509451032 + ], + "baseline_prepare_ms": 48.3965675230138 + }, + "planner": { + "step_ms": 106.88835144042969, + "prepare_ms": 9.285612439271063, + "one_use_ms": 116.17396387970075, + "prepare_min_ms": 9.05843498185277, + "prepare_max_ms": 10.61727199703455, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.44561194645775e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.3430278698572834e-05, + "d_pool_score": 5.700039307126877e-05, + "d_ape": 8.3915886081418e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.72726440429688, + 106.88835144042969, + 106.67664337158203, + 107.06781005859375, + 106.97030639648438, + 106.76934051513672, + 106.93551635742188 + ], + "critical_prepare_samples_ms": [ + 9.35503700748086, + 9.251538896933198, + 10.61727199703455, + 9.05843498185277, + 9.093354921787977, + 9.319685981608927, + 9.11523203831166, + 10.08200692012906 + ], + "baseline_prepare_ms": 9.633392037358135 + } + } + }, + "1024k-near-uniform-512": { + "documents": 512, + "tokens": 1048576, + "source_record_sha256": "5ff06f93b96c53292e5c777c40e19bc4f75b59888b574f4084c6304e27d06705", + "policies": { + "contiguous": { + "step_ms": 153.3333740234375, + "prepare_ms": 4.90699551301077, + "one_use_ms": 158.24036953644827, + "prepare_min_ms": 4.817495006136596, + "prepare_max_ms": 5.127965938299894, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.6945818513152077e-05, + "dq": 8.852569918672518e-06, + "dkv": 4.778136519064779e-05, + "d_index_q": 2.61152275055734e-06, + "d_weights": 2.609757780546301e-06, + "d_pool_kv": 6.295818459824281e-05, + "d_pool_score": 6.4597007352727e-05, + "d_ape": 8.241673159031186e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.1051788330078, + 153.3333740234375, + 153.77328491210938, + 153.28533935546875, + 153.67117309570312, + 153.2650909423828, + 153.51904296875 + ], + "critical_prepare_samples_ms": [ + 4.958503064699471, + 4.855487961322069, + 5.127965938299894, + 4.836354055441916, + 4.9949249951168895, + 4.826268996112049, + 4.817495006136596, + 5.11400296818465 + ], + "baseline_prepare_ms": 4.550960555206984 + }, + "zigzag": { + "step_ms": 162.93936157226562, + "prepare_ms": 24.463195994030684, + "one_use_ms": 187.4025575662963, + "prepare_min_ms": 24.322091951034963, + "prepare_max_ms": 27.672135969623923, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 1.6907351105767428e-05, + "dq": 8.810149949287884e-06, + "dkv": 4.1814702326799884e-05, + "d_index_q": 2.701785992294692e-06, + "d_weights": 2.6607871764855918e-06, + "d_pool_kv": 5.689533114065568e-05, + "d_pool_score": 5.6776242766562597e-05, + "d_ape": 7.905820789837788e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 162.92745971679688, + 163.33734130859375, + 162.88768005371094, + 162.93936157226562, + 162.84515380859375, + 163.23513793945312, + 162.99880981445312 + ], + "critical_prepare_samples_ms": [ + 27.672135969623923, + 25.097765144892037, + 24.366947007365525, + 24.44107295013964, + 24.474958889186382, + 24.664230993948877, + 24.322091951034963, + 24.451433098874986 + ], + "baseline_prepare_ms": 25.861825968604535 + }, + "planner": { + "step_ms": 153.63226318359375, + "prepare_ms": 6.0009079170413315, + "one_use_ms": 159.63317110063508, + "prepare_min_ms": 5.884354002773762, + "prepare_max_ms": 6.281962036155164, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.6380870010580877e-05, + "dq": 8.576173759861243e-06, + "dkv": 4.329798528645843e-05, + "d_index_q": 2.859165424391761e-06, + "d_weights": 2.390300293499087e-06, + "d_pool_kv": 6.976847445393797e-05, + "d_pool_score": 8.047768811080957e-05, + "d_ape": 8.925679845335159e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.46054077148438, + 154.20037841796875, + 153.07359313964844, + 153.19833374023438, + 153.63571166992188, + 154.10687255859375, + 153.63226318359375 + ], + "critical_prepare_samples_ms": [ + 6.0230528470128775, + 5.923283053562045, + 6.161181954666972, + 5.884354002773762, + 6.277190987020731, + 5.908905062824488, + 5.978762987069786, + 6.281962036155164 + ], + "baseline_prepare_ms": 5.739656975492835 + } + } + }, + "1024k-mixed-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "4c0764d76c3bfca709bc5cdc64a79988a30a203391319ceab498e84528d43186", + "policies": { + "contiguous": { + "step_ms": 567.9577026367188, + "prepare_ms": 9.956162073649466, + "one_use_ms": 577.9138647103682, + "prepare_min_ms": 9.866954875178635, + "prepare_max_ms": 11.781121022067964, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008054851232509614, + "dq": 0.00030349153325609713, + "dkv": 0.0003996891751172817, + "d_index_q": 0.0006246139925808879, + "d_weights": 0.0005733311451828375, + "d_pool_kv": 0.0006205091711447106, + "d_pool_score": 0.0006195543072615393, + "d_ape": 8.589075672373344e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 565.80810546875, + 568.4535522460938, + 567.845947265625, + 569.2208862304688, + 563.9498901367188, + 567.9577026367188, + 568.2915649414062 + ], + "critical_prepare_samples_ms": [ + 11.781121022067964, + 9.866954875178635, + 9.920966112986207, + 10.023329989053309, + 9.899892029352486, + 9.991358034312725, + 10.618924046866596, + 9.895945899188519 + ], + "baseline_prepare_ms": 10.622373956721276 + }, + "zigzag": { + "step_ms": 550.558837890625, + "prepare_ms": 48.51576604414731, + "one_use_ms": 599.0746039347723, + "prepare_min_ms": 46.67189810425043, + "prepare_max_ms": 174.14286790881306, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0008195260459777163, + "dq": 0.000301416184948774, + "dkv": 0.0005447644196372983, + "d_index_q": 0.0006037227037525915, + "d_weights": 0.0005483621762803569, + "d_pool_kv": 0.0007216194622467821, + "d_pool_score": 0.0007074085651829549, + "d_ape": 8.845595009769592e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 547.7075805664062, + 551.5230712890625, + 550.558837890625, + 550.6272583007812, + 551.8482666015625, + 549.5032348632812, + 549.90185546875 + ], + "critical_prepare_samples_ms": [ + 49.92610192857683, + 47.86653909832239, + 46.67189810425043, + 143.82607594598085, + 174.14286790881306, + 48.84456901345402, + 47.94406390283257, + 48.186963074840605 + ], + "baseline_prepare_ms": 48.262091528158635 + }, + "planner": { + "step_ms": 494.7050476074219, + "prepare_ms": 26.39079309301451, + "one_use_ms": 521.0958407004364, + "prepare_min_ms": 25.676422053948045, + "prepare_max_ms": 27.048783958889544, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0008365326911981586, + "dq": 0.000304561997298154, + "dkv": 0.0005395322403243995, + "d_index_q": 0.0006160848721497884, + "d_weights": 0.0005678273845894476, + "d_pool_kv": 0.000678002818444316, + "d_pool_score": 0.0006561034875025234, + "d_ape": 8.545834009840338e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 495.04931640625, + 495.3780822753906, + 492.96002197265625, + 494.4125671386719, + 495.3255615234375, + 493.7383117675781, + 494.7050476074219 + ], + "critical_prepare_samples_ms": [ + 26.55792306177318, + 27.048783958889544, + 26.075869100168347, + 26.223148917779326, + 26.27995307557285, + 26.50163311045617, + 25.676422053948045, + 26.891826069913805 + ], + "baseline_prepare_ms": 34.91155954543501 + } + } + }, + "1024k-long-tail-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "d3ad6415c7f6691fe0b88f453159fe81ee75cd82494a33142fdec3b521028dc9", + "policies": { + "contiguous": { + "step_ms": 800.95556640625, + "prepare_ms": 12.669915508013219, + "one_use_ms": 813.6254819142632, + "prepare_min_ms": 12.12127401959151, + "prepare_max_ms": 13.159677851945162, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009079882713568034, + "dq": 0.0002924982024565224, + "dkv": 0.0004493330361466962, + "d_index_q": 0.0005156894968942304, + "d_weights": 0.0005167765162935939, + "d_pool_kv": 0.0007712278582484425, + "d_pool_score": 0.0007494124280620555, + "d_ape": 7.171042317242159e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 800.335693359375, + 802.2348022460938, + 799.2125244140625, + 799.6749877929688, + 800.95556640625, + 801.217041015625, + 801.6029663085938 + ], + "critical_prepare_samples_ms": [ + 13.159677851945162, + 12.533110100775957, + 12.759627075865865, + 12.12127401959151, + 12.722588959150016, + 12.617242056876421, + 12.80664000660181, + 12.566132005304098 + ], + "baseline_prepare_ms": 14.17607138864696 + }, + "zigzag": { + "step_ms": 715.7775268554688, + "prepare_ms": 101.23607551213354, + "one_use_ms": 817.0136023676023, + "prepare_min_ms": 50.27641903143376, + "prepare_max_ms": 169.727465021424, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009004337940703815, + "dq": 0.0002908384586939322, + "dkv": 0.0008283107689368773, + "d_index_q": 0.0006025743415942871, + "d_weights": 0.0005952597316348166, + "d_pool_kv": 0.0009031995573276561, + "d_pool_score": 0.0008609984776183619, + "d_ape": 7.574199067023971e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 712.8131103515625, + 711.6310424804688, + 715.7775268554688, + 712.3639526367188, + 717.9108276367188, + 716.431396484375, + 716.0610961914062 + ], + "critical_prepare_samples_ms": [ + 169.727465021424, + 50.38151808548719, + 150.52157104946673, + 159.4792689429596, + 163.56680891476572, + 51.95057997480035, + 50.27641903143376, + 50.714182085357606 + ], + "baseline_prepare_ms": 151.8010766012594 + }, + "planner": { + "step_ms": 591.8178100585938, + "prepare_ms": 29.16059346171096, + "one_use_ms": 620.9784035203047, + "prepare_min_ms": 28.44624104909599, + "prepare_max_ms": 150.10115189943463, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0009046024533399656, + "dq": 0.00028358853532615296, + "dkv": 0.000828257132860518, + "d_index_q": 0.0007049305803991532, + "d_weights": 0.000616040566927685, + "d_pool_kv": 0.0009008018655028041, + "d_pool_score": 0.0008551253342927941, + "d_ape": 7.554278135195296e-06 + }, + "max_loss_abs_diff": 8.731149137020111e-11, + "critical_step_samples_ms": [ + 591.6988525390625, + 589.947509765625, + 591.8178100585938, + 596.8615112304688, + 592.572021484375, + 591.2091674804688, + 592.2348022460938 + ], + "critical_prepare_samples_ms": [ + 28.44624104909599, + 29.306308017112315, + 29.014878906309605, + 28.898976044729352, + 29.324857983738184, + 150.10115189943463, + 28.808553935959935, + 124.28496405482292 + ], + "baseline_prepare_ms": 38.67735603125766 + } + } + } + }, + "totals_by_context": { + "128": { + "contiguous": [ + 207.8772144317627, + 235.76852785609663 + ], + "zigzag": [ + 221.9881248474121, + 388.67559937527403 + ], + "planner": [ + 206.02739143371582, + 255.17840154655278 + ] + }, + "256": { + "contiguous": [ + 530.4014129638672, + 563.2268744944595 + ], + "zigzag": [ + 555.0891208648682, + 878.2458056765608 + ], + "planner": [ + 507.35546112060547, + 587.5882775196806 + ] + }, + "1024": { + "contiguous": [ + 3538.296340942383, + 3580.4464503666386 + ], + "zigzag": [ + 3169.0297241210938, + 3405.041263183113 + ], + "planner": [ + 2954.3650512695312, + 3048.1984586566687 + ] + } + }, + "max_relative_rms": { + "output": 0.0021340651638746743, + "dq": 0.0006697244454984808, + "dkv": 0.0010306683280579473, + "d_index_q": 0.0009173576570437036, + "d_weights": 0.0006640931260018509, + "d_pool_kv": 0.0013371215867479295, + "d_pool_score": 0.00138344136125153, + "d_ape": 2.5403258387015062e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-09, + "worst_steady_gap": [ + 0.3393008303370637, + "128k-mixed-22" + ], + "worst_one_use_gap": [ + 19.39923530952372, + "128k-mixed-22" + ], + "host_ab": { + "baseline_commit": "ea71df3b3bfad69141d1ee3ddd884ac9ccdb27ce", + "scope": "Host-only CsaCPSelectionPlan constructor on rank 7 of CP8; no GPU/workspace preparation", + "repeats": 20, + "cases": { + "128k-single-planner": { + "samples_ms": { + "before": [ + 6.241483031772077, + 6.129086017608643, + 6.2108299462124705, + 6.163439014926553, + 6.159020005725324, + 6.204652949236333, + 6.147739011794329, + 6.155765964649618, + 6.279730005189776, + 6.224854965694249, + 6.132257985882461, + 6.104929954744875, + 6.146484054625034, + 6.200143019668758, + 6.191351916640997, + 6.217735004611313, + 6.158133968710899, + 6.157501949928701, + 6.267914897762239, + 6.197582930326462 + ], + "after": [ + 6.05339405592531, + 6.09124603215605, + 6.0327439568936825, + 6.391837960109115, + 6.117061013355851, + 6.04431692045182, + 6.072498974390328, + 6.031388998962939, + 6.078919977881014, + 6.038562045432627, + 6.127536064013839, + 6.046790978871286, + 6.069870083592832, + 6.025816081091762, + 6.028156029060483, + 6.082112086005509, + 6.01962604559958, + 6.050291936844587, + 6.105632055550814, + 5.963249015621841 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.177395465783775, + "after": 6.0518429963849485 + }, + "metadata_exact": true + }, + "128k-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.40457199793308973, + 0.3915169509127736, + 0.3725989954546094, + 0.38956699427217245, + 0.37341401912271976, + 0.3859599819406867, + 0.3727299626916647, + 0.3891169326379895, + 0.3697650972753763, + 0.3940380411222577, + 0.37929299287498, + 0.390987959690392, + 0.37147896364331245, + 0.3826599568128586, + 0.37088198587298393, + 0.3915060078725219, + 0.372899929061532, + 0.3812409704551101, + 0.368123990483582, + 0.3834000090137124 + ], + "after": [ + 0.36852096673101187, + 0.3598059993237257, + 0.367147964425385, + 0.34407293424010277, + 0.3628099802881479, + 0.3502810141071677, + 0.3541019978001714, + 0.3426559269428253, + 0.358566059730947, + 0.3564689541235566, + 0.3626210382208228, + 0.34407898783683777, + 0.3595729358494282, + 0.35286403726786375, + 0.36064302548766136, + 0.343213090673089, + 0.35762006882578135, + 0.34747098106890917, + 0.3642140654847026, + 0.3391670761629939 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.3819504636339843, + "after": 0.357044511474669 + }, + "metadata_exact": true + }, + "128k-near-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.4026839742437005, + 0.39121892768889666, + 0.376117997802794, + 0.38380094338208437, + 0.3803960280492902, + 0.3904419718310237, + 0.37196208722889423, + 0.3914200933650136, + 0.3753060009330511, + 0.38710690569132566, + 0.3820260753855109, + 0.3933230182155967, + 0.3741809632629156, + 0.3947960212826729, + 0.3754019271582365, + 0.39576191920787096, + 0.3724439302459359, + 0.38169207982718945, + 0.37471100222319365, + 0.39148691575974226 + ], + "after": [ + 0.36272103898227215, + 0.3440839936956763, + 0.35585102159529924, + 0.3516109427437186, + 0.3656300250440836, + 0.34017907455563545, + 0.3507171059027314, + 0.35162700805813074, + 0.35890506114810705, + 0.3831470385193825, + 0.35917398054152727, + 0.34544302616268396, + 0.35676301922649145, + 0.3443220630288124, + 0.35430595744401217, + 0.34445198252797127, + 0.35806000232696533, + 0.34743198193609715, + 0.3568349638953805, + 0.33541000448167324 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.38291350938379765, + "after": 0.35296648275107145 + }, + "metadata_exact": true + }, + "128k-mixed-22-planner": { + "samples_ms": { + "before": [ + 6.297884043306112, + 6.32572709582746, + 6.304105976596475, + 6.388368899933994, + 6.2440200708806515, + 6.35086209513247, + 6.266923970542848, + 6.328370072878897, + 6.334865931421518, + 6.350334966555238, + 6.248642923310399, + 6.344994064420462, + 6.300061009824276, + 6.30893197376281, + 6.245687021873891, + 6.297999992966652, + 6.334190955385566, + 6.349519011564553, + 6.3354120356962085, + 6.280714995227754 + ], + "after": [ + 5.9602969558909535, + 5.947844008915126, + 5.957170040346682, + 5.975153995677829, + 5.950245074927807, + 5.9789830120280385, + 5.968288984149694, + 5.968034965917468, + 5.970570957288146, + 5.9893300058320165, + 5.974469007924199, + 5.955578992143273, + 5.940887960605323, + 6.066182977519929, + 5.963379982858896, + 5.992084043100476, + 5.993772996589541, + 6.073997006751597, + 5.978425033390522, + 6.0032380279153585 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.317329534795135, + "after": 5.972519982606173 + }, + "metadata_exact": true + }, + "128k-uniform-1024-planner": { + "samples_ms": { + "before": [ + 4.660016042180359, + 4.370620008558035, + 4.610156989656389, + 4.405395011417568, + 4.589467076584697, + 4.427237086929381, + 4.581127082929015, + 4.438054980710149, + 4.516609013080597, + 4.428007057867944, + 4.5784530229866505, + 4.393365001305938, + 4.563861060887575, + 4.433515015989542, + 4.587459960021079, + 4.470839980058372, + 4.528368008323014, + 4.389258101582527, + 4.762601922266185, + 4.427127074450254 + ], + "after": [ + 4.281781031750143, + 4.191130050458014, + 4.174580913968384, + 4.1822659550234675, + 4.152310895733535, + 4.233917919918895, + 4.1525529231876135, + 4.132509930059314, + 4.176480928435922, + 4.160788026638329, + 4.145585931837559, + 4.104895982891321, + 4.152894951403141, + 4.110871930606663, + 4.168654908426106, + 4.12381102796644, + 4.132851026952267, + 4.139319993555546, + 34.61243200581521, + 4.214663058519363 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 4.4937244965694845, + "after": 4.156841489020735 + }, + "metadata_exact": true + }, + "128k-near-uniform-512-planner": { + "samples_ms": { + "before": [ + 2.415246097370982, + 2.372286980971694, + 2.414268092252314, + 2.392580034211278, + 2.3668940411880612, + 2.36852397210896, + 2.380969002842903, + 2.3735250579193234, + 2.3787349928170443, + 2.379633951932192, + 2.394836046732962, + 2.3807089310139418, + 2.3655840195715427, + 2.373978029936552, + 2.379238954745233, + 2.3784040240570903, + 2.3559159599244595, + 2.3839951027184725, + 2.3599970154464245, + 2.3900530068203807 + ], + "after": [ + 2.221758011728525, + 2.173527958802879, + 2.2238719975575805, + 2.2164909169077873, + 2.238516928628087, + 2.2274290677160025, + 2.2253500064834952, + 2.1910270443186164, + 2.206207951530814, + 2.18932400457561, + 2.214886015281081, + 2.2102269576862454, + 2.2298539988696575, + 2.193198073655367, + 2.2093040170148015, + 2.2074709413573146, + 2.222349983640015, + 2.22071202006191, + 2.254899009130895, + 2.188967075198889 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.3789869737811387, + "after": 2.2156884660944343 + }, + "metadata_exact": true + }, + "128k-mixed-1000-planner": { + "samples_ms": { + "before": [ + 22.04404992517084, + 21.91894897259772, + 22.10096293129027, + 21.933616953901947, + 21.74217300489545, + 22.01524598058313, + 21.75239403732121, + 21.638283040374517, + 22.058080998249352, + 21.76110597793013, + 52.93793696910143, + 22.040894022211432, + 21.75105398055166, + 21.90667192917317, + 21.845451090484858, + 21.720419987104833, + 21.844093920663, + 21.968669956550002, + 21.688856999389827, + 21.834119921550155 + ], + "after": [ + 3.842544974759221, + 3.952922997996211, + 3.8634379161521792, + 3.8387340027838945, + 3.8389438996091485, + 3.857493051327765, + 3.85215796995908, + 3.9768930291756988, + 3.8245299365371466, + 3.8569149328395724, + 3.916134941391647, + 3.8611970376223326, + 3.885005018673837, + 3.850206034258008, + 3.961231908760965, + 3.874346031807363, + 3.897645976394415, + 3.849889966659248, + 3.866290906444192, + 3.8647529436275363 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 21.876061509829015, + "after": 3.862317476887256 + }, + "metadata_exact": true + }, + "128k-long-tail-1000-planner": { + "samples_ms": { + "before": [ + 6.098993006162345, + 6.06885296292603, + 6.041291984729469, + 6.102801999077201, + 6.055154954083264, + 6.118261022493243, + 6.063149077817798, + 6.077650003135204, + 6.003413000144064, + 6.109009962528944, + 6.0182519955560565, + 6.069358089007437, + 6.046032998710871, + 6.179652060382068, + 6.067587994039059, + 36.73038596753031, + 6.08207902405411, + 6.207393016666174, + 6.0486659640446305, + 6.28370197955519 + ], + "after": [ + 3.680306952446699, + 3.667220938950777, + 3.6552910460159183, + 3.65017494186759, + 3.651484032161534, + 3.630999941378832, + 3.6778110079467297, + 3.6716859322041273, + 3.616343019530177, + 3.675099927932024, + 3.6119039868935943, + 3.66475700866431, + 3.6578980507329106, + 3.6507350159808993, + 3.6120960721746087, + 3.7088380195200443, + 3.6899440456181765, + 3.7302899872884154, + 3.651808016002178, + 3.6856549559161067 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.073504046071321, + "after": 3.6613275296986103 + }, + "metadata_exact": true + }, + "256k-single-planner": { + "samples_ms": { + "before": [ + 6.273308070376515, + 6.1859110137447715, + 6.228829966858029, + 6.189352949149907, + 6.210490013472736, + 6.1686799162998796, + 6.207579048350453, + 6.210181047208607, + 6.191586027853191, + 6.232277024537325, + 6.19978504255414, + 6.172747001983225, + 6.196624017320573, + 6.1631620628759265, + 6.173041998408735, + 6.198281073011458, + 6.189342006109655, + 6.118671968579292, + 6.14934996701777, + 6.1761620454490185 + ], + "after": [ + 6.109548965469003, + 6.110334070399404, + 6.099766003899276, + 6.0219180304557085, + 6.071530049666762, + 6.033507990650833, + 6.164598977193236, + 6.1124550411477685, + 6.096701021306217, + 6.0586490435525775, + 6.0906229773536325, + 6.123605999164283, + 6.119196070358157, + 6.050423951819539, + 6.057007005438209, + 6.0730380937457085, + 6.070937961339951, + 6.0674239648506045, + 6.090157083235681, + 6.056089070625603 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.190469488501549, + "after": 6.081597588490695 + }, + "metadata_exact": true + }, + "256k-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.41015096940100193, + 0.3944459604099393, + 0.37739600520581007, + 0.3896639682352543, + 0.38664997555315495, + 0.4359870217740536, + 0.3803169820457697, + 0.3860710421577096, + 0.38487394340336323, + 0.3898339346051216, + 0.3733679186552763, + 0.38506602868437767, + 0.37351599894464016, + 0.38967595901340246, + 0.3705939743667841, + 0.3850290086120367, + 0.37040491588413715, + 0.38908596616238356, + 0.3693180624395609, + 0.39668893441557884 + ], + "after": [ + 0.3678260836750269, + 0.36171393003314734, + 0.3648300189524889, + 0.3432539524510503, + 0.35647302865982056, + 0.34370110370218754, + 0.35738199949264526, + 0.3357569221407175, + 0.3518399316817522, + 0.34252204932272434, + 0.35713601391762495, + 0.34019898157566786, + 0.35376299638301134, + 0.34015101846307516, + 0.35704299807548523, + 0.33987394999712706, + 0.3501370083540678, + 0.33766403794288635, + 0.35145890433341265, + 0.3514030249789357 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.38556853542104363, + "after": 0.3514309646561742 + }, + "metadata_exact": true + }, + "256k-near-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.4030209966003895, + 0.39380101952701807, + 0.37597701884806156, + 0.388526008464396, + 0.37151994183659554, + 0.379147008061409, + 0.37576595786958933, + 0.4110520239919424, + 0.37336302921175957, + 0.3853539237752557, + 0.3833520459011197, + 0.38638501428067684, + 0.37212600000202656, + 0.38512097671628, + 0.37283089477568865, + 0.395660987123847, + 0.3773580538108945, + 0.38487801793962717, + 0.3688790602609515, + 0.38768898230046034 + ], + "after": [ + 0.36612001713365316, + 0.3428839845582843, + 0.34924305509775877, + 0.33483700826764107, + 0.3550719702616334, + 0.3430369542911649, + 0.3512970870360732, + 0.3375509986653924, + 0.359527999535203, + 0.34393894020467997, + 0.35246205516159534, + 0.3352659987285733, + 0.35411003045737743, + 0.34323392901569605, + 0.36014197394251823, + 0.3383359871804714, + 0.3529870882630348, + 0.342561979778111, + 0.3608709666877985, + 0.3360230475664139 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.38411503192037344, + "after": 0.34659099765121937 + }, + "metadata_exact": true + }, + "256k-mixed-22-planner": { + "samples_ms": { + "before": [ + 6.795704015530646, + 6.791763938963413, + 6.846706965006888, + 6.778295035474002, + 6.843929062597454, + 6.815892062149942, + 6.78327598143369, + 6.800882983952761, + 6.778726004995406, + 6.860786932520568, + 6.743785925209522, + 6.825271062552929, + 6.850271951407194, + 6.783836986869574, + 6.734416005201638, + 6.789679056964815, + 6.794990971684456, + 6.805825047194958, + 6.787401041947305, + 6.9325019139796495 + ], + "after": [ + 6.480885902419686, + 6.506833015009761, + 6.519202026538551, + 6.67197594884783, + 6.540074944496155, + 6.439254037104547, + 6.543172057718039, + 6.608141935430467, + 6.494894973002374, + 6.468622013926506, + 6.479641073383391, + 6.490246974863112, + 6.506317062303424, + 6.468169973231852, + 6.521328934468329, + 6.469527957960963, + 6.47113798186183, + 6.5275689121335745, + 6.535366992466152, + 6.519639049656689 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.795347493607551, + "after": 6.506575038656592 + }, + "metadata_exact": true + }, + "256k-uniform-1024-planner": { + "samples_ms": { + "before": [ + 4.528131103143096, + 4.519441979937255, + 4.45167301222682, + 4.516285029239953, + 4.445967031642795, + 4.486287012696266, + 4.479257971979678, + 4.522491944953799, + 4.453120985999703, + 4.484906909056008, + 4.395911004394293, + 4.493816988542676, + 4.383869003504515, + 4.455235903151333, + 4.413711954839528, + 4.532746970653534, + 4.4780169846490026, + 4.572667996399105, + 4.481656011193991, + 4.5083899749442935 + ], + "after": [ + 4.2732099536806345, + 4.213123000226915, + 4.125150968320668, + 4.145705956034362, + 4.167816019617021, + 4.113686038181186, + 4.167040926404297, + 4.176416085101664, + 4.202626994810998, + 4.1443039663136005, + 4.177298047579825, + 4.142577992752194, + 4.125079023651779, + 4.1348160011693835, + 4.203732940368354, + 4.115461022593081, + 4.169943975284696, + 4.170452011749148, + 4.154626978561282, + 4.169474937953055 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 4.483281460124999, + "after": 4.167428473010659 + }, + "metadata_exact": true + }, + "256k-near-uniform-512-planner": { + "samples_ms": { + "before": [ + 2.4475599639117718, + 2.4344889679923654, + 2.395572024397552, + 2.387490007095039, + 2.3774970322847366, + 2.4405719013884664, + 2.4193800054490566, + 2.4099330184981227, + 2.4256249889731407, + 2.408756990917027, + 2.396582975052297, + 2.4163590278476477, + 2.3899219231680036, + 32.957385992631316, + 2.451278967782855, + 2.404685947112739, + 2.3890669690445065, + 2.4140229215845466, + 2.4009819608181715, + 2.457590075209737 + ], + "after": [ + 2.2638730006292462, + 2.2619450464844704, + 2.23950098734349, + 2.2017869632691145, + 2.215599990449846, + 2.204743097536266, + 2.232411061413586, + 2.207626006565988, + 2.206437988206744, + 2.177756978198886, + 2.250102930702269, + 2.2028969833627343, + 2.236993983387947, + 2.2443720372393727, + 2.2390360245481133, + 2.2466800874099135, + 2.2268480388447642, + 2.2540870122611523, + 2.21646495629102, + 2.2201479878276587 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.4119779700413346, + "after": 2.229629550129175 + }, + "metadata_exact": true + }, + "256k-mixed-1000-planner": { + "samples_ms": { + "before": [ + 24.54034995753318, + 24.531275033950806, + 24.561429978348315, + 24.81341699603945, + 54.413284990005195, + 24.231813033111393, + 24.326125043444335, + 24.51071992982179, + 24.568932014517486, + 24.544357089325786, + 24.218196980655193, + 24.278299999423325, + 54.848834057338536, + 24.51823907904327, + 24.61230696644634, + 24.68205103650689, + 24.32792796753347, + 24.375067907385528, + 24.580372031778097, + 24.808765039779246 + ], + "after": [ + 16.764546977356076, + 16.3035630248487, + 16.358520020730793, + 16.450993018224835, + 16.33723301347345, + 16.516595031134784, + 16.50680403690785, + 16.276226029731333, + 16.32769708521664, + 16.374025959521532, + 16.418756917119026, + 16.53418701607734, + 16.604391974397004, + 16.265636077150702, + 16.35824504774064, + 16.457217978313565, + 16.542282071895897, + 16.551429987885058, + 16.530111082829535, + 16.244163969531655 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 24.542353523429483, + "after": 16.43487496767193 + }, + "metadata_exact": true + }, + "256k-long-tail-1000-planner": { + "samples_ms": { + "before": [ + 23.28029298223555, + 22.953859996050596, + 23.113874020054936, + 23.350395960733294, + 22.959975991398096, + 22.866397979669273, + 53.57191897928715, + 23.445561062544584, + 22.97284291125834, + 23.276207968592644, + 23.004574002698064, + 23.091771989129484, + 23.446962935850024, + 23.451111977919936, + 23.194006993435323, + 53.4638250246644, + 23.02024490199983, + 23.022315930575132, + 23.23351800441742, + 23.345256922766566 + ], + "after": [ + 15.0655850302428, + 15.106525970622897, + 15.416737063787878, + 14.999473001807928, + 15.052370028570294, + 15.261941007338464, + 14.996233046986163, + 15.140222036279738, + 15.17251692712307, + 15.06192295346409, + 15.014206990599632, + 15.350549947470427, + 15.018568956293166, + 15.140537987463176, + 15.73672704398632, + 14.988586073741317, + 15.155900036916137, + 15.214379993267357, + 15.043093007989228, + 15.059495926834643 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 23.21376249892637, + "after": 15.086055500432849 + }, + "metadata_exact": true + }, + "1024k-single-planner": { + "samples_ms": { + "before": [ + 6.401306949555874, + 6.363680004142225, + 6.276860018260777, + 6.301490939222276, + 6.289783981628716, + 6.289764074608684, + 6.253407918848097, + 6.344108027406037, + 6.206598016433418, + 6.301910034380853, + 6.258120061829686, + 6.300506996922195, + 6.237262045033276, + 6.291158031672239, + 6.250014994293451, + 6.290553021244705, + 6.247104960493743, + 6.244288058951497, + 6.247733021155, + 6.355733028613031 + ], + "after": [ + 6.215190049260855, + 6.177706993184984, + 6.197663955390453, + 6.145993946120143, + 6.222699070349336, + 6.169933010824025, + 6.188363069668412, + 6.152393063530326, + 6.236356915906072, + 6.181873031891882, + 6.244497955776751, + 6.177121074870229, + 6.194881978444755, + 6.158321048133075, + 6.16318394895643, + 6.140960962511599, + 6.226410041563213, + 6.211370928213, + 6.178151001222432, + 6.177538889460266 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 6.2897740281187, + "after": 6.180012016557157 + }, + "metadata_exact": true + }, + "1024k-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.42080599814653397, + 0.4038750194013119, + 0.38857804611325264, + 0.4034269368276, + 0.38967293221503496, + 0.40273508056998253, + 0.3832620568573475, + 0.3959540044888854, + 0.3901791060343385, + 0.3977610031142831, + 0.43747795280069113, + 0.4107679706066847, + 0.3902378957718611, + 0.3971179248765111, + 0.38518698420375586, + 0.4006110830232501, + 0.3892940003424883, + 0.3975259605795145, + 0.3838939592242241, + 0.4060509381815791 + ], + "after": [ + 0.36720791831612587, + 0.3625159151852131, + 0.37771405186504126, + 0.35041000228375196, + 0.36165607161819935, + 0.349257024936378, + 0.36535400431603193, + 0.34397293347865343, + 0.36269298288971186, + 0.34938903991132975, + 0.3802860155701637, + 0.357507960870862, + 0.3609630512073636, + 0.34980091731995344, + 0.36741001531481743, + 0.3414710517972708, + 0.3660570364445448, + 0.3538549644872546, + 0.37089898250997066, + 0.35520095843821764 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.3973219427280128, + "after": 0.3613095614127815 + }, + "metadata_exact": true + }, + "1024k-near-uniform-64-planner": { + "samples_ms": { + "before": [ + 0.411357032135129, + 0.4092190647497773, + 0.38928911089897156, + 0.39956194814294577, + 0.39079703856259584, + 0.40034507401287556, + 0.3874669782817364, + 0.40125101804733276, + 0.38783892523497343, + 0.4079029895365238, + 0.38890400901436806, + 0.3955049905925989, + 0.3836510004475713, + 0.39660301990807056, + 0.37577003240585327, + 0.40066102519631386, + 0.38612191565334797, + 0.40559505578130484, + 0.38500898517668247, + 0.3992560086771846 + ], + "after": [ + 0.36906194873154163, + 0.3460940206423402, + 0.359949073754251, + 0.34843606408685446, + 0.36219297908246517, + 0.34608994610607624, + 0.36883691791445017, + 0.34593301825225353, + 0.3651459701359272, + 0.35133492201566696, + 0.36212196573615074, + 0.3420230932533741, + 0.356883043423295, + 0.34345604944974184, + 0.3590090200304985, + 0.34881499595940113, + 0.3593889996409416, + 0.34409610088914633, + 0.3649709979072213, + 0.3488330403342843 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 0.39605400525033474, + "after": 0.354108982719481 + }, + "metadata_exact": true + }, + "1024k-mixed-22-planner": { + "samples_ms": { + "before": [ + 7.116922060959041, + 7.177904015406966, + 7.13980000000447, + 7.036815048195422, + 7.111265091225505, + 7.035356014966965, + 7.030882057733834, + 7.0656288880854845, + 7.051595952361822, + 7.055683992803097, + 7.0082469610497355, + 7.1635430213063955, + 7.037836010567844, + 7.078544935211539, + 7.08668795414269, + 7.108084042556584, + 7.043128018267453, + 7.059555035084486, + 7.088234066031873, + 7.057390990667045 + ], + "after": [ + 6.80316099897027, + 6.773620028980076, + 6.781495991162956, + 6.73370691947639, + 6.742869038134813, + 6.748192943632603, + 6.805401062592864, + 6.764878984540701, + 6.730035995133221, + 6.813118001446128, + 6.788165075704455, + 6.766583072021604, + 6.699447054415941, + 6.768840947188437, + 6.729939021170139, + 6.789920036680996, + 6.8319570273160934, + 6.730383029207587, + 6.752830930054188, + 6.794777000322938 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 7.062591961584985, + "after": 6.76771200960502 + }, + "metadata_exact": true + }, + "1024k-uniform-1024-planner": { + "samples_ms": { + "before": [ + 4.723243066109717, + 4.931011004373431, + 4.759495961479843, + 4.715371993370354, + 4.765891004353762, + 4.700714023783803, + 4.798600100912154, + 4.737420007586479, + 4.807673045434058, + 4.715082002803683, + 4.753511981107295, + 4.659031983464956, + 4.738165996968746, + 4.7197270905599, + 37.12526906747371, + 4.684593062847853, + 4.794273059815168, + 4.75063594058156, + 4.797957022674382, + 4.689068999141455 + ], + "after": [ + 4.392486996948719, + 4.454082925803959, + 4.302648943848908, + 4.259853973053396, + 4.329560091719031, + 4.278030013665557, + 4.359407001174986, + 4.328979062847793, + 4.3300940888002515, + 4.302379908040166, + 4.365602973848581, + 4.28319105412811, + 4.396082018502057, + 4.303489928133786, + 4.510325030423701, + 4.342301981523633, + 4.266428062692285, + 4.304714035242796, + 4.332002019509673, + 4.264610004611313 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 4.744400968775153, + "after": 4.329269577283412 + }, + "metadata_exact": true + }, + "1024k-near-uniform-512-planner": { + "samples_ms": { + "before": [ + 2.515185042284429, + 2.4836119264364243, + 2.4774610064923763, + 2.4514689575880766, + 2.4803068954497576, + 2.4744559777900577, + 2.441972028464079, + 2.450105967000127, + 2.4273539893329144, + 2.5212119799107313, + 2.449671970680356, + 2.4625950027257204, + 2.465908066369593, + 2.4435248924419284, + 2.432778012007475, + 2.447120030410588, + 2.4312910391017795, + 2.4819980608299375, + 2.413704991340637, + 2.454346977174282 + ], + "after": [ + 2.2778710117563605, + 2.224409021437168, + 2.2687099408358335, + 2.189205028116703, + 2.4770909221842885, + 2.2167189745232463, + 2.2215070202946663, + 2.2258959943428636, + 2.24177201744169, + 2.220332040451467, + 2.2220200626179576, + 2.1935610566288233, + 2.365433028899133, + 2.3080429527908564, + 2.229491015896201, + 2.2006690269336104, + 2.2265500156208873, + 2.22190807107836, + 2.215657033957541, + 2.203262993134558 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.4529079673811793, + "after": 2.223214542027563 + }, + "metadata_exact": true + }, + "1024k-mixed-1000-planner": { + "samples_ms": { + "before": [ + 25.652137934230268, + 25.46787296887487, + 25.711324997246265, + 25.766848935745656, + 25.748226093128324, + 25.68940701894462, + 25.380223989486694, + 25.50192188937217, + 25.78106301371008, + 57.11816402617842, + 25.759859010577202, + 25.784690980799496, + 25.438977987505496, + 25.33386100549251, + 25.694990996271372, + 25.588147924281657, + 25.80513700377196, + 25.829382007941604, + 56.03781796526164, + 25.28274105861783 + ], + "after": [ + 17.50727999024093, + 48.44116396270692, + 17.65985193196684, + 17.201192094944417, + 17.23041303921491, + 17.503901035524905, + 17.601707950234413, + 17.612167983315885, + 17.567483941093087, + 17.297217971645296, + 17.34165393281728, + 17.509303987026215, + 17.61009602341801, + 17.63615501113236, + 17.625470063649118, + 17.18583598267287, + 17.181201023049653, + 17.410952015779912, + 17.32693100348115, + 17.414303962141275 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 25.70315799675882, + "after": 17.505590512882918 + }, + "metadata_exact": true + }, + "1024k-long-tail-1000-planner": { + "samples_ms": { + "before": [ + 25.22357099223882, + 25.02503409050405, + 24.86469899304211, + 24.80024192482233, + 24.994525010697544, + 24.8831610661, + 25.079349987208843, + 25.113655952736735, + 24.65354697778821, + 24.713349994271994, + 24.960741982795298, + 24.926676996983588, + 24.976362008601427, + 24.950529914349318, + 24.81701597571373, + 25.46109596733004, + 24.933145963586867, + 24.92614893708378, + 24.99815507326275, + 25.036363978870213 + ], + "after": [ + 16.639605979435146, + 16.80207799654454, + 47.643300029449165, + 16.94847596809268, + 17.137111048214138, + 16.74931705929339, + 16.65344601497054, + 16.915512969717383, + 17.00916001573205, + 17.092264024540782, + 17.00384693685919, + 46.60715803038329, + 16.701380955055356, + 16.74321002792567, + 16.781526035629213, + 16.892694984562695, + 17.648648004978895, + 16.605001059360802, + 16.71510294545442, + 48.10157907195389 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 24.955635948572308, + "after": 16.90410397714004 + }, + "metadata_exact": true + }, + "128k-uniform-1024-zigzag": { + "samples_ms": { + "before": [ + 69.64113505091518, + 73.64157098345459, + 38.45433599781245, + 72.69224699120969, + 38.10993500519544, + 38.43369905371219, + 38.421318982727826, + 38.43365400098264, + 38.649844005703926, + 38.74922904651612, + 38.66042196750641, + 38.57172594871372, + 75.46621793881059, + 38.436721079051495, + 72.60013394989073, + 76.3658790383488, + 38.860194967128336, + 73.75223899725825, + 38.60297589562833, + 39.060689974576235 + ], + "after": [ + 37.776404060423374, + 36.784694995731115, + 36.321370978839695, + 36.35120403487235, + 36.09032602980733, + 74.06754896510392, + 36.53954900801182, + 71.28948497120291, + 74.33333899825811, + 37.268035928718746, + 72.1440309425816, + 36.91740706562996, + 37.01140102930367, + 36.919855047017336, + 36.93395503796637, + 36.86134691815823, + 37.082409020513296, + 37.04060195013881, + 37.2941589448601, + 74.28962702397257 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 38.704825507011265, + "after": 37.02600148972124 + }, + "metadata_exact": true + }, + "128k-uniform-1024-contiguous": { + "samples_ms": { + "before": [ + 3.268019063398242, + 2.554591977968812, + 2.479085000231862, + 2.5484160287305713, + 2.4840160040184855, + 2.4452970828861, + 2.4797720834612846, + 2.4469959316775203, + 2.4430269841104746, + 2.4849720066413283, + 2.456852002069354, + 2.4770579766482115, + 2.437687013298273, + 2.4830519687384367, + 2.3991750786080956, + 2.492676954716444, + 2.4221889907494187, + 2.460542949847877, + 2.421214012429118, + 2.4535140255466104 + ], + "after": [ + 2.305899979546666, + 2.254140912555158, + 2.2742359433323145, + 2.231505000963807, + 2.252420992590487, + 2.2559830686077476, + 2.2199279628694057, + 2.2693349746987224, + 2.2399929584935308, + 2.216992899775505, + 2.2454659920185804, + 2.216396969743073, + 2.266811905428767, + 2.1937619894742966, + 2.2288409527391195, + 2.2255079820752144, + 2.255016006529331, + 2.1914190147072077, + 2.2441610926762223, + 2.207132987678051 + ] + }, + "orders": [ + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ], + [ + "before", + "after" + ], + [ + "after", + "before" + ] + ], + "median_ms": { + "before": 2.4688004632480443, + "after": 2.2420770255848765 + }, + "metadata_exact": true + } + }, + "source_sha256": { + "cula/distributed/thd_planner.py": "960a403b1e804b087653fd5c8032781fd2d1249f37a6a7b0f2b80c3fca1896f6", + "cula/hybrid/dsa.py": "853f7045ead4d230d4bd5b0a211d60861c840c6190c6502bfdcde310cf866bc2", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f" + }, + "metadata_comparison_scope": "All plan fields except evaluated_scores; omitted nonwinning candidates are checked in the separate planner parity suite", + "warmups_per_variant": 2, + "order": "Alternating before/after and after/before, 20 samples per variant" + }, + "schema_version": 1, + "tested_source_commit": "489779ba35e2c5f51eccce7c7ff95ed4cdd640d0", + "validation": { + "focused_cpu_tests_passed": 518, + "planner_comparisons": 660, + "planner_comparison_scope": "Selected layout, metrics and score exactly match baseline; retained candidate scores match; omitted candidates cannot meet migration margin", + "random_dsa_plans": 300, + "identical_dsa_windows": 69281, + "native_subgroup_checks": 128, + "native_subgroup_scope": "Unchanged native runtime: BF16 and FP32 weights, explicit subgroups, empty destinations, retained outputs", + "complete_csa_cases": 24, + "paths_per_case": 4, + "ranks_per_case": 8, + "compared_tensors_per_public_path": 8, + "comparison": "Full tensor output and seven gradients against legacy contiguous; native nondeterminism enabled", + "relative_rms_limit": 0.01, + "auxiliary_loss_abs_limit": 1e-05, + "small_fp32_oracle_cases": 3, + "small_fp32_oracle_scope": "All four paths; attention, teacher/KL and pooling with selected keys; no independent native ranking oracle", + "cp64_scope": "CPU metadata only" + } +} diff --git a/docs/results/csa_cp_three_way_20261116.json b/docs/results/csa_cp_three_way_20261116.json new file mode 100644 index 00000000..9ad4f8da --- /dev/null +++ b/docs/results/csa_cp_three_way_20261116.json @@ -0,0 +1,3681 @@ +{ + "scope": "Complete CSA operator forward/backward, including layouts and communication; no KDA/projections/MoE", + "platform": "B300 CP8", + "native_source_commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "runtime_commit": "d2cf4fdf7b7ec8fc4e22ecd780dd4a48d0a61670", + "cases": { + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "4ffc0c811647416d205412593d727352a535ab6cd11c26ad666b219d87fde48f", + "policies": { + "contiguous": { + "step_ms": 47.59836959838867, + "prepare_ms": 0.7295145187526941, + "one_use_ms": 48.327884117141366, + "prepare_min_ms": 0.7136678323149681, + "prepare_max_ms": 0.8075819350779057, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015765946646741124, + "dq": 0.0005790440951112175, + "dkv": 0.0006679192583025653, + "d_index_q": 0.0007036192249884126, + "d_weights": 0.0006587839768356234, + "d_pool_kv": 0.0008462754770025072, + "d_pool_score": 0.000887909900268915, + "d_ape": 2.647273742900797e-05 + }, + "max_loss_abs_diff": 1.4551915228366852e-10, + "critical_step_samples_ms": [ + 47.504417419433594, + 47.59836959838867, + 47.79350280761719, + 47.29996871948242, + 47.922752380371094, + 47.66716766357422, + 47.468990325927734 + ], + "critical_prepare_samples_ms": [ + 0.8075819350779057, + 0.7368980441242456, + 0.7332341047003865, + 0.7257949328050017, + 0.7136678323149681, + 0.7397379959002137, + 0.7191249169409275, + 0.7216879166662693 + ] + }, + "zigzag": { + "step_ms": 46.22537612915039, + "prepare_ms": 0.9122525225393474, + "one_use_ms": 47.13762865168974, + "prepare_min_ms": 0.8884459966793656, + "prepare_max_ms": 2.3740038741379976, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015766157105562468, + "dq": 0.0005791250183306168, + "dkv": 0.0005826837325579235, + "d_index_q": 0.0006550745397822342, + "d_weights": 0.0005112390665302352, + "d_pool_kv": 0.0007098261716766501, + "d_pool_score": 0.0007224017331145301, + "d_ape": 2.402751970718015e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 46.06342315673828, + 46.19878387451172, + 46.51465606689453, + 46.22537612915039, + 46.21033477783203, + 46.31951904296875, + 46.25299072265625 + ], + "critical_prepare_samples_ms": [ + 2.3740038741379976, + 0.9390920167788863, + 0.9824440348893404, + 0.9092459222301841, + 0.9014920797199011, + 0.9148720419034362, + 0.8884459966793656, + 0.9096330031752586 + ] + }, + "planner": { + "step_ms": 45.816734313964844, + "prepare_ms": 7.54898099694401, + "one_use_ms": 53.365715310908854, + "prepare_min_ms": 7.393982145003974, + "prepare_max_ms": 7.698302972130477, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0015837265043694255, + "dq": 0.0005816460263006044, + "dkv": 0.0007768235696223359, + "d_index_q": 0.0007047741714831017, + "d_weights": 0.000687690452839031, + "d_pool_kv": 0.0008500666348145983, + "d_pool_score": 0.0009004768884284642, + "d_ape": 2.473088932251163e-05 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 45.82166290283203, + 45.84272003173828, + 45.73017501831055, + 46.25267028808594, + 45.816734313964844, + 45.81379318237305, + 45.737918853759766 + ], + "critical_prepare_samples_ms": [ + 7.6499179704114795, + 7.698302972130477, + 7.51110992860049, + 7.575499941594899, + 7.636827998794615, + 7.517814985476434, + 7.522462052293122, + 7.393982145003974 + ] + } + } + }, + "128k-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "3a1a1ac62ea766a15b80829e415a9660c79cea6aed1be6fb755c4527f4bef8d4", + "policies": { + "contiguous": { + "step_ms": 20.111711502075195, + "prepare_ms": 0.9577220189385116, + "one_use_ms": 21.069433521013707, + "prepare_min_ms": 0.9080220479518175, + "prepare_max_ms": 1.176033983938396, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.589457181863983e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.33998632212166e-05, + "d_pool_score": 6.779927803572065e-05, + "d_ape": 1.564723126474068e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.175872802734375, + 20.117664337158203, + 20.179584503173828, + 20.060287475585938, + 20.016511917114258, + 20.111711502075195, + 20.057247161865234 + ], + "critical_prepare_samples_ms": [ + 1.0195919312536716, + 0.9686399716883898, + 0.9425180032849312, + 1.176033983938396, + 0.9080220479518175, + 0.9982179617509246, + 0.9080249583348632, + 0.9468040661886334 + ] + }, + "zigzag": { + "step_ms": 20.662303924560547, + "prepare_ms": 4.178662959020585, + "one_use_ms": 24.84096688358113, + "prepare_min_ms": 3.9806190179660916, + "prepare_max_ms": 4.345727036707103, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 5.160216835107211e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.664592625392898e-05, + "d_pool_score": 6.241826443094128e-05, + "d_ape": 1.4828357064696632e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.99295997619629, + 20.976959228515625, + 20.812480926513672, + 20.662303924560547, + 20.6615047454834, + 20.65987205505371, + 20.65795135498047 + ], + "critical_prepare_samples_ms": [ + 4.096206976100802, + 4.345727036707103, + 4.321377025917172, + 4.138701944611967, + 4.223175114020705, + 3.98633093573153, + 4.218623973429203, + 3.9806190179660916 + ] + }, + "planner": { + "step_ms": 20.038047790527344, + "prepare_ms": 1.2417834950610995, + "one_use_ms": 21.279831285588443, + "prepare_min_ms": 1.1157230474054813, + "prepare_max_ms": 1.3633579947054386, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.742825463536053e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.19493785934264e-05, + "d_pool_score": 6.324612395848447e-05, + "d_ape": 1.6624866979508424e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 20.352832794189453, + 20.19059181213379, + 20.023487091064453, + 20.057823181152344, + 20.014240264892578, + 20.036767959594727, + 20.038047790527344 + ], + "critical_prepare_samples_ms": [ + 1.2311399914324284, + 1.2524269986897707, + 1.3623570557683706, + 1.3633579947054386, + 1.257814932614565, + 1.134752994403243, + 1.1582979932427406, + 1.1157230474054813 + ] + } + } + }, + "128k-near-uniform-64": { + "documents": 64, + "tokens": 131072, + "source_record_sha256": "5ef7ec8fee87197b0c117b0eaf0008be59967fe8c900e7161fd432c33f97ea77", + "policies": { + "contiguous": { + "step_ms": 20.080768585205078, + "prepare_ms": 0.9788154857233167, + "one_use_ms": 21.059584070928395, + "prepare_min_ms": 0.9078129660338163, + "prepare_max_ms": 1.02512096054852, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016355387131441923, + "dq": 8.456624756602418e-05, + "dkv": 0.0001043296667775725, + "d_index_q": 2.4571583304236613e-05, + "d_weights": 2.937356095947118e-05, + "d_pool_kv": 9.235941677369348e-05, + "d_pool_score": 0.00012162571635460319, + "d_ape": 3.316459358658715e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.122495651245117, + 20.08835220336914, + 20.119455337524414, + 20.080768585205078, + 20.055072784423828, + 20.056095123291016, + 20.064416885375977 + ], + "critical_prepare_samples_ms": [ + 1.02512096054852, + 0.9624578524380922, + 0.9632399305701256, + 1.0062160436064005, + 0.9819930419325829, + 0.9756379295140505, + 0.9078129660338163, + 0.9824160952121019 + ] + }, + "zigzag": { + "step_ms": 20.79020881652832, + "prepare_ms": 4.162916971836239, + "one_use_ms": 24.95312578836456, + "prepare_min_ms": 4.007034818641841, + "prepare_max_ms": 4.921410931274295, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.000167355533096453, + "dq": 8.740554038293048e-05, + "dkv": 0.0001053261127085974, + "d_index_q": 2.491827918171799e-05, + "d_weights": 2.5499445570320522e-05, + "d_pool_kv": 8.765414636157637e-05, + "d_pool_score": 7.288222095933882e-05, + "d_ape": 2.6757349713227653e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.79020881652832, + 20.970272064208984, + 20.883359909057617, + 20.743200302124023, + 20.754911422729492, + 20.835552215576172, + 20.762752532958984 + ], + "critical_prepare_samples_ms": [ + 4.25333296880126, + 4.007034818641841, + 4.1503519751131535, + 4.179439041763544, + 4.06482198741287, + 4.118405049666762, + 4.921410931274295, + 4.175481968559325 + ] + }, + "planner": { + "step_ms": 20.055456161499023, + "prepare_ms": 1.181907020509243, + "one_use_ms": 21.237363182008266, + "prepare_min_ms": 1.1588750639930367, + "prepare_max_ms": 1.244336017407477, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00016034442556637804, + "dq": 8.377134480427006e-05, + "dkv": 0.00012869833642443928, + "d_index_q": 2.4304393144673242e-05, + "d_weights": 3.1874338742783715e-05, + "d_pool_kv": 7.765903597090855e-05, + "d_pool_score": 6.551608904168383e-05, + "d_ape": 2.6567923612493613e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 20.058752059936523, + 20.152511596679688, + 20.14681625366211, + 20.055456161499023, + 20.03536033630371, + 20.055200576782227, + 20.035104751586914 + ], + "critical_prepare_samples_ms": [ + 1.244336017407477, + 1.1704741045832634, + 1.1588750639930367, + 1.1804300593212247, + 1.1833839816972613, + 1.1672999244183302, + 1.1852069292217493, + 1.2041899608448148 + ] + } + } + }, + "128k-mixed-22": { + "documents": 22, + "tokens": 131072, + "source_record_sha256": "d54ce7bb5382e15f3d8a885c6e53f2d59fb252c6dbc73a8e030ff30ea58fd017", + "policies": { + "contiguous": { + "step_ms": 35.437599182128906, + "prepare_ms": 0.8113369694910944, + "one_use_ms": 36.24893615162, + "prepare_min_ms": 0.7874679286032915, + "prepare_max_ms": 11.578752077184618, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0019160342752732016, + "dq": 0.0005784768780049561, + "dkv": 0.0004627341121441643, + "d_index_q": 0.0003634974114396893, + "d_weights": 0.0003493773692632088, + "d_pool_kv": 0.0006592622470164842, + "d_pool_score": 0.0006617473534238676, + "d_ape": 1.5070293250763499e-05 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 35.15270233154297, + 34.845760345458984, + 35.5722541809082, + 35.437599182128906, + 36.6080322265625, + 35.424896240234375, + 35.540000915527344 + ], + "critical_prepare_samples_ms": [ + 0.8466748986393213, + 0.8107968606054783, + 11.578752077184618, + 0.8092040661722422, + 0.7907270919531584, + 0.8135039824992418, + 0.7874679286032915, + 0.8118770783767104 + ] + }, + "zigzag": { + "step_ms": 35.23379135131836, + "prepare_ms": 1.9275369704701006, + "one_use_ms": 37.16132832178846, + "prepare_min_ms": 1.8797999946400523, + "prepare_max_ms": 3.936605993658304, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0019299163049260265, + "dq": 0.0005823918678050893, + "dkv": 0.0004619453030516524, + "d_index_q": 0.0003622360991686167, + "d_weights": 0.0003449186778998967, + "d_pool_kv": 0.0006607552043101183, + "d_pool_score": 0.0006699460238477643, + "d_ape": 1.4678422801075643e-05 + }, + "max_loss_abs_diff": 8.149072527885437e-10, + "critical_step_samples_ms": [ + 35.07795333862305, + 35.11782455444336, + 35.7696647644043, + 35.212799072265625, + 35.694305419921875, + 35.23379135131836, + 36.08281707763672 + ], + "critical_prepare_samples_ms": [ + 3.936605993658304, + 1.9458660390228033, + 1.9063919316977262, + 1.9079309422522783, + 1.9645949359983206, + 1.9459249451756477, + 1.909207901917398, + 1.8797999946400523 + ] + }, + "planner": { + "step_ms": 35.436702728271484, + "prepare_ms": 7.6061730505898595, + "one_use_ms": 43.042875778861344, + "prepare_min_ms": 7.441366091370583, + "prepare_max_ms": 7.997068925760686, + "selected_policy": "selection-local-3x", + "max_relative_rms": { + "output": 0.001919035177989802, + "dq": 0.0005792319248342236, + "dkv": 0.00046265994814852977, + "d_index_q": 0.00036264936861235027, + "d_weights": 0.00034889076944032783, + "d_pool_kv": 0.0006608845275100428, + "d_pool_score": 0.0006599980949303307, + "d_ape": 1.2610507503426976e-05 + }, + "max_loss_abs_diff": 5.820766091346741e-10, + "critical_step_samples_ms": [ + 34.854976654052734, + 35.8928337097168, + 35.35036849975586, + 35.62351989746094, + 35.61305618286133, + 35.436702728271484, + 35.13798522949219 + ], + "critical_prepare_samples_ms": [ + 7.997068925760686, + 7.598275085911155, + 7.614071015268564, + 7.47464899905026, + 7.661403971724212, + 7.441366091370583, + 7.570564979687333, + 7.846773136407137 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "38e8fdad2096841ce9b57196cdb05dfe4cd17b7c2311cf5bede714c90c5d3a91", + "policies": { + "contiguous": { + "step_ms": 9.287199974060059, + "prepare_ms": 6.7794660571962595, + "one_use_ms": 16.066666031256318, + "prepare_min_ms": 6.7019599955528975, + "prepare_max_ms": 7.129473960958421, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7034465202561103e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.80841349081689e-05, + "d_pool_score": 4.969745146211584e-05, + "d_ape": 2.463488004291754e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.327584266662598, + 9.295040130615234, + 9.287199974060059, + 9.278176307678223, + 9.278143882751465, + 9.258208274841309, + 9.473567962646484 + ], + "critical_prepare_samples_ms": [ + 6.837799912318587, + 6.7019599955528975, + 6.714918999932706, + 6.821699091233313, + 7.129473960958421, + 6.737233023159206, + 6.707626045681536, + 6.954675889573991 + ] + }, + "zigzag": { + "step_ms": 11.102463722229004, + "prepare_ms": 47.22392506664619, + "one_use_ms": 58.32638878887519, + "prepare_min_ms": 45.078030903823674, + "prepare_max_ms": 156.1478169169277, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.723088398789181e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.7408244692422145e-05, + "d_pool_score": 6.371221934010526e-05, + "d_ape": 3.3825512629530646e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.100447654724121, + 11.142175674438477, + 11.123104095458984, + 11.102463722229004, + 11.112383842468262, + 11.097536087036133, + 11.057215690612793 + ], + "critical_prepare_samples_ms": [ + 48.150649992749095, + 46.42646899446845, + 45.078030903823674, + 151.0155029827729, + 47.763709095306695, + 46.68414103798568, + 45.34025501925498, + 156.1478169169277 + ] + }, + "planner": { + "step_ms": 9.295295715332031, + "prepare_ms": 8.97316396003589, + "one_use_ms": 18.26845967536792, + "prepare_min_ms": 8.708094945177436, + "prepare_max_ms": 123.31375910434872, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.9905925067018862e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.216577144844658e-05, + "d_pool_score": 6.720340578861908e-05, + "d_ape": 3.3511195947336086e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 9.287327766418457, + 9.30742359161377, + 9.270848274230957, + 9.30361557006836, + 9.306976318359375, + 9.284768104553223, + 9.295295715332031 + ], + "critical_prepare_samples_ms": [ + 8.869540877640247, + 8.825646946206689, + 123.31375910434872, + 9.473679936490953, + 8.962570922449231, + 8.708094945177436, + 9.538239915855229, + 8.98375699762255 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "be33650d531537cf859041aa184f7846ca91ee7d6f07866e13f5eff3df7ff982", + "policies": { + "contiguous": { + "step_ms": 10.01318359375, + "prepare_ms": 3.743164998013526, + "one_use_ms": 13.756348591763526, + "prepare_min_ms": 3.6540150176733732, + "prepare_max_ms": 3.838455071672797, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 4.31067005302863e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.485686835600299e-05, + "d_pool_score": 3.734317152918191e-05, + "d_ape": 1.5884632522867948e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 10.00160026550293, + 10.01318359375, + 9.96771240234375, + 10.067392349243164, + 9.997440338134766, + 10.017472267150879, + 10.042176246643066 + ], + "critical_prepare_samples_ms": [ + 3.793359035626054, + 3.752059070393443, + 3.740546992048621, + 3.739589941687882, + 3.700686036609113, + 3.7457830039784312, + 3.6540150176733732, + 3.838455071672797 + ] + }, + "zigzag": { + "step_ms": 11.205792427062988, + "prepare_ms": 24.085226003080606, + "one_use_ms": 35.291018430143595, + "prepare_min_ms": 23.83457194082439, + "prepare_max_ms": 139.23344609793276, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7925220371458967e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 7.136588948385061e-05, + "d_pool_score": 4.065888036840021e-05, + "d_ape": 1.7333242400394557e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 11.202655792236328, + 11.212991714477539, + 11.187968254089355, + 11.230527877807617, + 11.205792427062988, + 11.16857624053955, + 11.311712265014648 + ], + "critical_prepare_samples_ms": [ + 27.768890955485404, + 25.140583980828524, + 24.11484904587269, + 23.88202305883169, + 24.055602960288525, + 24.04588391073048, + 23.83457194082439, + 139.23344609793276 + ] + }, + "planner": { + "step_ms": 10.027359962463379, + "prepare_ms": 4.955698561388999, + "one_use_ms": 14.983058523852378, + "prepare_min_ms": 4.824013100005686, + "prepare_max_ms": 5.206209025345743, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3273367675160776e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.542720423634009e-05, + "d_pool_score": 5.10650784981466e-05, + "d_ape": 2.1403371983524285e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 10.017024040222168, + 10.369183540344238, + 10.027359962463379, + 10.05395221710205, + 10.025664329528809, + 10.148415565490723, + 10.026752471923828 + ], + "critical_prepare_samples_ms": [ + 5.206209025345743, + 5.033118999563158, + 4.875925951637328, + 4.824013100005686, + 4.920054925605655, + 4.9812550423666835, + 4.930142080411315, + 5.035332054831088 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "a42d7db2614d9e21c3bd24a8de34199977030c3a75e1b28856d51a604529dfdb", + "policies": { + "contiguous": { + "step_ms": 33.07494354248047, + "prepare_ms": 7.641692471224815, + "one_use_ms": 40.71663601370528, + "prepare_min_ms": 7.504745037294924, + "prepare_max_ms": 110.33575492911041, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002124358368844286, + "dq": 0.0006708973533251172, + "dkv": 0.0006748384557586351, + "d_index_q": 0.000419026902945799, + "d_weights": 0.0003304557285110155, + "d_pool_kv": 0.0005772261277278353, + "d_pool_score": 0.0005797688319552085, + "d_ape": 7.489935465690975e-06 + }, + "max_loss_abs_diff": 3.4924596548080444e-10, + "critical_step_samples_ms": [ + 33.100799560546875, + 33.073055267333984, + 33.057918548583984, + 33.08787155151367, + 33.07494354248047, + 33.04643249511719, + 33.11958312988281 + ], + "critical_prepare_samples_ms": [ + 8.83635098580271, + 7.582117919810116, + 110.33575492911041, + 7.812974974513054, + 7.556077907793224, + 7.6242099748924375, + 7.504745037294924, + 7.659174967557192 + ] + }, + "zigzag": { + "step_ms": 38.74604797363281, + "prepare_ms": 45.72551843011752, + "one_use_ms": 84.47156640375033, + "prepare_min_ms": 44.49985595420003, + "prepare_max_ms": 152.91648893617094, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.002136063325688249, + "dq": 0.0006448208867270936, + "dkv": 0.0005397046574182863, + "d_index_q": 0.0003537235166520844, + "d_weights": 0.0003170419424130787, + "d_pool_kv": 0.0005661726253782145, + "d_pool_score": 0.0005473705393115235, + "d_ape": 7.236730944585608e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.72604751586914, + 38.74604797363281, + 38.78224182128906, + 38.78335952758789, + 38.72016143798828, + 38.75788879394531, + 38.730464935302734 + ], + "critical_prepare_samples_ms": [ + 47.61352890636772, + 45.467405929230154, + 44.49985595420003, + 152.91648893617094, + 45.98363093100488, + 45.106612145900726, + 44.917382998391986, + 145.95098292920738 + ] + }, + "planner": { + "step_ms": 33.0742073059082, + "prepare_ms": 28.312876995187253, + "one_use_ms": 61.387084301095456, + "prepare_min_ms": 28.01844000350684, + "prepare_max_ms": 123.6400690395385, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0020962956426669433, + "dq": 0.0006347467076768539, + "dkv": 0.0005276886282440079, + "d_index_q": 0.00035682941768751356, + "d_weights": 0.00032945679607176856, + "d_pool_kv": 0.0005648985794352423, + "d_pool_score": 0.0005625030405139739, + "d_ape": 6.634011369785548e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 33.05827331542969, + 33.0742073059082, + 33.59049606323242, + 33.06959915161133, + 33.093441009521484, + 33.08927917480469, + 33.03887939453125 + ], + "critical_prepare_samples_ms": [ + 28.298633988015354, + 28.01844000350684, + 123.6400690395385, + 28.612435911782086, + 28.067155042663217, + 28.327120002359152, + 28.597721946425736, + 28.228612965904176 + ] + } + } + }, + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "1ce20ae25f95e468a8ceb74f0607b96cd4b3c96afba3f565edacffb45121c460", + "policies": { + "contiguous": { + "step_ms": 32.37452697753906, + "prepare_ms": 7.578457531053573, + "one_use_ms": 39.952984508592635, + "prepare_min_ms": 7.474332000128925, + "prepare_max_ms": 12.586126918904483, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015154178483681854, + "dq": 0.0004590856367981372, + "dkv": 0.00046865559368039917, + "d_index_q": 0.0003544140670990448, + "d_weights": 0.0003176278618649191, + "d_pool_kv": 0.0005387196908820256, + "d_pool_score": 0.0005430168708674836, + "d_ape": 4.7473252849530434e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.37452697753906, + 32.392513275146484, + 32.36988830566406, + 32.39535903930664, + 32.35891342163086, + 32.35808181762695, + 32.382911682128906 + ], + "critical_prepare_samples_ms": [ + 7.727506919763982, + 7.595576927997172, + 7.546272943727672, + 12.586126918904483, + 7.49861600343138, + 7.561338134109974, + 7.474332000128925, + 7.61300593148917 + ] + }, + "zigzag": { + "step_ms": 37.90224075317383, + "prepare_ms": 45.84232851630077, + "one_use_ms": 83.7445692694746, + "prepare_min_ms": 44.70354691147804, + "prepare_max_ms": 155.66171507816762, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016252246640432245, + "dq": 0.0004916736119971625, + "dkv": 0.0004861918573380407, + "d_index_q": 0.0003569257876233377, + "d_weights": 0.0003177650459406723, + "d_pool_kv": 0.0005486278732828732, + "d_pool_score": 0.0005445220484194565, + "d_ape": 3.860767227143574e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 37.866241455078125, + 37.90224075317383, + 37.91574478149414, + 37.9218864440918, + 37.847808837890625, + 37.96307373046875, + 37.85862350463867 + ], + "critical_prepare_samples_ms": [ + 47.00984014198184, + 45.00916204415262, + 44.70354691147804, + 153.49186910316348, + 46.21615505311638, + 45.468501979485154, + 44.9322130298242, + 155.66171507816762 + ] + }, + "planner": { + "step_ms": 32.353633880615234, + "prepare_ms": 12.12866249261424, + "one_use_ms": 44.482296373229474, + "prepare_min_ms": 11.894998839125037, + "prepare_max_ms": 123.44329594634473, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015138876329661513, + "dq": 0.00045860825477340696, + "dkv": 0.00046849135874668684, + "d_index_q": 0.0003562482112602212, + "d_weights": 0.00032118335948173643, + "d_pool_kv": 0.0005438274579043697, + "d_pool_score": 0.0005519648229923422, + "d_ape": 4.258189850384637e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.353633880615234, + 32.412353515625, + 32.39923095703125, + 32.37456130981445, + 32.34396743774414, + 32.31990432739258, + 32.341121673583984 + ], + "critical_prepare_samples_ms": [ + 12.077874038368464, + 12.179450946860015, + 123.44329594634473, + 12.385182897560298, + 11.991281062364578, + 12.406638008542359, + 11.894998839125037, + 11.912805959582329 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "27e077b4ff55aa00c71a035377fafcb97d9db8bfb198c213673c877e5ded916b", + "policies": { + "contiguous": { + "step_ms": 115.2938232421875, + "prepare_ms": 1.1161149595864117, + "one_use_ms": 116.40993820177391, + "prepare_min_ms": 1.087293028831482, + "prepare_max_ms": 1.304103061556816, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014842079275755727, + "dq": 0.0005035188023387811, + "dkv": 0.0010278174526612903, + "d_index_q": 0.0006256382167037119, + "d_weights": 0.000563762569940494, + "d_pool_kv": 0.0013303364084540658, + "d_pool_score": 0.001377639655655214, + "d_ape": 2.0452408219956506e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 114.4939193725586, + 115.84454345703125, + 115.2938232421875, + 114.8194580078125, + 116.123779296875, + 114.58399963378906, + 115.50758361816406 + ], + "critical_prepare_samples_ms": [ + 1.304103061556816, + 1.1089229956269264, + 1.0957310441881418, + 1.1007870780304074, + 1.1440730886533856, + 1.123306923545897, + 1.087293028831482, + 1.187960966490209 + ] + }, + "zigzag": { + "step_ms": 105.58483123779297, + "prepare_ms": 1.1357140028849244, + "one_use_ms": 106.7205452406779, + "prepare_min_ms": 1.1315711308270693, + "prepare_max_ms": 2.692296984605491, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014841542205465054, + "dq": 0.00050380319214946, + "dkv": 0.0010287937173898806, + "d_index_q": 0.0006482205309808245, + "d_weights": 0.000583151648179517, + "d_pool_kv": 0.0013331964396419756, + "d_pool_score": 0.0013779578045436441, + "d_ape": 1.9963974118421826e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 106.04412841796875, + 105.58483123779297, + 105.72246551513672, + 105.1151351928711, + 105.20396423339844, + 106.02706909179688, + 105.4378890991211 + ], + "critical_prepare_samples_ms": [ + 2.692296984605491, + 1.1655869893729687, + 1.1315711308270693, + 1.1326848762109876, + 1.1354410089552402, + 1.15663290489465, + 1.1359869968146086, + 1.1334370356053114 + ] + }, + "planner": { + "step_ms": 104.14985656738281, + "prepare_ms": 7.791060954332352, + "one_use_ms": 111.94091752171516, + "prepare_min_ms": 7.6777610229328275, + "prepare_max_ms": 8.784707984887064, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014818148634560368, + "dq": 0.0005030012855905718, + "dkv": 0.0010310600226582155, + "d_index_q": 0.0007705202387904052, + "d_weights": 0.0006243137936169513, + "d_pool_kv": 0.0013332458370744855, + "d_pool_score": 0.0013751792972335336, + "d_ape": 2.096916046719033e-05 + }, + "max_loss_abs_diff": 2.0372681319713593e-10, + "critical_step_samples_ms": [ + 103.95587158203125, + 104.33695983886719, + 103.87484741210938, + 104.14985656738281, + 104.16598510742188, + 104.65894317626953, + 104.02249908447266 + ], + "critical_prepare_samples_ms": [ + 7.850671885535121, + 8.708197972737253, + 7.70237494725734, + 7.6777610229328275, + 8.784707984887064, + 7.718592998571694, + 7.731450023129582, + 8.389059104956686 + ] + } + } + }, + "256k-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "b206d1914825ce27ceb8f280beaa9efb8b8c2d6e09e8192df5fe030a36b95f42", + "policies": { + "contiguous": { + "step_ms": 50.3370246887207, + "prepare_ms": 1.0602579568512738, + "one_use_ms": 51.39728264557198, + "prepare_min_ms": 1.0142670944333076, + "prepare_max_ms": 1.16502505261451, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008995063625665686, + "dq": 0.000417439995711948, + "dkv": 0.00029408694183826416, + "d_index_q": 0.00010925775240076204, + "d_weights": 9.570165139064877e-05, + "d_pool_kv": 0.0002036231762018183, + "d_pool_score": 0.00023355335680617782, + "d_ape": 7.1891911871349595e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.318817138671875, + 50.3370246887207, + 50.29235076904297, + 50.279136657714844, + 50.51945495605469, + 50.36774444580078, + 50.34364700317383 + ], + "critical_prepare_samples_ms": [ + 1.16502505261451, + 1.0669470066204667, + 1.0217679664492607, + 1.0694119846448302, + 1.0273349471390247, + 1.0535689070820808, + 1.0142670944333076, + 1.0685940505936742 + ] + }, + "zigzag": { + "step_ms": 51.72915267944336, + "prepare_ms": 4.144389531575143, + "one_use_ms": 55.8735422110185, + "prepare_min_ms": 4.095686017535627, + "prepare_max_ms": 4.8910570330917835, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009047350578015278, + "dq": 0.00041966050461122436, + "dkv": 0.00029402900005215225, + "d_index_q": 0.00010933438743714884, + "d_weights": 9.557770476272342e-05, + "d_pool_kv": 0.0001933948013080206, + "d_pool_score": 0.0001997392626168786, + "d_ape": 6.032497424065086e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 51.72719955444336, + 51.7239990234375, + 51.834590911865234, + 51.799041748046875, + 51.7083854675293, + 51.72915267944336, + 51.82553482055664 + ], + "critical_prepare_samples_ms": [ + 4.361170926131308, + 4.214552929624915, + 4.100008052773774, + 4.132551024667919, + 4.152706009335816, + 4.095686017535627, + 4.8910570330917835, + 4.136073053814471 + ] + }, + "planner": { + "step_ms": 50.334686279296875, + "prepare_ms": 1.275484450161457, + "one_use_ms": 51.61017072945833, + "prepare_min_ms": 1.2338400119915605, + "prepare_max_ms": 1.3704260345548391, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0008968787811652317, + "dq": 0.00041599997551765766, + "dkv": 0.0002978264449580809, + "d_index_q": 0.00010914478084630114, + "d_weights": 9.54750575422348e-05, + "d_pool_kv": 0.00019495316721958097, + "d_pool_score": 0.00020016815004170784, + "d_ape": 6.878236502860825e-06 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 50.27433776855469, + 50.653377532958984, + 50.30992126464844, + 50.319297790527344, + 50.40998458862305, + 50.47235107421875, + 50.334686279296875 + ], + "critical_prepare_samples_ms": [ + 1.3704260345548391, + 1.2794609647244215, + 1.2759730452671647, + 1.2749958550557494, + 1.2793471105396748, + 1.2338400119915605, + 1.27033784519881, + 1.248182961717248 + ] + } + } + }, + "256k-near-uniform-64": { + "documents": 64, + "tokens": 262144, + "source_record_sha256": "ccc7f36371b2ca093244df96efefd898c4c05ac4599ade1e2027012193b8ea43", + "policies": { + "contiguous": { + "step_ms": 50.341758728027344, + "prepare_ms": 1.0578265646472573, + "one_use_ms": 51.3995852926746, + "prepare_min_ms": 1.0304378811269999, + "prepare_max_ms": 2.42595502641052, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009040294318661402, + "dq": 0.0004176309327453784, + "dkv": 0.00028700438317877033, + "d_index_q": 0.00010909463263037384, + "d_weights": 9.385798875582676e-05, + "d_pool_kv": 0.0002099997308788692, + "d_pool_score": 0.00021174261363919708, + "d_ape": 6.975002356106306e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 50.3015022277832, + 50.3072624206543, + 50.37491226196289, + 50.39254379272461, + 50.31679916381836, + 50.34892654418945, + 50.341758728027344 + ], + "critical_prepare_samples_ms": [ + 2.42595502641052, + 1.056926092132926, + 1.0342529276385903, + 1.0686619207262993, + 1.0407939553260803, + 1.0587270371615887, + 1.0304378811269999, + 1.0622970294207335 + ] + }, + "zigzag": { + "step_ms": 51.770111083984375, + "prepare_ms": 4.253186983987689, + "one_use_ms": 56.023298067972064, + "prepare_min_ms": 4.211014020256698, + "prepare_max_ms": 4.994260962121189, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0009021376190146195, + "dq": 0.00042020065818351493, + "dkv": 0.0003025562304355097, + "d_index_q": 0.00010794658869771671, + "d_weights": 9.589076007313737e-05, + "d_pool_kv": 0.00021834571632293082, + "d_pool_score": 0.00021545790675349828, + "d_ape": 7.255162179958939e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 51.72880172729492, + 51.770111083984375, + 51.73446273803711, + 51.797088623046875, + 51.87964630126953, + 51.770816802978516, + 51.68940734863281 + ], + "critical_prepare_samples_ms": [ + 4.418599070049822, + 4.292018827982247, + 4.211014020256698, + 4.258326953276992, + 4.248047014698386, + 4.212184925563633, + 4.994260962121189, + 4.242594935931265 + ] + }, + "planner": { + "step_ms": 50.34764862060547, + "prepare_ms": 1.2725520064122975, + "one_use_ms": 51.620200627017766, + "prepare_min_ms": 1.2156248558312654, + "prepare_max_ms": 1.4591740909963846, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009001018368343759, + "dq": 0.0004180679445311549, + "dkv": 0.0002908204232464405, + "d_index_q": 0.00010824020032508636, + "d_weights": 9.559161306044745e-05, + "d_pool_kv": 0.0002088096055712727, + "d_pool_score": 0.00022862128266242155, + "d_ape": 7.098986447802436e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 50.308414459228516, + 50.31196975708008, + 50.657249450683594, + 50.34764862060547, + 50.37120056152344, + 50.28732681274414, + 50.731998443603516 + ], + "critical_prepare_samples_ms": [ + 1.3486190000548959, + 1.4591740909963846, + 1.2668580748140812, + 1.2407059548422694, + 1.282125012949109, + 1.2156248558312654, + 1.2782459380105138, + 1.236111973412335 + ] + } + } + }, + "256k-mixed-22": { + "documents": 22, + "tokens": 262144, + "source_record_sha256": "db3e840999278944a7dc0c815b5832e7cf601e9dcaa7db9db4627377f326570b", + "policies": { + "contiguous": { + "step_ms": 90.6918716430664, + "prepare_ms": 1.150823023635894, + "one_use_ms": 91.8426946667023, + "prepare_min_ms": 1.115339924581349, + "prepare_max_ms": 1.3260958949103951, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014396839839641682, + "dq": 0.0004912932869663864, + "dkv": 0.000782527660447053, + "d_index_q": 0.0007096421066149192, + "d_weights": 0.0004776414729478394, + "d_pool_kv": 0.001229747392195446, + "d_pool_score": 0.0012867899272477345, + "d_ape": 1.346208842230701e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 91.04662322998047, + 90.35794830322266, + 90.62847900390625, + 90.47663879394531, + 92.35906982421875, + 90.6918716430664, + 91.08707427978516 + ], + "critical_prepare_samples_ms": [ + 1.3260958949103951, + 1.1735139414668083, + 1.1378390481695533, + 1.1485170107334852, + 1.115339924581349, + 1.1682850308716297, + 1.1497350642457604, + 1.1519109830260277 + ] + }, + "zigzag": { + "step_ms": 87.94976043701172, + "prepare_ms": 2.339682076126337, + "one_use_ms": 90.28944251313806, + "prepare_min_ms": 2.3260059533640742, + "prepare_max_ms": 5.5571189150214195, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014655759967181566, + "dq": 0.0005149950687110133, + "dkv": 0.0007858004056596082, + "d_index_q": 0.0007967333850337871, + "d_weights": 0.0006135588148793877, + "d_pool_kv": 0.0012313505882381754, + "d_pool_score": 0.0012816470119322591, + "d_ape": 1.3536862404843282e-05 + }, + "max_loss_abs_diff": 5.238689482212067e-10, + "critical_step_samples_ms": [ + 87.89424133300781, + 87.9170913696289, + 88.21635437011719, + 87.94976043701172, + 87.63407897949219, + 88.22838592529297, + 88.08528137207031 + ], + "critical_prepare_samples_ms": [ + 3.682921058498323, + 5.5571189150214195, + 2.3260059533640742, + 2.33569392003119, + 2.3404060630127788, + 2.3389580892398953, + 2.3376650642603636, + 2.347580040805042 + ] + }, + "planner": { + "step_ms": 87.38617706298828, + "prepare_ms": 8.772762026637793, + "one_use_ms": 96.15893908962607, + "prepare_min_ms": 8.418329060077667, + "prepare_max_ms": 9.713605977594852, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0014719262651841122, + "dq": 0.0005169495137705205, + "dkv": 0.0005843813325958284, + "d_index_q": 0.000568126948707376, + "d_weights": 0.0005875102674471812, + "d_pool_kv": 0.0006705463099119529, + "d_pool_score": 0.0006839089222165456, + "d_ape": 1.3325801383008022e-05 + }, + "max_loss_abs_diff": 4.0745362639427185e-10, + "critical_step_samples_ms": [ + 87.38617706298828, + 87.18109130859375, + 87.44198608398438, + 87.81423950195312, + 87.28131103515625, + 87.66595458984375, + 87.18303680419922 + ], + "critical_prepare_samples_ms": [ + 8.892927085980773, + 8.418329060077667, + 9.39794396981597, + 8.462049067020416, + 9.713605977594852, + 8.53787490632385, + 9.19189490377903, + 8.652596967294812 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "a3e04775cd4674a149c44452e86572771d80fd867f99c915f4d8703ea7cf55c1", + "policies": { + "contiguous": { + "step_ms": 18.96121597290039, + "prepare_ms": 6.907838920596987, + "one_use_ms": 25.869054893497378, + "prepare_min_ms": 6.844045943580568, + "prepare_max_ms": 7.187637034803629, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.724041124260225e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.511801279303253e-05, + "d_pool_score": 5.232610864426897e-05, + "d_ape": 1.4967124181438183e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 19.019712448120117, + 18.97760009765625, + 18.96121597290039, + 18.9564151763916, + 18.95644760131836, + 18.9434871673584, + 19.144479751586914 + ], + "critical_prepare_samples_ms": [ + 7.187637034803629, + 6.97691401001066, + 6.844045943580568, + 6.993726012296975, + 6.876001018099487, + 6.887514842674136, + 6.885394104756415, + 6.928162998519838 + ] + }, + "zigzag": { + "step_ms": 22.557504653930664, + "prepare_ms": 46.86611460056156, + "one_use_ms": 69.42361925449222, + "prepare_min_ms": 46.30116908811033, + "prepare_max_ms": 167.44421899784356, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8637480027866625e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.271880161331187e-05, + "d_pool_score": 5.610531234471807e-05, + "d_ape": 1.5175327931420397e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 22.557504653930664, + 22.60767936706543, + 22.548383712768555, + 22.567264556884766, + 22.551103591918945, + 22.51372718811035, + 22.592575073242188 + ], + "critical_prepare_samples_ms": [ + 47.66656889114529, + 46.35808803141117, + 46.458124998025596, + 167.44421899784356, + 47.36804007552564, + 47.025946085341275, + 46.30116908811033, + 46.706283115781844 + ] + }, + "planner": { + "step_ms": 18.94291114807129, + "prepare_ms": 8.967155939899385, + "one_use_ms": 27.910067087970674, + "prepare_min_ms": 8.858141023665667, + "prepare_max_ms": 14.381637098267674, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.590593616004352e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.534328689387801e-05, + "d_pool_score": 5.3623912772759155e-05, + "d_ape": 1.7238388124362549e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 18.938079833984375, + 18.94291114807129, + 18.93289566040039, + 18.952415466308594, + 18.924928665161133, + 18.946847915649414, + 19.089344024658203 + ], + "critical_prepare_samples_ms": [ + 9.043016005307436, + 8.858141023665667, + 8.99953895714134, + 8.93477292265743, + 9.00569697842002, + 8.906646049581468, + 8.866342948749661, + 14.381637098267674 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "3abdfc48993cc3747d2f08fcfbd801e8c6fa1abaa8a607d585da37c318d9b529", + "policies": { + "contiguous": { + "step_ms": 21.785919189453125, + "prepare_ms": 3.9414139464497566, + "one_use_ms": 25.72733313590288, + "prepare_min_ms": 3.8159850519150496, + "prepare_max_ms": 4.042547079734504, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.297053926215977e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.412638872669938e-05, + "d_pool_score": 4.495461915173074e-05, + "d_ape": 1.4301480341692518e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.769792556762695, + 21.76304054260254, + 22.114463806152344, + 21.785919189453125, + 21.940128326416016, + 21.767871856689453, + 21.940895080566406 + ], + "critical_prepare_samples_ms": [ + 4.042547079734504, + 3.9293068693950772, + 3.8159850519150496, + 3.9682361530140042, + 3.8398168981075287, + 3.9214659482240677, + 3.953521023504436, + 3.995029954239726 + ] + }, + "zigzag": { + "step_ms": 24.209856033325195, + "prepare_ms": 24.67734646052122, + "one_use_ms": 48.88720249384642, + "prepare_min_ms": 24.372468004003167, + "prepare_max_ms": 142.0474809128791, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2708240134177204e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.3500598722775094e-05, + "d_pool_score": 5.347177795761659e-05, + "d_ape": 1.327084474519006e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 24.196128845214844, + 24.229408264160156, + 24.214399337768555, + 24.209856033325195, + 24.24527931213379, + 24.169696807861328, + 24.205184936523438 + ], + "critical_prepare_samples_ms": [ + 26.387990918010473, + 24.77585698943585, + 24.372468004003167, + 24.47995904367417, + 24.4268769165501, + 24.750579963438213, + 24.60411295760423, + 142.0474809128791 + ] + }, + "planner": { + "step_ms": 21.7589111328125, + "prepare_ms": 5.023348960094154, + "one_use_ms": 26.782260092906654, + "prepare_min_ms": 4.9473929684609175, + "prepare_max_ms": 5.072213942185044, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.554213974193077e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.1905810341815174e-05, + "d_pool_score": 4.350734098547027e-05, + "d_ape": 1.4490775323448985e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 21.780927658081055, + 21.7589111328125, + 21.778976440429688, + 21.859455108642578, + 21.743295669555664, + 21.751680374145508, + 21.750688552856445 + ], + "critical_prepare_samples_ms": [ + 5.0161018734797835, + 5.0161059480160475, + 5.0472968723624945, + 5.041101947426796, + 4.989685956388712, + 5.03059197217226, + 5.072213942185044, + 4.9473929684609175 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "299ffffd41e8b2e4c2be973c6bdb05ae54dc4a991bc3051df8ee4e0a6127615f", + "policies": { + "contiguous": { + "step_ms": 83.6385269165039, + "prepare_ms": 9.585648542270064, + "one_use_ms": 93.22417545877397, + "prepare_min_ms": 9.470761171542108, + "prepare_max_ms": 109.36465603299439, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017248119636322726, + "dq": 0.0005231477545538955, + "dkv": 0.0008859578942512667, + "d_index_q": 0.0003679247450818875, + "d_weights": 0.0003583038720109353, + "d_pool_kv": 0.0013756446763457486, + "d_pool_score": 0.0013569902061103135, + "d_ape": 9.213348057789348e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 83.79923248291016, + 83.6385269165039, + 83.57939147949219, + 83.14595031738281, + 84.14755249023438, + 83.68710327148438, + 83.60105895996094 + ], + "critical_prepare_samples_ms": [ + 11.086768005043268, + 9.873543865978718, + 9.49363096151501, + 9.626674116589129, + 9.470761171542108, + 9.488550946116447, + 109.36465603299439, + 9.544622967951 + ] + }, + "zigzag": { + "step_ms": 96.04755401611328, + "prepare_ms": 92.28663396788761, + "one_use_ms": 188.3341879840009, + "prepare_min_ms": 46.753630973398685, + "prepare_max_ms": 166.36592405848205, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017395277068004997, + "dq": 0.0005277805775999957, + "dkv": 0.0006782649359844867, + "d_index_q": 0.0003691170318294211, + "d_weights": 0.0003624685324509999, + "d_pool_kv": 0.0007577977882457726, + "d_pool_score": 0.0007313251119719116, + "d_ape": 8.559385750338412e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 96.06777954101562, + 96.25202941894531, + 96.04755401611328, + 95.81129455566406, + 95.63484954833984, + 96.13632202148438, + 95.58025360107422 + ], + "critical_prepare_samples_ms": [ + 48.80867002066225, + 166.36592405848205, + 149.53875401988626, + 48.5578749794513, + 46.753630973398685, + 135.76459791511297, + 152.24266005679965, + 47.11186105851084 + ] + }, + "planner": { + "step_ms": 82.14995574951172, + "prepare_ms": 32.668465457390994, + "one_use_ms": 114.81842120690271, + "prepare_min_ms": 32.24971203599125, + "prepare_max_ms": 33.218687982298434, + "selected_policy": "selection-local-5x", + "max_relative_rms": { + "output": 0.0017299188701990226, + "dq": 0.0005249688471103487, + "dkv": 0.0006932649020557219, + "d_index_q": 0.0003879256230062098, + "d_weights": 0.0003670382435938008, + "d_pool_kv": 0.0013198355292136361, + "d_pool_score": 0.0013151740439955597, + "d_ape": 8.808128280757483e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 81.99295806884766, + 82.14995574951172, + 82.31292724609375, + 82.3354263305664, + 81.54524993896484, + 82.6478042602539, + 81.62854766845703 + ], + "critical_prepare_samples_ms": [ + 32.491752062924206, + 32.684653997421265, + 32.89240214508027, + 32.540695974603295, + 32.65227691736072, + 33.04584801662713, + 32.24971203599125, + 33.218687982298434 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "f6f2fb90fa19746799d7f723f65a57a1d55b4e768a2fbaff20ba8977a77dfb5e", + "policies": { + "contiguous": { + "step_ms": 99.36064147949219, + "prepare_ms": 9.696168475784361, + "one_use_ms": 109.05680995527655, + "prepare_min_ms": 9.520240011624992, + "prepare_max_ms": 10.234966059215367, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010563700585577058, + "dq": 0.00035831076606808053, + "dkv": 0.0004733622916756713, + "d_index_q": 0.0005939123996360771, + "d_weights": 0.0005024215185567405, + "d_pool_kv": 0.000752053852842638, + "d_pool_score": 0.0007688502326254845, + "d_ape": 6.434388548257363e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 98.40025329589844, + 99.20460510253906, + 100.11779022216797, + 99.36064147949219, + 99.45503997802734, + 98.65779113769531, + 99.67919921875 + ], + "critical_prepare_samples_ms": [ + 9.97902499511838, + 9.690101025626063, + 9.577112039551139, + 9.55722399521619, + 9.520240011624992, + 9.813543991185725, + 10.234966059215367, + 9.70223592594266 + ] + }, + "zigzag": { + "step_ms": 115.25299072265625, + "prepare_ms": 147.0409840112552, + "one_use_ms": 262.29397473391145, + "prepare_min_ms": 47.70176007878035, + "prepare_max_ms": 167.99743298906833, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001096030426138749, + "dq": 0.00036871769863560327, + "dkv": 0.0005522801000187844, + "d_index_q": 0.000646529994069314, + "d_weights": 0.0005200898687794669, + "d_pool_kv": 0.0007782778462285144, + "d_pool_score": 0.0007902708796079504, + "d_ape": 6.243475107842311e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 115.171875, + 115.26223754882812, + 115.35302734375, + 115.21993255615234, + 115.44630432128906, + 115.25299072265625, + 115.22217559814453 + ], + "critical_prepare_samples_ms": [ + 137.1572030475363, + 154.45262903813273, + 167.99743298906833, + 48.344484996050596, + 47.70176007878035, + 143.54554004967213, + 155.44524206779897, + 150.53642797283828 + ] + }, + "planner": { + "step_ms": 92.2047348022461, + "prepare_ms": 31.8146159988828, + "one_use_ms": 124.0193508011289, + "prepare_min_ms": 31.22708504088223, + "prepare_max_ms": 132.93269090354443, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.001048036358338339, + "dq": 0.00036006689300403495, + "dkv": 0.0004952787980932682, + "d_index_q": 0.0005006808620709585, + "d_weights": 0.00047204477185006934, + "d_pool_kv": 0.0006647742697551269, + "d_pool_score": 0.0006600247570603834, + "d_ape": 6.129309354351373e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 91.95887756347656, + 92.06761932373047, + 91.93360137939453, + 92.53968048095703, + 92.28575897216797, + 92.2804183959961, + 92.2047348022461 + ], + "critical_prepare_samples_ms": [ + 31.714363023638725, + 126.031952095218, + 31.22708504088223, + 31.914868974126875, + 31.32477297913283, + 31.713226926513016, + 132.93269090354443, + 32.04708802513778 + ] + } + } + }, + "1024k-single": { + "documents": 1, + "tokens": 1048576, + "source_record_sha256": "f8f59d7261da96dacfa2bc6680f2e607ee1272751d31448f68672deee7924dcc", + "policies": { + "contiguous": { + "step_ms": 853.462646484375, + "prepare_ms": 2.235381514765322, + "one_use_ms": 855.6980279991403, + "prepare_min_ms": 2.180504030548036, + "prepare_max_ms": 2.6538941310718656, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011390925500643692, + "dq": 0.00036677751548457586, + "dkv": 0.00045029686879957095, + "d_index_q": 0.0005475285484143609, + "d_weights": 0.0005234083334124328, + "d_pool_kv": 0.0006765433793383455, + "d_pool_score": 0.000669507534555031, + "d_ape": 1.201251494769134e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 848.7756958007812, + 851.5589599609375, + 854.1879272460938, + 853.2410888671875, + 853.76123046875, + 853.462646484375, + 854.3615112304688 + ], + "critical_prepare_samples_ms": [ + 2.6538941310718656, + 2.25232494994998, + 2.2514740703627467, + 2.2216499783098698, + 2.2194701014086604, + 2.229652018286288, + 2.180504030548036, + 2.2411110112443566 + ] + }, + "zigzag": { + "step_ms": 638.503173828125, + "prepare_ms": 2.2861745092086494, + "one_use_ms": 640.7893483373336, + "prepare_min_ms": 2.2653070045635104, + "prepare_max_ms": 5.670589045621455, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011415486958252505, + "dq": 0.00036769420295685456, + "dkv": 0.0004914265330137578, + "d_index_q": 0.000556942590103184, + "d_weights": 0.000536722297153423, + "d_pool_kv": 0.0006794240688639852, + "d_pool_score": 0.0006762446273154329, + "d_ape": 1.2332330725673362e-05 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 638.0089721679688, + 638.694091796875, + 640.7294921875, + 635.8269653320312, + 638.503173828125, + 637.2936401367188, + 639.0613403320312 + ], + "critical_prepare_samples_ms": [ + 5.670589045621455, + 2.431723987683654, + 2.2653070045635104, + 2.296129008755088, + 2.2838700097054243, + 2.2884790087118745, + 2.2718310356140137, + 2.270005992613733 + ] + }, + "planner": { + "step_ms": 637.5537719726562, + "prepare_ms": 9.320752928033471, + "one_use_ms": 646.8745249006897, + "prepare_min_ms": 8.959550876170397, + "prepare_max_ms": 10.955121018923819, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0011399811891549397, + "dq": 0.0003671549652483216, + "dkv": 0.0004688434719635236, + "d_index_q": 0.00054714713668071, + "d_weights": 0.0005235608730508459, + "d_pool_kv": 0.0007342648722923458, + "d_pool_score": 0.0007281192925473291, + "d_ape": 1.2245290306330182e-05 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 637.5820922851562, + 637.1992797851562, + 637.2449951171875, + 637.8410034179688, + 636.2861938476562, + 637.5537719726562, + 641.8402709960938 + ], + "critical_prepare_samples_ms": [ + 10.955121018923819, + 10.685780085623264, + 9.343747980892658, + 9.170369943603873, + 9.297757875174284, + 8.959550876170397, + 9.293015929870307, + 9.915986098349094 + ] + } + } + }, + "1024k-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "467788be0454c5eb43ec073cd768122a5221cca5675c0bb30ce968068c6245a9", + "policies": { + "contiguous": { + "step_ms": 239.31231689453125, + "prepare_ms": 1.490010938141495, + "one_use_ms": 240.80232783267274, + "prepare_min_ms": 1.4407910639420152, + "prepare_max_ms": 1.7354709561914206, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015971718647385203, + "dq": 0.0005946386006291191, + "dkv": 0.0004893087559395691, + "d_index_q": 0.0002655780560894656, + "d_weights": 0.00020677860634936432, + "d_pool_kv": 0.0002909984083585693, + "d_pool_score": 0.00030601339199424316, + "d_ape": 6.1058968008983315e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 239.48367309570312, + 238.9126739501953, + 239.31820678710938, + 239.31231689453125, + 239.07810974121094, + 239.46946716308594, + 238.660888671875 + ], + "critical_prepare_samples_ms": [ + 1.7354709561914206, + 1.5307000139728189, + 1.4793298905715346, + 1.4931149780750275, + 1.4869068982079625, + 1.4774100854992867, + 1.4407910639420152, + 1.4997579855844378 + ] + }, + "zigzag": { + "step_ms": 244.19993591308594, + "prepare_ms": 4.6627100091427565, + "one_use_ms": 248.8626459222287, + "prepare_min_ms": 4.619401996023953, + "prepare_max_ms": 6.727989995852113, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.00161892792378639, + "dq": 0.00061183109697404, + "dkv": 0.0005255603590014857, + "d_index_q": 0.00027796173921266023, + "d_weights": 0.00021531215731985807, + "d_pool_kv": 0.00028469963192253694, + "d_pool_score": 0.00029151841170628354, + "d_ape": 6.315599964674472e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 244.90960693359375, + 243.77923583984375, + 245.494140625, + 244.19993591308594, + 243.84393310546875, + 244.16079711914062, + 244.61228942871094 + ], + "critical_prepare_samples_ms": [ + 6.727989995852113, + 4.855223116464913, + 4.65498398989439, + 4.665578017011285, + 4.734496003948152, + 4.624465946108103, + 4.659842001274228, + 4.619401996023953 + ] + }, + "planner": { + "step_ms": 239.0471649169922, + "prepare_ms": 1.7240439774468541, + "one_use_ms": 240.77120889443904, + "prepare_min_ms": 1.6765620093792677, + "prepare_max_ms": 1.8191479612141848, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.001606236638194518, + "dq": 0.000603183325778411, + "dkv": 0.0005147282440500237, + "d_index_q": 0.00028285470187259566, + "d_weights": 0.00021272988166387048, + "d_pool_kv": 0.00029001033928657117, + "d_pool_score": 0.00028677249205629184, + "d_ape": 6.258893265663241e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 238.40419006347656, + 239.01718139648438, + 238.46278381347656, + 239.06573486328125, + 239.3005828857422, + 239.5536346435547, + 239.0471649169922 + ], + "critical_prepare_samples_ms": [ + 1.8191479612141848, + 1.7172510270029306, + 1.7308369278907776, + 1.682960893958807, + 1.7325289081782103, + 1.730848103761673, + 1.6999311046674848, + 1.6765620093792677 + ] + } + } + }, + "1024k-near-uniform-64": { + "documents": 64, + "tokens": 1048576, + "source_record_sha256": "ce3857fde76c10e837d4359215986ae8706ddba02140df49bb2b6014276e94fb", + "policies": { + "contiguous": { + "step_ms": 239.05783081054688, + "prepare_ms": 1.514466479420662, + "one_use_ms": 240.57229728996754, + "prepare_min_ms": 1.4685369096696377, + "prepare_max_ms": 1.6245651058852673, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0016036861711426922, + "dq": 0.0006002606451464885, + "dkv": 0.0005078364704675495, + "d_index_q": 0.00027393963055896855, + "d_weights": 0.00021436935672360165, + "d_pool_kv": 0.00029689000244164466, + "d_pool_score": 0.0002755855279061583, + "d_ape": 6.5191501926423906e-06 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 238.77810668945312, + 239.35433959960938, + 239.67471313476562, + 238.6045684814453, + 239.44578552246094, + 238.9220428466797, + 239.05783081054688 + ], + "critical_prepare_samples_ms": [ + 1.6245651058852673, + 1.5482361195608974, + 1.4911219477653503, + 1.5080280136317015, + 1.4883260009810328, + 1.5209049452096224, + 1.4685369096696377, + 1.5511849196627736 + ] + }, + "zigzag": { + "step_ms": 244.37164306640625, + "prepare_ms": 4.788588033989072, + "one_use_ms": 249.16023110039532, + "prepare_min_ms": 4.718890995718539, + "prepare_max_ms": 6.958352983929217, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016149985147692735, + "dq": 0.0006012411018216009, + "dkv": 0.0004889619138367483, + "d_index_q": 0.00027384235416954054, + "d_weights": 0.00021508569090086002, + "d_pool_kv": 0.00028208163900576497, + "d_pool_score": 0.00029433482150776626, + "d_ape": 6.324001411904957e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 244.35714721679688, + 243.5417022705078, + 244.73663330078125, + 244.49600219726562, + 244.37164306640625, + 244.37135314941406, + 244.45872497558594 + ], + "critical_prepare_samples_ms": [ + 6.958352983929217, + 4.946545930579305, + 4.794639069586992, + 4.782536998391151, + 5.79125399235636, + 4.723230958916247, + 4.718890995718539, + 4.728949978016317 + ] + }, + "planner": { + "step_ms": 239.24879455566406, + "prepare_ms": 1.75143702654168, + "one_use_ms": 241.00023158220574, + "prepare_min_ms": 1.686341012828052, + "prepare_max_ms": 1.818434102460742, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015996148111491304, + "dq": 0.0005959255331279302, + "dkv": 0.00048740660403977806, + "d_index_q": 0.00025509571834305165, + "d_weights": 0.00019903247405294528, + "d_pool_kv": 0.00029018364601671756, + "d_pool_score": 0.0002817420290576312, + "d_ape": 5.9639498577661876e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 238.96421813964844, + 238.8380126953125, + 239.6174774169922, + 238.64874267578125, + 239.3194580078125, + 239.24879455566406, + 239.29000854492188 + ], + "critical_prepare_samples_ms": [ + 1.818434102460742, + 1.7426980193704367, + 1.7638478893786669, + 1.7180261202156544, + 1.7601760337129235, + 1.686341012828052, + 1.7822969239205122, + 1.6957151237875223 + ] + } + } + }, + "1024k-mixed-22": { + "documents": 22, + "tokens": 1048576, + "source_record_sha256": "94a3d9ad0055a320defc6d8a0f7147cda48f2bec810ce85de4e405b8c33cfeb9", + "policies": { + "contiguous": { + "step_ms": 576.2921752929688, + "prepare_ms": 2.234563056845218, + "one_use_ms": 578.526738349814, + "prepare_min_ms": 2.176858950406313, + "prepare_max_ms": 2.457708935253322, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0011241890551148522, + "dq": 0.0003703792465013327, + "dkv": 0.0005637775340177526, + "d_index_q": 0.0004923585316969156, + "d_weights": 0.0004762473505531833, + "d_pool_kv": 0.0007050138286532109, + "d_pool_score": 0.000693366416930018, + "d_ape": 1.0040782845648817e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 575.5206909179688, + 579.4055786132812, + 576.2921752929688, + 575.3203125, + 579.8231811523438, + 575.8522338867188, + 578.5023803710938 + ], + "critical_prepare_samples_ms": [ + 2.457708935253322, + 2.2585249971598387, + 2.2657379740849137, + 2.2382939932867885, + 2.176858950406313, + 2.2308321204036474, + 2.1903219167143106, + 2.2301790304481983 + ] + }, + "zigzag": { + "step_ms": 493.968017578125, + "prepare_ms": 3.2312944531440735, + "one_use_ms": 497.1993120312691, + "prepare_min_ms": 3.0827909940853715, + "prepare_max_ms": 19.6960570756346, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0011268685984526365, + "dq": 0.0003714092777960759, + "dkv": 0.0005664412351356137, + "d_index_q": 0.0005001608323414847, + "d_weights": 0.0005154558961486076, + "d_pool_kv": 0.0007053187475275309, + "d_pool_score": 0.0007033408865167627, + "d_ape": 1.1331546979456586e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 491.4543151855469, + 493.968017578125, + 493.0928649902344, + 494.013916015625, + 493.9161376953125, + 496.6776123046875, + 496.3877258300781 + ], + "critical_prepare_samples_ms": [ + 19.6960570756346, + 3.2806959934532642, + 3.1818929128348827, + 3.8822469068691134, + 3.1228240113705397, + 3.135847859084606, + 3.0827909940853715, + 3.6610440583899617 + ] + }, + "planner": { + "step_ms": 491.5184326171875, + "prepare_ms": 9.799090563319623, + "one_use_ms": 501.3175231805071, + "prepare_min_ms": 9.549538022838533, + "prepare_max_ms": 12.66833208501339, + "selected_policy": "selection-paired-local", + "max_relative_rms": { + "output": 0.0011264548823997533, + "dq": 0.00037080288591740675, + "dkv": 0.0005661102261241025, + "d_index_q": 0.0004907002290738318, + "d_weights": 0.00047702545312988654, + "d_pool_kv": 0.0007029215298665386, + "d_pool_score": 0.0006920376268098218, + "d_ape": 1.0082613861991237e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 491.75732421875, + 491.4028625488281, + 492.5019836425781, + 491.5184326171875, + 492.1903381347656, + 491.46417236328125, + 491.2967529296875 + ], + "critical_prepare_samples_ms": [ + 11.265398934483528, + 9.871467016637325, + 10.41808701120317, + 12.66833208501339, + 9.726714110001922, + 9.549538022838533, + 9.598031989298761, + 9.689398924820125 + ] + } + } + }, + "1024k-uniform-1024": { + "documents": 1024, + "tokens": 1048576, + "source_record_sha256": "5dcf57b1ff0c2b430ff56323c1062b39a1a343d01dae0faef1de67b8fef47bf9", + "policies": { + "contiguous": { + "step_ms": 106.8575668334961, + "prepare_ms": 7.491494528949261, + "one_use_ms": 114.34906136244535, + "prepare_min_ms": 7.3957579443231225, + "prepare_max_ms": 7.639074116013944, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.340375730780978e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.8596359553728106e-05, + "d_pool_score": 5.8019952559618404e-05, + "d_ape": 8.688634497722616e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.89324951171875, + 106.8575668334961, + 106.78739166259766, + 107.19465637207031, + 107.24988555908203, + 106.72332763671875, + 106.78819274902344 + ], + "critical_prepare_samples_ms": [ + 7.517205900512636, + 7.3957579443231225, + 7.419119938276708, + 7.494135992601514, + 7.56581814493984, + 7.639074116013944, + 7.488853065297008, + 7.402385119348764 + ] + }, + "zigzag": { + "step_ms": 121.1388168334961, + "prepare_ms": 48.3965675230138, + "one_use_ms": 169.5353843565099, + "prepare_min_ms": 47.7493159705773, + "prepare_max_ms": 161.87846509274095, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.983033003732531e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.297525455088219e-05, + "d_pool_score": 6.027668702368681e-05, + "d_ape": 8.260924534593652e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 121.27545928955078, + 120.9210205078125, + 121.1388168334961, + 120.83197021484375, + 120.97875213623047, + 121.54752349853516, + 121.18406677246094 + ], + "critical_prepare_samples_ms": [ + 51.41136993188411, + 47.85480094142258, + 47.7493159705773, + 161.87846509274095, + 48.38546901009977, + 48.245941987261176, + 48.40766603592783, + 50.705861882306635 + ] + }, + "planner": { + "step_ms": 106.80134582519531, + "prepare_ms": 9.633392037358135, + "one_use_ms": 116.43473786255345, + "prepare_min_ms": 9.522294043563306, + "prepare_max_ms": 10.079156025312841, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.190915790570461e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.686079011901337e-05, + "d_pool_score": 6.588902419850418e-05, + "d_ape": 9.060561985811648e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 106.7529296875, + 106.80134582519531, + 106.69001770019531, + 106.8199691772461, + 106.92153930664062, + 107.30044555664062, + 106.72911834716797 + ], + "critical_prepare_samples_ms": [ + 9.983930038288236, + 9.658558061346412, + 9.608226013369858, + 10.079156025312841, + 9.522294043563306, + 9.544037049636245, + 9.525719098746777, + 9.793340926989913 + ] + } + } + }, + "1024k-near-uniform-512": { + "documents": 512, + "tokens": 1048576, + "source_record_sha256": "0b8a7ab3c025afba1b9ff31062ef317a231a85fed72a4f9399d77f0a11a5d450", + "policies": { + "contiguous": { + "step_ms": 153.78297424316406, + "prepare_ms": 4.550960555206984, + "one_use_ms": 158.33393479837105, + "prepare_min_ms": 4.407087923027575, + "prepare_max_ms": 4.7475280007347465, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.635166627093273e-05, + "dq": 8.583753797678552e-06, + "dkv": 3.908999526342516e-05, + "d_index_q": 2.4805252190121222e-06, + "d_weights": 2.5308917739233984e-06, + "d_pool_kv": 6.176247649990505e-05, + "d_pool_score": 5.8405774399699e-05, + "d_ape": 8.241450646605944e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.78297424316406, + 154.11363220214844, + 154.24351501464844, + 153.45152282714844, + 153.8989715576172, + 153.48739624023438, + 153.42739868164062 + ], + "critical_prepare_samples_ms": [ + 4.606066038832068, + 4.407087923027575, + 4.484036937355995, + 4.617327940650284, + 4.496491979807615, + 4.602779052220285, + 4.499142058193684, + 4.7475280007347465 + ] + }, + "zigzag": { + "step_ms": 162.9409942626953, + "prepare_ms": 25.861825968604535, + "one_use_ms": 188.80282023129985, + "prepare_min_ms": 25.590608012862504, + "prepare_max_ms": 141.61201403476298, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 1.690072397931956e-05, + "dq": 8.902184563800953e-06, + "dkv": 3.583246096075656e-05, + "d_index_q": 2.6051406625142775e-06, + "d_weights": 2.4387942932217848e-06, + "d_pool_kv": 6.690809516440746e-05, + "d_pool_score": 5.7566133313703044e-05, + "d_ape": 9.085070529934042e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 163.11216735839844, + 162.67152404785156, + 162.5786895751953, + 163.08656311035156, + 163.2576904296875, + 162.770751953125, + 162.9409942626953 + ], + "critical_prepare_samples_ms": [ + 29.362841974943876, + 28.138961060903966, + 25.614228914491832, + 25.82601795438677, + 25.8976339828223, + 25.744552025571465, + 25.590608012862504, + 141.61201403476298 + ] + }, + "planner": { + "step_ms": 153.50277709960938, + "prepare_ms": 5.739656975492835, + "one_use_ms": 159.2424340751022, + "prepare_min_ms": 5.382872885093093, + "prepare_max_ms": 5.910926032811403, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 1.66493234380544e-05, + "dq": 8.69517004748584e-06, + "dkv": 3.7684565826764085e-05, + "d_index_q": 2.546078095564981e-06, + "d_weights": 2.625010459222263e-06, + "d_pool_kv": 6.008571022238068e-05, + "d_pool_score": 5.6580447419918704e-05, + "d_ape": 8.854442371773076e-07 + }, + "max_loss_abs_diff": 0.0, + "critical_step_samples_ms": [ + 153.19667053222656, + 153.4423065185547, + 153.50277709960938, + 153.30099487304688, + 153.84597778320312, + 153.7281951904297, + 153.51461791992188 + ], + "critical_prepare_samples_ms": [ + 5.910926032811403, + 5.768615053966641, + 5.845936946570873, + 5.667127901688218, + 5.738166975788772, + 5.730206961743534, + 5.382872885093093, + 5.741146975196898 + ] + } + } + }, + "1024k-mixed-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "dddf71dc1a5edeecb47eec7b85b24e6c842fb9e8736fd55552388a3f37a1dc77", + "policies": { + "contiguous": { + "step_ms": 566.1741333007812, + "prepare_ms": 10.622373956721276, + "one_use_ms": 576.7965072575025, + "prepare_min_ms": 10.451745009049773, + "prepare_max_ms": 10.986343026161194, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0007698426967776738, + "dq": 0.000278721133254185, + "dkv": 0.0005366458202454897, + "d_index_q": 0.0004893065256747059, + "d_weights": 0.000496051600621093, + "d_pool_kv": 0.0006594956893625123, + "d_pool_score": 0.0006637931035948877, + "d_ape": 9.298445474597294e-06 + }, + "max_loss_abs_diff": 1.4551915228366852e-10, + "critical_step_samples_ms": [ + 568.0925903320312, + 565.6495971679688, + 565.1004638671875, + 567.221435546875, + 566.1741333007812, + 567.1893920898438, + 563.3007202148438 + ], + "critical_prepare_samples_ms": [ + 10.948296985588968, + 10.565283009782434, + 10.503393015824258, + 10.87179605383426, + 10.451745009049773, + 10.473948088474572, + 10.986343026161194, + 10.679464903660119 + ] + }, + "zigzag": { + "step_ms": 549.7163696289062, + "prepare_ms": 48.262091528158635, + "one_use_ms": 597.9784611570649, + "prepare_min_ms": 46.8996309209615, + "prepare_max_ms": 165.2717370307073, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0007832338017054126, + "dq": 0.0002632348658705566, + "dkv": 0.0004852835254326828, + "d_index_q": 0.0005267448260146545, + "d_weights": 0.0005010625147896707, + "d_pool_kv": 0.0006835671852074876, + "d_pool_score": 0.0006593969952528296, + "d_ape": 9.171136322244436e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 548.9651489257812, + 549.9647216796875, + 552.8906860351562, + 549.7163696289062, + 548.3233642578125, + 550.1369018554688, + 548.8441772460938 + ], + "critical_prepare_samples_ms": [ + 50.29113800264895, + 47.78582998551428, + 46.8996309209615, + 165.2717370307073, + 48.17056504543871, + 48.35361801087856, + 47.71811398677528, + 48.362344969064 + ] + }, + "planner": { + "step_ms": 493.4424743652344, + "prepare_ms": 34.91155954543501, + "one_use_ms": 528.3540339106694, + "prepare_min_ms": 34.27027305588126, + "prepare_max_ms": 119.49189391452819, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.0008045749900604678, + "dq": 0.00028106644892919336, + "dkv": 0.0005445073846626518, + "d_index_q": 0.0005790968752337036, + "d_weights": 0.0005290486376168959, + "d_pool_kv": 0.000667951163237759, + "d_pool_score": 0.0006640420442457095, + "d_ape": 9.158098557262821e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 493.3392028808594, + 494.3460693359375, + 492.5264892578125, + 493.4424743652344, + 493.0041809082031, + 495.9510192871094, + 494.9887390136719 + ], + "critical_prepare_samples_ms": [ + 34.88306805957109, + 35.257180919870734, + 34.5360190840438, + 35.578013863414526, + 34.29650503676385, + 34.940051031298935, + 34.27027305588126, + 119.49189391452819 + ] + } + } + }, + "1024k-long-tail-1000": { + "documents": 1000, + "tokens": 1048576, + "source_record_sha256": "e0714dbf812ab0b51ad0e854f8e5cc6fc0fd8a8549ffb279e54cb753bb104aa1", + "policies": { + "contiguous": { + "step_ms": 801.3711547851562, + "prepare_ms": 14.17607138864696, + "one_use_ms": 815.5472261738032, + "prepare_min_ms": 13.991917949169874, + "prepare_max_ms": 14.449334004893899, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0009458803536275202, + "dq": 0.00032713645135391036, + "dkv": 0.0006961451986581857, + "d_index_q": 0.0007918691377933198, + "d_weights": 0.0006913715310103977, + "d_pool_kv": 0.000853195284882402, + "d_pool_score": 0.0008060075488969843, + "d_ape": 8.446302463058031e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 799.466796875, + 804.5145263671875, + 800.3975219726562, + 805.9153442382812, + 802.1598510742188, + 801.3711547851562, + 798.016845703125 + ], + "critical_prepare_samples_ms": [ + 14.449334004893899, + 14.040772104635835, + 14.401104883290827, + 14.088836847804487, + 14.436830999329686, + 14.040581998415291, + 13.991917949169874, + 14.263305929489434 + ] + }, + "zigzag": { + "step_ms": 715.4530029296875, + "prepare_ms": 151.8010766012594, + "one_use_ms": 867.2540795309469, + "prepare_min_ms": 51.4363938709721, + "prepare_max_ms": 163.23676297906786, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0008850672295580158, + "dq": 0.0002783013933514211, + "dkv": 0.0006988498625940763, + "d_index_q": 0.0006129151100521033, + "d_weights": 0.0005560157421175833, + "d_pool_kv": 0.0008523786742666889, + "d_pool_score": 0.0008077892330878795, + "d_ape": 7.12458281847456e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 717.642578125, + 714.7274169921875, + 715.4530029296875, + 714.2869873046875, + 714.8427124023438, + 717.7915649414062, + 717.44677734375 + ], + "critical_prepare_samples_ms": [ + 163.23676297906786, + 51.87820305582136, + 156.81109705474228, + 154.88058305345476, + 158.93646096810699, + 51.80381902027875, + 51.4363938709721, + 148.72157014906406 + ] + }, + "planner": { + "step_ms": 591.85302734375, + "prepare_ms": 38.67735603125766, + "one_use_ms": 630.5303833750077, + "prepare_min_ms": 38.26623398344964, + "prepare_max_ms": 126.19063805323094, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.000942381079150994, + "dq": 0.0003192592988327247, + "dkv": 0.0005112119580584089, + "d_index_q": 0.0005326490446759798, + "d_weights": 0.0005266672889364208, + "d_pool_kv": 0.0006746754160394707, + "d_pool_score": 0.0006688692091720076, + "d_ape": 7.366243955007939e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-11, + "critical_step_samples_ms": [ + 588.470458984375, + 591.85302734375, + 589.7974243164062, + 595.7284545898438, + 592.322021484375, + 593.126953125, + 587.995849609375 + ], + "critical_prepare_samples_ms": [ + 38.26623398344964, + 39.0528819989413, + 126.19063805323094, + 38.612914038822055, + 38.74179802369326, + 38.45288895536214, + 38.46422384958714, + 39.24016701057553 + ] + } + } + } + }, + "cohorts": { + "windows-final": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "34c2d866f81c1127623ba482adce4ff365a7a1663f66bfdbf717a39cbb7c3beb", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + }, + "dozens": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": [ + "uniform-64", + "near-uniform-64", + "mixed-22" + ], + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + }, + "one-m-single": { + "arguments": { + "tokens": [ + 1048576 + ], + "cases": [ + "single" + ], + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + }, + "one-m-remaining": { + "arguments": { + "tokens": [ + 1048576 + ], + "cases": [ + "uniform-64", + "near-uniform-64", + "mixed-22", + "uniform-1024", + "near-uniform-512", + "mixed-1000", + "long-tail-1000" + ], + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "d115273ecfc9c37d1b93aeffd84325472528b05ee7c32865aae8a47eab2df827", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + } + } + }, + "totals_by_context": { + "128": { + "contiguous": [ + 207.97830295562744, + 237.19847300602123 + ], + "zigzag": [ + 221.86822509765625, + 395.9265925376676 + ], + "planner": [ + 206.09743785858154, + 278.04668443091214 + ] + }, + "256": { + "contiguous": [ + 530.4107818603516, + 564.9268742501736 + ], + "zigzag": [ + 555.1017608642578, + 877.8458124990575 + ], + "planner": [ + 507.27488136291504, + 604.8603271567263 + ] + }, + "1024": { + "contiguous": [ + 3536.3107986450195, + 3580.6261210637167 + ], + "zigzag": [ + 3170.2919540405273, + 3459.5822826670483 + ], + "planner": [ + 2952.967788696289, + 3064.5250777811743 + ] + } + }, + "max_relative_rms": { + "output": 0.002136063325688249, + "dq": 0.0006708973533251172, + "dkv": 0.0010310600226582155, + "d_index_q": 0.0007967333850337871, + "d_weights": 0.0006913715310103977, + "d_pool_kv": 0.0013756446763457486, + "d_pool_score": 0.0013779578045436441, + "d_ape": 2.647273742900797e-05 + }, + "max_loss_abs_diff": 8.149072527885437e-10 +} diff --git a/docs/results/csa_cp_windows_20261116.json b/docs/results/csa_cp_windows_20261116.json new file mode 100644 index 00000000..6dc9905c --- /dev/null +++ b/docs/results/csa_cp_windows_20261116.json @@ -0,0 +1,5433 @@ +{ + "schema_version": 1, + "date": "2026-11-16", + "platform": "B300 CP8", + "scope": "Complete CSA operator forward/backward; no KDA, projections, MoE or optimizer", + "native_source_verification": { + "commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "checked_files": 3285, + "mismatches": [] + }, + "cpu_tests": 435, + "historical_exact_plan_comparisons": 124, + "runs": { + "full_search": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "prepare_repeats": 8, + "min_gain_ms": 0.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 6, + "check_only": false + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 0.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "cases": { + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "1ce20ae25f95e468a8ceb74f0607b96cd4b3c96afba3f565edacffb45121c460", + "policies": { + "legacy-contiguous": { + "step_ms": 32.311038970947266, + "prepare_ms": 8.923984481953084, + "one_use_ms": 41.23502345290035, + "prepare_min_ms": 8.652233984321356, + "prepare_max_ms": 9.180870023556054, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 32.3218879699707, + 32.34201431274414, + 32.311038970947266, + 32.27436828613281, + 32.30780792236328, + 32.3006706237793, + 32.32668685913086 + ], + "critical_prepare_samples_ms": [ + 9.180870023556054, + 8.702786988578737, + 8.885186980478466, + 9.141317917965353, + 8.652233984321356, + 8.755585993640125, + 8.962781983427703, + 8.985426160506904 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "contiguous": { + "step_ms": 32.37452697753906, + "prepare_ms": 7.578457531053573, + "one_use_ms": 39.952984508592635, + "prepare_min_ms": 7.474332000128925, + "prepare_max_ms": 12.586126918904483, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015154178483681854, + "dq": 0.0004590856367981372, + "dkv": 0.00046865559368039917, + "d_index_q": 0.0003544140670990448, + "d_weights": 0.0003176278618649191, + "d_pool_kv": 0.0005387196908820256, + "d_pool_score": 0.0005430168708674836, + "d_ape": 4.7473252849530434e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.37452697753906, + 32.392513275146484, + 32.36988830566406, + 32.39535903930664, + 32.35891342163086, + 32.35808181762695, + 32.382911682128906 + ], + "critical_prepare_samples_ms": [ + 7.727506919763982, + 7.595576927997172, + 7.546272943727672, + 12.586126918904483, + 7.49861600343138, + 7.561338134109974, + 7.474332000128925, + 7.61300593148917 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "zigzag": { + "step_ms": 37.90224075317383, + "prepare_ms": 45.84232851630077, + "one_use_ms": 83.7445692694746, + "prepare_min_ms": 44.70354691147804, + "prepare_max_ms": 155.66171507816762, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016252246640432245, + "dq": 0.0004916736119971625, + "dkv": 0.0004861918573380407, + "d_index_q": 0.0003569257876233377, + "d_weights": 0.0003177650459406723, + "d_pool_kv": 0.0005486278732828732, + "d_pool_score": 0.0005445220484194565, + "d_ape": 3.860767227143574e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 37.866241455078125, + 37.90224075317383, + 37.91574478149414, + 37.9218864440918, + 37.847808837890625, + 37.96307373046875, + 37.85862350463867 + ], + "critical_prepare_samples_ms": [ + 47.00984014198184, + 45.00916204415262, + 44.70354691147804, + 153.49186910316348, + 46.21615505311638, + 45.468501979485154, + 44.9322130298242, + 155.66171507816762 + ], + "candidate_workspace_bytes_per_rank": [ + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344 + ] + }, + "planner": { + "step_ms": 32.353633880615234, + "prepare_ms": 12.12866249261424, + "one_use_ms": 44.482296373229474, + "prepare_min_ms": 11.894998839125037, + "prepare_max_ms": 123.44329594634473, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015138876329661513, + "dq": 0.00045860825477340696, + "dkv": 0.00046849135874668684, + "d_index_q": 0.0003562482112602212, + "d_weights": 0.00032118335948173643, + "d_pool_kv": 0.0005438274579043697, + "d_pool_score": 0.0005519648229923422, + "d_ape": 4.258189850384637e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.353633880615234, + 32.412353515625, + 32.39923095703125, + 32.37456130981445, + 32.34396743774414, + 32.31990432739258, + 32.341121673583984 + ], + "critical_prepare_samples_ms": [ + 12.077874038368464, + 12.179450946860015, + 123.44329594634473, + 12.385182897560298, + 11.991281062364578, + 12.406638008542359, + 11.894998839125037, + 11.912805959582329 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "a42d7db2614d9e21c3bd24a8de34199977030c3a75e1b28856d51a604529dfdb", + "policies": { + "legacy-contiguous": { + "step_ms": 33.05331039428711, + "prepare_ms": 8.72880796669051, + "one_use_ms": 41.78211836097762, + "prepare_min_ms": 8.669499889947474, + "prepare_max_ms": 9.25529608502984, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 33.07484817504883, + 33.05331039428711, + 33.061248779296875, + 33.03206253051758, + 33.019039154052734, + 33.1328010559082, + 33.0307502746582 + ], + "critical_prepare_samples_ms": [ + 9.25529608502984, + 8.874272927641869, + 8.747634943574667, + 8.709980989806354, + 8.782866061665118, + 8.703469997271895, + 8.687923080287874, + 8.669499889947474 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "contiguous": { + "step_ms": 33.07494354248047, + "prepare_ms": 7.641692471224815, + "one_use_ms": 40.71663601370528, + "prepare_min_ms": 7.504745037294924, + "prepare_max_ms": 110.33575492911041, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002124358368844286, + "dq": 0.0006708973533251172, + "dkv": 0.0006748384557586351, + "d_index_q": 0.000419026902945799, + "d_weights": 0.0003304557285110155, + "d_pool_kv": 0.0005772261277278353, + "d_pool_score": 0.0005797688319552085, + "d_ape": 7.489935465690975e-06 + }, + "max_loss_abs_diff": 3.4924596548080444e-10, + "critical_step_samples_ms": [ + 33.100799560546875, + 33.073055267333984, + 33.057918548583984, + 33.08787155151367, + 33.07494354248047, + 33.04643249511719, + 33.11958312988281 + ], + "critical_prepare_samples_ms": [ + 8.83635098580271, + 7.582117919810116, + 110.33575492911041, + 7.812974974513054, + 7.556077907793224, + 7.6242099748924375, + 7.504745037294924, + 7.659174967557192 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "zigzag": { + "step_ms": 38.74604797363281, + "prepare_ms": 45.72551843011752, + "one_use_ms": 84.47156640375033, + "prepare_min_ms": 44.49985595420003, + "prepare_max_ms": 152.91648893617094, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.002136063325688249, + "dq": 0.0006448208867270936, + "dkv": 0.0005397046574182863, + "d_index_q": 0.0003537235166520844, + "d_weights": 0.0003170419424130787, + "d_pool_kv": 0.0005661726253782145, + "d_pool_score": 0.0005473705393115235, + "d_ape": 7.236730944585608e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.72604751586914, + 38.74604797363281, + 38.78224182128906, + 38.78335952758789, + 38.72016143798828, + 38.75788879394531, + 38.730464935302734 + ], + "critical_prepare_samples_ms": [ + 47.61352890636772, + 45.467405929230154, + 44.49985595420003, + 152.91648893617094, + 45.98363093100488, + 45.106612145900726, + 44.917382998391986, + 145.95098292920738 + ], + "candidate_workspace_bytes_per_rank": [ + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104 + ] + }, + "planner": { + "step_ms": 33.0742073059082, + "prepare_ms": 28.312876995187253, + "one_use_ms": 61.387084301095456, + "prepare_min_ms": 28.01844000350684, + "prepare_max_ms": 123.6400690395385, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0020962956426669433, + "dq": 0.0006347467076768539, + "dkv": 0.0005276886282440079, + "d_index_q": 0.00035682941768751356, + "d_weights": 0.00032945679607176856, + "d_pool_kv": 0.0005648985794352423, + "d_pool_score": 0.0005625030405139739, + "d_ape": 6.634011369785548e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 33.05827331542969, + 33.0742073059082, + 33.59049606323242, + 33.06959915161133, + 33.093441009521484, + 33.08927917480469, + 33.03887939453125 + ], + "critical_prepare_samples_ms": [ + 28.298633988015354, + 28.01844000350684, + 123.6400690395385, + 28.612435911782086, + 28.067155042663217, + 28.327120002359152, + 28.597721946425736, + 28.228612965904176 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "be33650d531537cf859041aa184f7846ca91ee7d6f07866e13f5eff3df7ff982", + "policies": { + "legacy-contiguous": { + "step_ms": 9.969632148742676, + "prepare_ms": 4.448324500117451, + "one_use_ms": 14.417956648860127, + "prepare_min_ms": 4.3677909998223186, + "prepare_max_ms": 5.127329030074179, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.9651517868042, + 9.94489574432373, + 9.969632148742676, + 9.992608070373535, + 10.025152206420898, + 9.945216178894043, + 10.024991989135742 + ], + "critical_prepare_samples_ms": [ + 4.6669780276715755, + 5.127329030074179, + 4.3677909998223186, + 4.452915978617966, + 4.42988402210176, + 4.433039925061166, + 4.443733021616936, + 4.50379098765552 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "contiguous": { + "step_ms": 10.01318359375, + "prepare_ms": 3.743164998013526, + "one_use_ms": 13.756348591763526, + "prepare_min_ms": 3.6540150176733732, + "prepare_max_ms": 3.838455071672797, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 4.31067005302863e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.485686835600299e-05, + "d_pool_score": 3.734317152918191e-05, + "d_ape": 1.5884632522867948e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.00160026550293, + 10.01318359375, + 9.96771240234375, + 10.067392349243164, + 9.997440338134766, + 10.017472267150879, + 10.042176246643066 + ], + "critical_prepare_samples_ms": [ + 3.793359035626054, + 3.752059070393443, + 3.740546992048621, + 3.739589941687882, + 3.700686036609113, + 3.7457830039784312, + 3.6540150176733732, + 3.838455071672797 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "zigzag": { + "step_ms": 11.205792427062988, + "prepare_ms": 24.085226003080606, + "one_use_ms": 35.291018430143595, + "prepare_min_ms": 23.83457194082439, + "prepare_max_ms": 139.23344609793276, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7925220371458967e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 7.136588948385061e-05, + "d_pool_score": 4.065888036840021e-05, + "d_ape": 1.7333242400394557e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.202655792236328, + 11.212991714477539, + 11.187968254089355, + 11.230527877807617, + 11.205792427062988, + 11.16857624053955, + 11.311712265014648 + ], + "critical_prepare_samples_ms": [ + 27.768890955485404, + 25.140583980828524, + 24.11484904587269, + 23.88202305883169, + 24.055602960288525, + 24.04588391073048, + 23.83457194082439, + 139.23344609793276 + ], + "candidate_workspace_bytes_per_rank": [ + 1855488, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536 + ] + }, + "planner": { + "step_ms": 10.027359962463379, + "prepare_ms": 4.955698561388999, + "one_use_ms": 14.983058523852378, + "prepare_min_ms": 4.824013100005686, + "prepare_max_ms": 5.206209025345743, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3273367675160776e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.542720423634009e-05, + "d_pool_score": 5.10650784981466e-05, + "d_ape": 2.1403371983524285e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.017024040222168, + 10.369183540344238, + 10.027359962463379, + 10.05395221710205, + 10.025664329528809, + 10.148415565490723, + 10.026752471923828 + ], + "critical_prepare_samples_ms": [ + 5.206209025345743, + 5.033118999563158, + 4.875925951637328, + 4.824013100005686, + 4.920054925605655, + 4.9812550423666835, + 4.930142080411315, + 5.035332054831088 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + } + } + }, + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "4ffc0c811647416d205412593d727352a535ab6cd11c26ad666b219d87fde48f", + "policies": { + "legacy-contiguous": { + "step_ms": 47.70070266723633, + "prepare_ms": 0.7209100876934826, + "one_use_ms": 48.42161275492981, + "prepare_min_ms": 0.6982759805396199, + "prepare_max_ms": 0.9826208697631955, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 47.30112075805664, + 47.723167419433594, + 47.41270446777344, + 47.268062591552734, + 47.727745056152344, + 47.810176849365234, + 47.70070266723633 + ], + "critical_prepare_samples_ms": [ + 0.9826208697631955, + 0.7432779530063272, + 0.7265520980581641, + 0.6982759805396199, + 0.7152680773288012, + 0.714245019480586, + 0.7317300187423825, + 0.7042690413072705 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "contiguous": { + "step_ms": 47.59836959838867, + "prepare_ms": 0.7295145187526941, + "one_use_ms": 48.327884117141366, + "prepare_min_ms": 0.7136678323149681, + "prepare_max_ms": 0.8075819350779057, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015765946646741124, + "dq": 0.0005790440951112175, + "dkv": 0.0006679192583025653, + "d_index_q": 0.0007036192249884126, + "d_weights": 0.0006587839768356234, + "d_pool_kv": 0.0008462754770025072, + "d_pool_score": 0.000887909900268915, + "d_ape": 2.647273742900797e-05 + }, + "max_loss_abs_diff": 1.4551915228366852e-10, + "critical_step_samples_ms": [ + 47.504417419433594, + 47.59836959838867, + 47.79350280761719, + 47.29996871948242, + 47.922752380371094, + 47.66716766357422, + 47.468990325927734 + ], + "critical_prepare_samples_ms": [ + 0.8075819350779057, + 0.7368980441242456, + 0.7332341047003865, + 0.7257949328050017, + 0.7136678323149681, + 0.7397379959002137, + 0.7191249169409275, + 0.7216879166662693 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "zigzag": { + "step_ms": 46.22537612915039, + "prepare_ms": 0.9122525225393474, + "one_use_ms": 47.13762865168974, + "prepare_min_ms": 0.8884459966793656, + "prepare_max_ms": 2.3740038741379976, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015766157105562468, + "dq": 0.0005791250183306168, + "dkv": 0.0005826837325579235, + "d_index_q": 0.0006550745397822342, + "d_weights": 0.0005112390665302352, + "d_pool_kv": 0.0007098261716766501, + "d_pool_score": 0.0007224017331145301, + "d_ape": 2.402751970718015e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 46.06342315673828, + 46.19878387451172, + 46.51465606689453, + 46.22537612915039, + 46.21033477783203, + 46.31951904296875, + 46.25299072265625 + ], + "critical_prepare_samples_ms": [ + 2.3740038741379976, + 0.9390920167788863, + 0.9824440348893404, + 0.9092459222301841, + 0.9014920797199011, + 0.9148720419034362, + 0.8884459966793656, + 0.9096330031752586 + ], + "candidate_workspace_bytes_per_rank": [ + 536861644, + 536828104, + 536838528, + 536812296, + 536778684, + 536816940, + 536795936, + 536828336 + ] + }, + "planner": { + "step_ms": 45.816734313964844, + "prepare_ms": 7.54898099694401, + "one_use_ms": 53.365715310908854, + "prepare_min_ms": 7.393982145003974, + "prepare_max_ms": 7.698302972130477, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0015837265043694255, + "dq": 0.0005816460263006044, + "dkv": 0.0007768235696223359, + "d_index_q": 0.0007047741714831017, + "d_weights": 0.000687690452839031, + "d_pool_kv": 0.0008500666348145983, + "d_pool_score": 0.0009004768884284642, + "d_ape": 2.473088932251163e-05 + }, + "max_loss_abs_diff": 4.656612873077393e-10, + "critical_step_samples_ms": [ + 45.82166290283203, + 45.84272003173828, + 45.73017501831055, + 46.25267028808594, + 45.816734313964844, + 45.81379318237305, + 45.737918853759766 + ], + "critical_prepare_samples_ms": [ + 7.6499179704114795, + 7.698302972130477, + 7.51110992860049, + 7.575499941594899, + 7.636827998794615, + 7.517814985476434, + 7.522462052293122, + 7.393982145003974 + ], + "candidate_workspace_bytes_per_rank": [ + 536828104, + 536812296, + 536816940, + 536864252, + 536795936, + 536778684, + 536838528, + 536861644 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "38e8fdad2096841ce9b57196cdb05dfe4cd17b7c2311cf5bede714c90c5d3a91", + "policies": { + "legacy-contiguous": { + "step_ms": 9.238431930541992, + "prepare_ms": 8.05632706033066, + "one_use_ms": 17.29475899087265, + "prepare_min_ms": 7.888036081567407, + "prepare_max_ms": 8.295490988530219, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.228096008300781, + 9.246272087097168, + 9.238431930541992, + 9.236512184143066, + 10.079296112060547, + 9.257311820983887, + 9.214655876159668 + ], + "critical_prepare_samples_ms": [ + 8.295490988530219, + 7.888036081567407, + 8.236170979216695, + 8.010777062736452, + 8.126633008942008, + 7.968081976287067, + 8.101877057924867, + 7.93901807628572 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "contiguous": { + "step_ms": 9.287199974060059, + "prepare_ms": 6.7794660571962595, + "one_use_ms": 16.066666031256318, + "prepare_min_ms": 6.7019599955528975, + "prepare_max_ms": 7.129473960958421, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7034465202561103e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.80841349081689e-05, + "d_pool_score": 4.969745146211584e-05, + "d_ape": 2.463488004291754e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.327584266662598, + 9.295040130615234, + 9.287199974060059, + 9.278176307678223, + 9.278143882751465, + 9.258208274841309, + 9.473567962646484 + ], + "critical_prepare_samples_ms": [ + 6.837799912318587, + 6.7019599955528975, + 6.714918999932706, + 6.821699091233313, + 7.129473960958421, + 6.737233023159206, + 6.707626045681536, + 6.954675889573991 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "zigzag": { + "step_ms": 11.102463722229004, + "prepare_ms": 47.22392506664619, + "one_use_ms": 58.32638878887519, + "prepare_min_ms": 45.078030903823674, + "prepare_max_ms": 156.1478169169277, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.723088398789181e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.7408244692422145e-05, + "d_pool_score": 6.371221934010526e-05, + "d_ape": 3.3825512629530646e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.100447654724121, + 11.142175674438477, + 11.123104095458984, + 11.102463722229004, + 11.112383842468262, + 11.097536087036133, + 11.057215690612793 + ], + "critical_prepare_samples_ms": [ + 48.150649992749095, + 46.42646899446845, + 45.078030903823674, + 151.0155029827729, + 47.763709095306695, + 46.68414103798568, + 45.34025501925498, + 156.1478169169277 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "planner": { + "step_ms": 9.295295715332031, + "prepare_ms": 8.97316396003589, + "one_use_ms": 18.26845967536792, + "prepare_min_ms": 8.708094945177436, + "prepare_max_ms": 123.31375910434872, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.9905925067018862e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.216577144844658e-05, + "d_pool_score": 6.720340578861908e-05, + "d_ape": 3.3511195947336086e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.287327766418457, + 9.30742359161377, + 9.270848274230957, + 9.30361557006836, + 9.306976318359375, + 9.284768104553223, + 9.295295715332031 + ], + "critical_prepare_samples_ms": [ + 8.869540877640247, + 8.825646946206689, + 123.31375910434872, + 9.473679936490953, + 8.962570922449231, + 8.708094945177436, + 9.538239915855229, + 8.98375699762255 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "f6f2fb90fa19746799d7f723f65a57a1d55b4e768a2fbaff20ba8977a77dfb5e", + "policies": { + "legacy-contiguous": { + "step_ms": 98.89091491699219, + "prepare_ms": 10.643926972988993, + "one_use_ms": 109.53484188998118, + "prepare_min_ms": 10.527928941883147, + "prepare_max_ms": 11.221883120015264, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 98.17501068115234, + 99.16777801513672, + 99.69891357421875, + 98.36495971679688, + 98.89091491699219, + 100.19251251220703, + 98.40354919433594 + ], + "critical_prepare_samples_ms": [ + 11.221883120015264, + 10.699034086428583, + 10.865055839531124, + 10.588819859549403, + 10.527928941883147, + 10.56276191957295, + 11.06762106064707, + 10.588489938527346 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "contiguous": { + "step_ms": 99.36064147949219, + "prepare_ms": 9.696168475784361, + "one_use_ms": 109.05680995527655, + "prepare_min_ms": 9.520240011624992, + "prepare_max_ms": 10.234966059215367, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010563700585577058, + "dq": 0.00035831076606808053, + "dkv": 0.0004733622916756713, + "d_index_q": 0.0005939123996360771, + "d_weights": 0.0005024215185567405, + "d_pool_kv": 0.000752053852842638, + "d_pool_score": 0.0007688502326254845, + "d_ape": 6.434388548257363e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 98.40025329589844, + 99.20460510253906, + 100.11779022216797, + 99.36064147949219, + 99.45503997802734, + 98.65779113769531, + 99.67919921875 + ], + "critical_prepare_samples_ms": [ + 9.97902499511838, + 9.690101025626063, + 9.577112039551139, + 9.55722399521619, + 9.520240011624992, + 9.813543991185725, + 10.234966059215367, + 9.70223592594266 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "zigzag": { + "step_ms": 115.25299072265625, + "prepare_ms": 147.0409840112552, + "one_use_ms": 262.29397473391145, + "prepare_min_ms": 47.70176007878035, + "prepare_max_ms": 167.99743298906833, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001096030426138749, + "dq": 0.00036871769863560327, + "dkv": 0.0005522801000187844, + "d_index_q": 0.000646529994069314, + "d_weights": 0.0005200898687794669, + "d_pool_kv": 0.0007782778462285144, + "d_pool_score": 0.0007902708796079504, + "d_ape": 6.243475107842311e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 115.171875, + 115.26223754882812, + 115.35302734375, + 115.21993255615234, + 115.44630432128906, + 115.25299072265625, + 115.22217559814453 + ], + "critical_prepare_samples_ms": [ + 137.1572030475363, + 154.45262903813273, + 167.99743298906833, + 48.344484996050596, + 47.70176007878035, + 143.54554004967213, + 155.44524206779897, + 150.53642797283828 + ], + "candidate_workspace_bytes_per_rank": [ + 1073708016, + 1073692980, + 1073648060, + 536804832, + 536760312, + 536814420, + 536809884, + 536864760 + ] + }, + "planner": { + "step_ms": 92.2047348022461, + "prepare_ms": 31.8146159988828, + "one_use_ms": 124.0193508011289, + "prepare_min_ms": 31.22708504088223, + "prepare_max_ms": 132.93269090354443, + "selected_policy": "selection-local-6x", + "max_relative_rms": { + "output": 0.001048036358338339, + "dq": 0.00036006689300403495, + "dkv": 0.0004952787980932682, + "d_index_q": 0.0005006808620709585, + "d_weights": 0.00047204477185006934, + "d_pool_kv": 0.0006647742697551269, + "d_pool_score": 0.0006600247570603834, + "d_ape": 6.129309354351373e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 91.95887756347656, + 92.06761932373047, + 91.93360137939453, + 92.53968048095703, + 92.28575897216797, + 92.2804183959961, + 92.2047348022461 + ], + "critical_prepare_samples_ms": [ + 31.714363023638725, + 126.031952095218, + 31.22708504088223, + 31.914868974126875, + 31.32477297913283, + 31.713226926513016, + 132.93269090354443, + 32.04708802513778 + ], + "candidate_workspace_bytes_per_rank": [ + 412638216, + 526081780, + 536819724, + 696705780, + 782017780, + 754142216, + 725136004, + 1073725588 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "299ffffd41e8b2e4c2be973c6bdb05ae54dc4a991bc3051df8ee4e0a6127615f", + "policies": { + "legacy-contiguous": { + "step_ms": 83.7985610961914, + "prepare_ms": 10.569794510956854, + "one_use_ms": 94.36835560714826, + "prepare_min_ms": 10.402328800410032, + "prepare_max_ms": 11.000228114426136, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 84.08464050292969, + 83.7985610961914, + 83.04704284667969, + 84.08707427978516, + 83.55884552001953, + 84.34912109375, + 83.70649719238281 + ], + "critical_prepare_samples_ms": [ + 11.000228114426136, + 10.834337095730007, + 10.54192113224417, + 10.52551600150764, + 10.856117936782539, + 10.443822015076876, + 10.597667889669538, + 10.402328800410032 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "contiguous": { + "step_ms": 83.6385269165039, + "prepare_ms": 9.585648542270064, + "one_use_ms": 93.22417545877397, + "prepare_min_ms": 9.470761171542108, + "prepare_max_ms": 109.36465603299439, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017248119636322726, + "dq": 0.0005231477545538955, + "dkv": 0.0008859578942512667, + "d_index_q": 0.0003679247450818875, + "d_weights": 0.0003583038720109353, + "d_pool_kv": 0.0013756446763457486, + "d_pool_score": 0.0013569902061103135, + "d_ape": 9.213348057789348e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 83.79923248291016, + 83.6385269165039, + 83.57939147949219, + 83.14595031738281, + 84.14755249023438, + 83.68710327148438, + 83.60105895996094 + ], + "critical_prepare_samples_ms": [ + 11.086768005043268, + 9.873543865978718, + 9.49363096151501, + 9.626674116589129, + 9.470761171542108, + 9.488550946116447, + 109.36465603299439, + 9.544622967951 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "zigzag": { + "step_ms": 96.04755401611328, + "prepare_ms": 92.28663396788761, + "one_use_ms": 188.3341879840009, + "prepare_min_ms": 46.753630973398685, + "prepare_max_ms": 166.36592405848205, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017395277068004997, + "dq": 0.0005277805775999957, + "dkv": 0.0006782649359844867, + "d_index_q": 0.0003691170318294211, + "d_weights": 0.0003624685324509999, + "d_pool_kv": 0.0007577977882457726, + "d_pool_score": 0.0007313251119719116, + "d_ape": 8.559385750338412e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 96.06777954101562, + 96.25202941894531, + 96.04755401611328, + 95.81129455566406, + 95.63484954833984, + 96.13632202148438, + 95.58025360107422 + ], + "critical_prepare_samples_ms": [ + 48.80867002066225, + 166.36592405848205, + 149.53875401988626, + 48.5578749794513, + 46.753630973398685, + 135.76459791511297, + 152.24266005679965, + 47.11186105851084 + ], + "candidate_workspace_bytes_per_rank": [ + 536828196, + 536788788, + 518462572, + 480072156, + 441681740, + 403291324, + 364900908, + 326510492 + ] + }, + "planner": { + "step_ms": 82.14995574951172, + "prepare_ms": 32.668465457390994, + "one_use_ms": 114.81842120690271, + "prepare_min_ms": 32.24971203599125, + "prepare_max_ms": 33.218687982298434, + "selected_policy": "selection-local-5x", + "max_relative_rms": { + "output": 0.0017299188701990226, + "dq": 0.0005249688471103487, + "dkv": 0.0006932649020557219, + "d_index_q": 0.0003879256230062098, + "d_weights": 0.0003670382435938008, + "d_pool_kv": 0.0013198355292136361, + "d_pool_score": 0.0013151740439955597, + "d_ape": 8.808128280757483e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 81.99295806884766, + 82.14995574951172, + 82.31292724609375, + 82.3354263305664, + 81.54524993896484, + 82.6478042602539, + 81.62854766845703 + ], + "critical_prepare_samples_ms": [ + 32.491752062924206, + 32.684653997421265, + 32.89240214508027, + 32.540695974603295, + 32.65227691736072, + 33.04584801662713, + 32.24971203599125, + 33.218687982298434 + ], + "candidate_workspace_bytes_per_rank": [ + 536858700, + 536788788, + 536788788, + 536828196, + 173740692, + 518264420, + 479874004, + 536828196 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "3abdfc48993cc3747d2f08fcfbd801e8c6fa1abaa8a607d585da37c318d9b529", + "policies": { + "legacy-contiguous": { + "step_ms": 21.674495697021484, + "prepare_ms": 4.580917942803353, + "one_use_ms": 26.255413639824837, + "prepare_min_ms": 4.535833024419844, + "prepare_max_ms": 5.053048022091389, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.720863342285156, + 21.699359893798828, + 21.674495697021484, + 21.658496856689453, + 21.68822479248047, + 21.657312393188477, + 21.64521598815918 + ], + "critical_prepare_samples_ms": [ + 5.053048022091389, + 4.535833024419844, + 4.551360965706408, + 4.582296940498054, + 4.611545125953853, + 4.579538945108652, + 4.551580990664661, + 4.621528903953731 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "contiguous": { + "step_ms": 21.785919189453125, + "prepare_ms": 3.9414139464497566, + "one_use_ms": 25.72733313590288, + "prepare_min_ms": 3.8159850519150496, + "prepare_max_ms": 4.042547079734504, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.297053926215977e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.412638872669938e-05, + "d_pool_score": 4.495461915173074e-05, + "d_ape": 1.4301480341692518e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.769792556762695, + 21.76304054260254, + 22.114463806152344, + 21.785919189453125, + 21.940128326416016, + 21.767871856689453, + 21.940895080566406 + ], + "critical_prepare_samples_ms": [ + 4.042547079734504, + 3.9293068693950772, + 3.8159850519150496, + 3.9682361530140042, + 3.8398168981075287, + 3.9214659482240677, + 3.953521023504436, + 3.995029954239726 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "zigzag": { + "step_ms": 24.209856033325195, + "prepare_ms": 24.67734646052122, + "one_use_ms": 48.88720249384642, + "prepare_min_ms": 24.372468004003167, + "prepare_max_ms": 142.0474809128791, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2708240134177204e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.3500598722775094e-05, + "d_pool_score": 5.347177795761659e-05, + "d_ape": 1.327084474519006e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 24.196128845214844, + 24.229408264160156, + 24.214399337768555, + 24.209856033325195, + 24.24527931213379, + 24.169696807861328, + 24.205184936523438 + ], + "critical_prepare_samples_ms": [ + 26.387990918010473, + 24.77585698943585, + 24.372468004003167, + 24.47995904367417, + 24.4268769165501, + 24.750579963438213, + 24.60411295760423, + 142.0474809128791 + ], + "candidate_workspace_bytes_per_rank": [ + 7901184, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376 + ] + }, + "planner": { + "step_ms": 21.7589111328125, + "prepare_ms": 5.023348960094154, + "one_use_ms": 26.782260092906654, + "prepare_min_ms": 4.9473929684609175, + "prepare_max_ms": 5.072213942185044, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.554213974193077e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.1905810341815174e-05, + "d_pool_score": 4.350734098547027e-05, + "d_ape": 1.4490775323448985e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.780927658081055, + 21.7589111328125, + 21.778976440429688, + 21.859455108642578, + 21.743295669555664, + 21.751680374145508, + 21.750688552856445 + ], + "critical_prepare_samples_ms": [ + 5.0161018734797835, + 5.0161059480160475, + 5.0472968723624945, + 5.041101947426796, + 4.989685956388712, + 5.03059197217226, + 5.072213942185044, + 4.9473929684609175 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "27e077b4ff55aa00c71a035377fafcb97d9db8bfb198c213673c877e5ded916b", + "policies": { + "legacy-contiguous": { + "step_ms": 115.51570892333984, + "prepare_ms": 1.11213797936216, + "one_use_ms": 116.627846902702, + "prepare_min_ms": 1.0933621088042855, + "prepare_max_ms": 1.5027470653876662, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 116.38162994384766, + 115.88809967041016, + 115.89097595214844, + 115.51570892333984, + 114.80028533935547, + 115.36595153808594, + 115.4543685913086 + ], + "critical_prepare_samples_ms": [ + 1.5027470653876662, + 1.1409649159759283, + 1.1095319641754031, + 1.0933621088042855, + 1.114262966439128, + 1.1321530910208821, + 1.110012992285192, + 1.104076043702662 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "contiguous": { + "step_ms": 115.2938232421875, + "prepare_ms": 1.1161149595864117, + "one_use_ms": 116.40993820177391, + "prepare_min_ms": 1.087293028831482, + "prepare_max_ms": 1.304103061556816, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014842079275755727, + "dq": 0.0005035188023387811, + "dkv": 0.0010278174526612903, + "d_index_q": 0.0006256382167037119, + "d_weights": 0.000563762569940494, + "d_pool_kv": 0.0013303364084540658, + "d_pool_score": 0.001377639655655214, + "d_ape": 2.0452408219956506e-05 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 114.4939193725586, + 115.84454345703125, + 115.2938232421875, + 114.8194580078125, + 116.123779296875, + 114.58399963378906, + 115.50758361816406 + ], + "critical_prepare_samples_ms": [ + 1.304103061556816, + 1.1089229956269264, + 1.0957310441881418, + 1.1007870780304074, + 1.1440730886533856, + 1.123306923545897, + 1.087293028831482, + 1.187960966490209 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "zigzag": { + "step_ms": 105.58483123779297, + "prepare_ms": 1.1357140028849244, + "one_use_ms": 106.7205452406779, + "prepare_min_ms": 1.1315711308270693, + "prepare_max_ms": 2.692296984605491, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014841542205465054, + "dq": 0.00050380319214946, + "dkv": 0.0010287937173898806, + "d_index_q": 0.0006482205309808245, + "d_weights": 0.000583151648179517, + "d_pool_kv": 0.0013331964396419756, + "d_pool_score": 0.0013779578045436441, + "d_ape": 1.9963974118421826e-05 + }, + "max_loss_abs_diff": 3.7834979593753815e-10, + "critical_step_samples_ms": [ + 106.04412841796875, + 105.58483123779297, + 105.72246551513672, + 105.1151351928711, + 105.20396423339844, + 106.02706909179688, + 105.4378890991211 + ], + "critical_prepare_samples_ms": [ + 2.692296984605491, + 1.1655869893729687, + 1.1315711308270693, + 1.1326848762109876, + 1.1354410089552402, + 1.15663290489465, + 1.1359869968146086, + 1.1334370356053114 + ], + "candidate_workspace_bytes_per_rank": [ + 1073712188, + 1073660864, + 1073659492, + 1073631792, + 1073736736, + 1073737680, + 1073713936, + 1073713120 + ] + }, + "planner": { + "step_ms": 104.14985656738281, + "prepare_ms": 7.791060954332352, + "one_use_ms": 111.94091752171516, + "prepare_min_ms": 7.6777610229328275, + "prepare_max_ms": 8.784707984887064, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014818148634560368, + "dq": 0.0005030012855905718, + "dkv": 0.0010310600226582155, + "d_index_q": 0.0007705202387904052, + "d_weights": 0.0006243137936169513, + "d_pool_kv": 0.0013332458370744855, + "d_pool_score": 0.0013751792972335336, + "d_ape": 2.096916046719033e-05 + }, + "max_loss_abs_diff": 2.0372681319713593e-10, + "critical_step_samples_ms": [ + 103.95587158203125, + 104.33695983886719, + 103.87484741210938, + 104.14985656738281, + 104.16598510742188, + 104.65894317626953, + 104.02249908447266 + ], + "critical_prepare_samples_ms": [ + 7.850671885535121, + 8.708197972737253, + 7.70237494725734, + 7.6777610229328275, + 8.784707984887064, + 7.718592998571694, + 7.731450023129582, + 8.389059104956686 + ], + "candidate_workspace_bytes_per_rank": [ + 1073660864, + 1073631792, + 1073737680, + 1073736096, + 1073713936, + 1073736736, + 1073659492, + 1073712188 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "a3e04775cd4674a149c44452e86572771d80fd867f99c915f4d8703ea7cf55c1", + "policies": { + "legacy-contiguous": { + "step_ms": 18.887359619140625, + "prepare_ms": 8.322144509293139, + "one_use_ms": 27.209504128433764, + "prepare_min_ms": 7.978446898050606, + "prepare_max_ms": 10.85648185107857, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.922527313232422, + 18.899232864379883, + 18.88604736328125, + 18.904191970825195, + 18.860639572143555, + 18.850208282470703, + 18.887359619140625 + ], + "critical_prepare_samples_ms": [ + 8.449674933217466, + 7.978446898050606, + 8.195264963433146, + 8.395488141104579, + 8.383898064494133, + 8.15566093660891, + 8.260390954092145, + 10.85648185107857 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "contiguous": { + "step_ms": 18.96121597290039, + "prepare_ms": 6.907838920596987, + "one_use_ms": 25.869054893497378, + "prepare_min_ms": 6.844045943580568, + "prepare_max_ms": 7.187637034803629, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.724041124260225e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.511801279303253e-05, + "d_pool_score": 5.232610864426897e-05, + "d_ape": 1.4967124181438183e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 19.019712448120117, + 18.97760009765625, + 18.96121597290039, + 18.9564151763916, + 18.95644760131836, + 18.9434871673584, + 19.144479751586914 + ], + "critical_prepare_samples_ms": [ + 7.187637034803629, + 6.97691401001066, + 6.844045943580568, + 6.993726012296975, + 6.876001018099487, + 6.887514842674136, + 6.885394104756415, + 6.928162998519838 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "zigzag": { + "step_ms": 22.557504653930664, + "prepare_ms": 46.86611460056156, + "one_use_ms": 69.42361925449222, + "prepare_min_ms": 46.30116908811033, + "prepare_max_ms": 167.44421899784356, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8637480027866625e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.271880161331187e-05, + "d_pool_score": 5.610531234471807e-05, + "d_ape": 1.5175327931420397e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 22.557504653930664, + 22.60767936706543, + 22.548383712768555, + 22.567264556884766, + 22.551103591918945, + 22.51372718811035, + 22.592575073242188 + ], + "critical_prepare_samples_ms": [ + 47.66656889114529, + 46.35808803141117, + 46.458124998025596, + 167.44421899784356, + 47.36804007552564, + 47.025946085341275, + 46.30116908811033, + 46.706283115781844 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "planner": { + "step_ms": 18.94291114807129, + "prepare_ms": 8.967155939899385, + "one_use_ms": 27.910067087970674, + "prepare_min_ms": 8.858141023665667, + "prepare_max_ms": 14.381637098267674, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.590593616004352e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.534328689387801e-05, + "d_pool_score": 5.3623912772759155e-05, + "d_ape": 1.7238388124362549e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.938079833984375, + 18.94291114807129, + 18.93289566040039, + 18.952415466308594, + 18.924928665161133, + 18.946847915649414, + 19.089344024658203 + ], + "critical_prepare_samples_ms": [ + 9.043016005307436, + 8.858141023665667, + 8.99953895714134, + 8.93477292265743, + 9.00569697842002, + 8.906646049581468, + 8.866342948749661, + 14.381637098267674 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + } + } + } + } + }, + "reduced_search_sensitivity": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 7, + "min_gain_ms": 2.0, + "min_chunk_tokens": 1024, + "max_chunks_per_rank": 2, + "check_only": false + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 2.0, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "cases": { + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "206ac833d00cc5bf1c2437b126f72c4d797ec86a89c522f164b5ef808f1fd8d6", + "policies": { + "legacy-contiguous": { + "step_ms": 32.31577682495117, + "prepare_ms": 9.774404112249613, + "one_use_ms": 42.090180937200785, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 32.31577682495117, + 32.28287887573242, + 32.33244705200195, + 32.309471130371094, + 32.30284881591797, + 32.32038497924805, + 32.39052963256836 + ], + "critical_prepare_samples_ms": [ + 9.07967402599752, + 9.774404112249613, + 9.776547900401056 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "contiguous": { + "step_ms": 32.3752326965332, + "prepare_ms": 8.127258974127471, + "one_use_ms": 40.502491670660675, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015114969438123802, + "dq": 0.0004578816300774803, + "dkv": 0.0004676909672103567, + "d_index_q": 0.00035461648640452504, + "d_weights": 0.0003343071935462977, + "d_pool_kv": 0.0005292565852777516, + "d_pool_score": 0.000537489115750937, + "d_ape": 4.008685712418047e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.35612869262695, + 32.37507247924805, + 32.387840270996094, + 32.435489654541016, + 32.35036849975586, + 32.3752326965332, + 32.56003189086914 + ], + "critical_prepare_samples_ms": [ + 7.735361927188933, + 8.529279031790793, + 8.127258974127471 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "zigzag": { + "step_ms": 37.885311126708984, + "prepare_ms": 45.99909298121929, + "one_use_ms": 83.88440410792828, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0016234906071296098, + "dq": 0.0004906073228918477, + "dkv": 0.000485240208978095, + "d_index_q": 0.0003558193972514481, + "d_weights": 0.00032124146055954113, + "d_pool_kv": 0.0005457322304231309, + "d_pool_score": 0.0005399599227922468, + "d_ape": 4.763308121602455e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 37.87958526611328, + 37.90604782104492, + 37.908992767333984, + 38.01612854003906, + 37.85958480834961, + 37.88185501098633, + 37.885311126708984 + ], + "critical_prepare_samples_ms": [ + 47.39365400746465, + 45.99909298121929, + 45.87791208177805 + ], + "candidate_workspace_bytes_per_rank": [ + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344 + ] + }, + "planner": { + "step_ms": 32.36540985107422, + "prepare_ms": 10.196399874985218, + "one_use_ms": 42.56180972605944, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.00150859254036341, + "dq": 0.0004572904703051933, + "dkv": 0.0004729963619223918, + "d_index_q": 0.0003518774483353445, + "d_weights": 0.0003240170008018262, + "d_pool_kv": 0.0005420697840284305, + "d_pool_score": 0.0005310764478034278, + "d_ape": 4.198265664849056e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 32.37472152709961, + 32.34428787231445, + 32.33184051513672, + 32.42252731323242, + 32.364959716796875, + 32.383583068847656, + 32.36540985107422 + ], + "critical_prepare_samples_ms": [ + 10.196399874985218, + 10.216351016424596, + 10.10116096585989 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "14d573aed02d0fbe56347afe73070308fa4b52a1f50a5f39099069f9a822f622", + "policies": { + "legacy-contiguous": { + "step_ms": 33.04886245727539, + "prepare_ms": 8.66100809071213, + "one_use_ms": 41.70987054798752, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 33.05459213256836, + 33.069950103759766, + 33.01395034790039, + 33.04579162597656, + 33.04886245727539, + 33.069664001464844, + 33.03974533081055 + ], + "critical_prepare_samples_ms": [ + 9.070100029930472, + 8.66100809071213, + 8.639303967356682 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "contiguous": { + "step_ms": 33.116798400878906, + "prepare_ms": 7.43433297611773, + "one_use_ms": 40.551131376996636, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002093747276231706, + "dq": 0.0006334083022316513, + "dkv": 0.0005305985785479698, + "d_index_q": 0.00035164505841025513, + "d_weights": 0.0003191535424605087, + "d_pool_kv": 0.000567973707910079, + "d_pool_score": 0.0005569264865225348, + "d_ape": 6.866102141759954e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 33.125152587890625, + 33.091617584228516, + 33.11123275756836, + 33.104000091552734, + 33.144351959228516, + 33.116798400878906, + 33.14662551879883 + ], + "critical_prepare_samples_ms": [ + 7.8012129524722695, + 7.354254019446671, + 7.43433297611773 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "zigzag": { + "step_ms": 38.7558708190918, + "prepare_ms": 45.50877504516393, + "one_use_ms": 84.26464586425573, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.002181374361486575, + "dq": 0.0006867861882885665, + "dkv": 0.0006765172784291122, + "d_index_q": 0.00041965392520146793, + "d_weights": 0.0003215599066397513, + "d_pool_kv": 0.0005907853387548043, + "d_pool_score": 0.0005889070011877098, + "d_ape": 7.251414744984451e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 38.72137451171875, + 38.7558708190918, + 38.76057434082031, + 38.8023681640625, + 38.688480377197266, + 38.73535919189453, + 38.8474235534668 + ], + "critical_prepare_samples_ms": [ + 47.14965296443552, + 45.50877504516393, + 44.8957160115242 + ], + "candidate_workspace_bytes_per_rank": [ + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104 + ] + }, + "planner": { + "step_ms": 33.0863037109375, + "prepare_ms": 9.439464891329408, + "one_use_ms": 42.52576860226691, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0020996398701580354, + "dq": 0.0006346109709168223, + "dkv": 0.0005287856528683287, + "d_index_q": 0.0003540673085342941, + "d_weights": 0.00031974887292759063, + "d_pool_kv": 0.0005708471487096938, + "d_pool_score": 0.0005611318284191702, + "d_ape": 7.035264280944537e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 33.08342361450195, + 33.0863037109375, + 33.07936096191406, + 33.14601516723633, + 33.088958740234375, + 33.13372802734375, + 33.07321548461914 + ], + "critical_prepare_samples_ms": [ + 9.505677036941051, + 9.329304099082947, + 9.439464891329408 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "c37fdc7491d81ef27e39eeb993950c31376deee5b81033de6c80951402a75014", + "policies": { + "legacy-contiguous": { + "step_ms": 9.961440086364746, + "prepare_ms": 4.771576030179858, + "one_use_ms": 14.733016116544604, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.95577621459961, + 9.969663619995117, + 9.967071533203125, + 10.025728225708008, + 9.94979190826416, + 9.959168434143066, + 9.961440086364746 + ], + "critical_prepare_samples_ms": [ + 4.771576030179858, + 7.079613162204623, + 4.413567017763853 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "contiguous": { + "step_ms": 10.006815910339355, + "prepare_ms": 3.7964109797030687, + "one_use_ms": 13.803226890042424, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.9978755880494213e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.444174851955467e-05, + "d_pool_score": 2.8797816112323566e-05, + "d_ape": 1.2955959260855573e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.015935897827148, + 10.006815910339355, + 9.975359916687012, + 10.021504402160645, + 9.999263763427734, + 10.000639915466309, + 10.02940845489502 + ], + "critical_prepare_samples_ms": [ + 3.7964109797030687, + 3.9685630472376943, + 3.6812990438193083 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "zigzag": { + "step_ms": 11.198399543762207, + "prepare_ms": 25.439652032218874, + "one_use_ms": 36.63805157598108, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.1161980846562704e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.0865890337618216e-05, + "d_pool_score": 4.153862716484465e-05, + "d_ape": 1.7307645934332891e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.415776252746582, + 11.247648239135742, + 11.255423545837402, + 11.198399543762207, + 11.182592391967773, + 11.181695938110352, + 11.188159942626953 + ], + "critical_prepare_samples_ms": [ + 25.439652032218874, + 26.81133185978979, + 23.72458192985505 + ], + "candidate_workspace_bytes_per_rank": [ + 1855488, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536 + ] + }, + "planner": { + "step_ms": 10.006272315979004, + "prepare_ms": 4.958153003826737, + "one_use_ms": 14.964425319805741, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.4528841297910174e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 3.920732113107061e-05, + "d_pool_score": 3.440938115604093e-05, + "d_ape": 1.4369673349993467e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 10.022047996520996, + 9.988991737365723, + 10.006272315979004, + 10.018207550048828, + 9.986016273498535, + 10.019583702087402, + 9.992416381835938 + ], + "critical_prepare_samples_ms": [ + 4.973354982212186, + 4.958153003826737, + 4.738058079965413 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + } + } + }, + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "9d588616976075235c01154d101322f7df108c26a45382507bffa41dc275f3c4", + "policies": { + "legacy-contiguous": { + "step_ms": 47.360191345214844, + "prepare_ms": 0.7180591346696019, + "one_use_ms": 48.078250479884446, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 47.293216705322266, + 47.327392578125, + 47.29286575317383, + 47.53891372680664, + 47.360191345214844, + 47.48175811767578, + 47.39324951171875 + ], + "critical_prepare_samples_ms": [ + 0.9503101464360952, + 0.7180591346696019, + 0.7097340421751142 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "contiguous": { + "step_ms": 47.65436935424805, + "prepare_ms": 0.7628588937222958, + "one_use_ms": 48.41722824797034, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015751207193441088, + "dq": 0.0005783199131152748, + "dkv": 0.0006685131626782165, + "d_index_q": 0.0007961003786003505, + "d_weights": 0.0006623062461361172, + "d_pool_kv": 0.0008504045561905792, + "d_pool_score": 0.0009088681729959377, + "d_ape": 2.734829661652657e-05 + }, + "max_loss_abs_diff": 6.111804395914078e-10, + "critical_step_samples_ms": [ + 47.42502212524414, + 47.697471618652344, + 47.871456146240234, + 47.40972900390625, + 47.312225341796875, + 47.71052932739258, + 47.65436935424805 + ], + "critical_prepare_samples_ms": [ + 0.8085910230875015, + 0.7628588937222958, + 0.7170919561758637 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "zigzag": { + "step_ms": 46.215328216552734, + "prepare_ms": 0.9315180359408259, + "one_use_ms": 47.14684625249356, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015815559567693414, + "dq": 0.0005808441534881571, + "dkv": 0.0005049512279862388, + "d_index_q": 0.0006542155084291897, + "d_weights": 0.0005079701559067017, + "d_pool_kv": 0.0007064207861540901, + "d_pool_score": 0.0007066278713246955, + "d_ape": 2.418222008041728e-05 + }, + "max_loss_abs_diff": 2.6193447411060333e-10, + "critical_step_samples_ms": [ + 45.99225616455078, + 46.215328216552734, + 46.77628707885742, + 46.08924865722656, + 46.17289733886719, + 46.40963363647461, + 46.41756820678711 + ], + "critical_prepare_samples_ms": [ + 2.246724907308817, + 0.9315180359408259, + 0.899365870282054 + ], + "candidate_workspace_bytes_per_rank": [ + 536861644, + 536828104, + 536838528, + 536812296, + 536778684, + 536816940, + 536795936, + 536828336 + ] + }, + "planner": { + "step_ms": 45.837215423583984, + "prepare_ms": 2.398140146397054, + "one_use_ms": 48.23535556998104, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0015835534868685192, + "dq": 0.0005814588792502117, + "dkv": 0.0006013543899328077, + "d_index_q": 0.0005534685427820214, + "d_weights": 0.000646131642927837, + "d_pool_kv": 0.0007677387392274655, + "d_pool_score": 0.0007507812475082407, + "d_ape": 2.2943076205792098e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 45.837215423583984, + 45.95561599731445, + 45.775390625, + 45.85027313232422, + 45.8144645690918, + 46.047359466552734, + 45.776161193847656 + ], + "critical_prepare_samples_ms": [ + 2.542854053899646, + 2.398140146397054, + 2.3720700992271304 + ], + "candidate_workspace_bytes_per_rank": [ + 536828104, + 536812296, + 536816940, + 536864252, + 536795936, + 536778684, + 536838528, + 536861644 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "7ac651892601ff79612fe7b78ef4aab5b87db6c5202d8a79ea3ebc2ca232cb13", + "policies": { + "legacy-contiguous": { + "step_ms": 9.289216041564941, + "prepare_ms": 7.89993314538151, + "one_use_ms": 17.18914918694645, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.295328140258789, + 9.296895980834961, + 9.288703918457031, + 9.29372787475586, + 9.246944427490234, + 9.253151893615723, + 9.289216041564941 + ], + "critical_prepare_samples_ms": [ + 8.253396954387426, + 7.89993314538151, + 7.871040143072605 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "contiguous": { + "step_ms": 9.327391624450684, + "prepare_ms": 6.7193860886618495, + "one_use_ms": 16.046777713112533, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.093166359762604e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.9217821848352475e-05, + "d_pool_score": 4.03182609837131e-05, + "d_ape": 2.242167264657663e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.32156753540039, + 9.48083209991455, + 9.290656089782715, + 9.440832138061523, + 9.508383750915527, + 9.291359901428223, + 9.327391624450684 + ], + "critical_prepare_samples_ms": [ + 6.848165998235345, + 6.7193860886618495, + 6.64277991745621 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "zigzag": { + "step_ms": 11.107904434204102, + "prepare_ms": 50.2719139913097, + "one_use_ms": 61.379818425513804, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.234117372908357e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.218470753097149e-05, + "d_pool_score": 3.817734976583123e-05, + "d_ape": 1.9858886890377225e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 11.10422420501709, + 11.310079574584961, + 11.14742374420166, + 11.099967956542969, + 11.13548755645752, + 11.08351993560791, + 11.107904434204102 + ], + "critical_prepare_samples_ms": [ + 50.2719139913097, + 45.780996908433735, + 164.21099100261927 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "planner": { + "step_ms": 9.316512107849121, + "prepare_ms": 8.645661058835685, + "one_use_ms": 17.962173166684806, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.8816705153238948e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.762204604666035e-05, + "d_pool_score": 4.186287177697969e-05, + "d_ape": 2.1236779901697946e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 9.291647911071777, + 9.311360359191895, + 9.318400382995605, + 9.316512107849121, + 9.325599670410156, + 10.244704246520996, + 9.314784049987793 + ], + "critical_prepare_samples_ms": [ + 8.862095070071518, + 8.567675948143005, + 8.645661058835685 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "306c17e1b10c9055bca24d23542d6e0ae72ba86f18743e7c1db0e21333ee9e34", + "policies": { + "legacy-contiguous": { + "step_ms": 98.6659164428711, + "prepare_ms": 11.852037045173347, + "one_use_ms": 110.51795348804444, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 98.6659164428711, + 98.62493133544922, + 98.76188659667969, + 98.96463775634766, + 98.2577896118164, + 98.4927978515625, + 98.73375701904297 + ], + "critical_prepare_samples_ms": [ + 11.852037045173347, + 11.283355066552758, + 12.048232019878924 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "contiguous": { + "step_ms": 99.30899047851562, + "prepare_ms": 10.390229988843203, + "one_use_ms": 109.69922046735883, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010573826208271084, + "dq": 0.0003592223006292566, + "dkv": 0.0005484814447866894, + "d_index_q": 0.0006455438914783363, + "d_weights": 0.0005120783559164215, + "d_pool_kv": 0.0007752975553353049, + "d_pool_score": 0.0007896526582265112, + "d_ape": 6.200486588157099e-06 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 99.23235321044922, + 98.54048156738281, + 99.35852813720703, + 99.44525146484375, + 99.30899047851562, + 98.63680267333984, + 100.43740844726562 + ], + "critical_prepare_samples_ms": [ + 10.390229988843203, + 9.821086074225605, + 10.521577904000878 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "zigzag": { + "step_ms": 115.28396606445312, + "prepare_ms": 144.87404597457498, + "one_use_ms": 260.1580120390281, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0010834614507619967, + "dq": 0.0003684552651301375, + "dkv": 0.00039418431972343653, + "d_index_q": 0.0003783078523965674, + "d_weights": 0.0003770943893548319, + "d_pool_kv": 0.0006627945441974228, + "d_pool_score": 0.0006772935978763263, + "d_ape": 5.986287096458122e-06 + }, + "max_loss_abs_diff": 5.820766091346741e-11, + "critical_step_samples_ms": [ + 115.21177673339844, + 115.45823669433594, + 115.331298828125, + 115.17340850830078, + 115.28396606445312, + 115.195068359375, + 115.5271987915039 + ], + "critical_prepare_samples_ms": [ + 50.712813972495496, + 144.87404597457498, + 168.62917703110725 + ], + "candidate_workspace_bytes_per_rank": [ + 1073708016, + 1073692980, + 1073648060, + 536804832, + 536760312, + 536814420, + 536809884, + 536864760 + ] + }, + "planner": { + "step_ms": 92.93939208984375, + "prepare_ms": 106.1690280912444, + "one_use_ms": 199.10842018108815, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0011127535868433215, + "dq": 0.0003930554637779379, + "dkv": 0.0005549246019563618, + "d_index_q": 0.000646853644347867, + "d_weights": 0.0005176427284762521, + "d_pool_kv": 0.0007843419483566719, + "d_pool_score": 0.0007981760582909821, + "d_ape": 6.011172940416037e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 92.96339416503906, + 92.93939208984375, + 93.06483459472656, + 92.72089385986328, + 92.65577697753906, + 93.29020690917969, + 92.58598327636719 + ], + "critical_prepare_samples_ms": [ + 106.1690280912444, + 122.90593807119876, + 22.352106869220734 + ], + "candidate_workspace_bytes_per_rank": [ + 536852000, + 536840844, + 536869000, + 536794800, + 1073680320, + 536834316, + 536856816, + 1073601720 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "c6cfc83abb6f9da3890128a6674d1a2d33a3dd4f89b4d7f6d15f4fa70bd3d5b8", + "policies": { + "legacy-contiguous": { + "step_ms": 83.52304077148438, + "prepare_ms": 10.926121030934155, + "one_use_ms": 94.44916180241853, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 83.31452941894531, + 84.28518676757812, + 83.41043090820312, + 83.52304077148438, + 83.24291229248047, + 83.67024230957031, + 83.7089614868164 + ], + "critical_prepare_samples_ms": [ + 11.107583995908499, + 10.926121030934155, + 10.472901980392635 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "contiguous": { + "step_ms": 83.8061752319336, + "prepare_ms": 10.013072984293103, + "one_use_ms": 93.8192482162267, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017285115614443013, + "dq": 0.0005245019570177233, + "dkv": 0.0008850784611107852, + "d_index_q": 0.0003679564403986905, + "d_weights": 0.00036400805474966116, + "d_pool_kv": 0.001373117872850444, + "d_pool_score": 0.001360731586663582, + "d_ape": 9.068534898484564e-06 + }, + "max_loss_abs_diff": 1.1641532182693481e-10, + "critical_step_samples_ms": [ + 84.08528137207031, + 83.67862701416016, + 83.81353759765625, + 83.15155029296875, + 84.11788940429688, + 83.8061752319336, + 83.74156951904297 + ], + "critical_prepare_samples_ms": [ + 11.039006989449263, + 10.013072984293103, + 9.5121389022097 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "zigzag": { + "step_ms": 96.0337905883789, + "prepare_ms": 154.22721300274134, + "one_use_ms": 250.26100359112024, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0017382082680359764, + "dq": 0.0005269916395286206, + "dkv": 0.00045928158371192967, + "d_index_q": 0.0003679860460189847, + "d_weights": 0.0003632725669426103, + "d_pool_kv": 0.0006520830116070919, + "d_pool_score": 0.0006456380441297271, + "d_ape": 7.835311140252914e-06 + }, + "max_loss_abs_diff": 2.9103830456733704e-10, + "critical_step_samples_ms": [ + 96.42201232910156, + 95.8373794555664, + 96.0337905883789, + 95.5379867553711, + 95.66000366210938, + 96.18585968017578, + 96.0810546875 + ], + "critical_prepare_samples_ms": [ + 52.832534071058035, + 155.64053610432893, + 154.22721300274134 + ], + "candidate_workspace_bytes_per_rank": [ + 536828196, + 536788788, + 518462572, + 480072156, + 441681740, + 403291324, + 364900908, + 326510492 + ] + }, + "planner": { + "step_ms": 86.18553924560547, + "prepare_ms": 21.83594205416739, + "one_use_ms": 108.02148129977286, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.001745158118362825, + "dq": 0.0005293636644694899, + "dkv": 0.0005248456195568502, + "d_index_q": 0.0003697755327810351, + "d_weights": 0.00036312147739179905, + "d_pool_kv": 0.000647131001315703, + "d_pool_score": 0.0006558826907249498, + "d_ape": 7.933114227903847e-06 + }, + "max_loss_abs_diff": 2.3283064365386963e-10, + "critical_step_samples_ms": [ + 86.18553924560547, + 86.66947174072266, + 86.13078308105469, + 86.79126739501953, + 85.99791717529297, + 86.30726623535156, + 86.09295654296875 + ], + "critical_prepare_samples_ms": [ + 21.673545124940574, + 130.55709702894092, + 21.83594205416739 + ], + "candidate_workspace_bytes_per_rank": [ + 536820936, + 536822596, + 536837028, + 536865908, + 101228656, + 536822596, + 536865908, + 536837028 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "466b346789a4d59098aad2ca2b04aca877953815c3163fc8f2c0f749a341ceae", + "policies": { + "legacy-contiguous": { + "step_ms": 21.694368362426758, + "prepare_ms": 5.549413966946304, + "one_use_ms": 27.24378232937306, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.685184478759766, + 21.670495986938477, + 21.69536018371582, + 21.73049545288086, + 21.663488388061523, + 21.694368362426758, + 21.732704162597656 + ], + "critical_prepare_samples_ms": [ + 4.9613380106166005, + 5.684494040906429, + 5.549413966946304 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "contiguous": { + "step_ms": 21.787263870239258, + "prepare_ms": 4.69140603672713, + "one_use_ms": 26.47866990696639, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.2664848913808245e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.527269782608548e-05, + "d_pool_score": 5.23650389771624e-05, + "d_ape": 1.326061817639286e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.81724739074707, + 21.77782440185547, + 21.77788734436035, + 21.807296752929688, + 21.818464279174805, + 21.787263870239258, + 21.775104522705078 + ], + "critical_prepare_samples_ms": [ + 4.69140603672713, + 4.833335056900978, + 4.65017999522388 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "zigzag": { + "step_ms": 24.22969627380371, + "prepare_ms": 25.856652995571494, + "one_use_ms": 50.086349269375205, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.4942269741698106e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.163642701493704e-05, + "d_pool_score": 6.043722187737683e-05, + "d_ape": 1.4394808166454493e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 24.23721694946289, + 24.284608840942383, + 24.22969627380371, + 24.18796730041504, + 24.21548843383789, + 24.231679916381836, + 24.21513557434082 + ], + "critical_prepare_samples_ms": [ + 26.876677060499787, + 25.856652995571494, + 24.792716838419437 + ], + "candidate_workspace_bytes_per_rank": [ + 7901184, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376 + ] + }, + "planner": { + "step_ms": 21.749343872070312, + "prepare_ms": 5.762123968452215, + "one_use_ms": 27.511467840522528, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.8881779744528944e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.6376933606401454e-05, + "d_pool_score": 6.304859853384069e-05, + "d_ape": 1.3838066052628374e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 21.749055862426758, + 21.768640518188477, + 21.900991439819336, + 21.749343872070312, + 21.737119674682617, + 21.747936248779297, + 21.77097511291504 + ], + "critical_prepare_samples_ms": [ + 5.77679299749434, + 5.70077495649457, + 5.762123968452215 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "f85bfbf7322809f6c47450cfbe646a38e29419a8c8cc8a0e1f14662b5242ecc1", + "policies": { + "legacy-contiguous": { + "step_ms": 115.33245086669922, + "prepare_ms": 1.1144890449941158, + "one_use_ms": 116.44693991169333, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 115.0580825805664, + 115.42076873779297, + 115.65145874023438, + 115.51197052001953, + 114.57266998291016, + 114.61859130859375, + 115.33245086669922 + ], + "critical_prepare_samples_ms": [ + 1.432059914804995, + 1.1144890449941158, + 1.092861988581717 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "contiguous": { + "step_ms": 115.45356750488281, + "prepare_ms": 1.1493739439174533, + "one_use_ms": 116.60294144880027, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014814819659666792, + "dq": 0.0005025477042892497, + "dkv": 0.0010280580317239322, + "d_index_q": 0.0007709961298283565, + "d_weights": 0.0006240027811370497, + "d_pool_kv": 0.0013297219628949863, + "d_pool_score": 0.0013706802273474096, + "d_ape": 2.047579904414999e-05 + }, + "max_loss_abs_diff": 1.7462298274040222e-10, + "critical_step_samples_ms": [ + 115.91276550292969, + 114.74950408935547, + 115.78205108642578, + 115.53488159179688, + 115.12838745117188, + 115.45356750488281, + 114.86653137207031 + ], + "critical_prepare_samples_ms": [ + 1.2612229911610484, + 1.1493739439174533, + 1.085231895558536 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "zigzag": { + "step_ms": 105.22351837158203, + "prepare_ms": 1.1590360663831234, + "one_use_ms": 106.38255443796515, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001487600849902013, + "dq": 0.000504489249271614, + "dkv": 0.0006238415299610192, + "d_index_q": 0.0007130704206747019, + "d_weights": 0.0005857291818919476, + "d_pool_kv": 0.0009670844224943009, + "d_pool_score": 0.00099420840848902, + "d_ape": 2.209921814997075e-05 + }, + "max_loss_abs_diff": 3.2014213502407074e-10, + "critical_step_samples_ms": [ + 105.09027099609375, + 105.3821792602539, + 104.91484832763672, + 105.22351837158203, + 105.06269073486328, + 105.24441528320312, + 105.65392303466797 + ], + "critical_prepare_samples_ms": [ + 4.508284968324006, + 1.1590360663831234, + 1.105463015846908 + ], + "candidate_workspace_bytes_per_rank": [ + 1073712188, + 1073660864, + 1073659492, + 1073631792, + 1073736736, + 1073737680, + 1073713936, + 1073713120 + ] + }, + "planner": { + "step_ms": 104.40592193603516, + "prepare_ms": 2.6892030145972967, + "one_use_ms": 107.09512495063245, + "selected_policy": "selection-local-2x", + "max_relative_rms": { + "output": 0.0014828593003667822, + "dq": 0.000503123061268814, + "dkv": 0.0010299556200644897, + "d_index_q": 0.0007715033409916601, + "d_weights": 0.0006256374689194227, + "d_pool_kv": 0.001331098022685323, + "d_pool_score": 0.001368611082483086, + "d_ape": 2.171553401170655e-05 + }, + "max_loss_abs_diff": 3.2014213502407074e-10, + "critical_step_samples_ms": [ + 105.03376007080078, + 104.3993911743164, + 104.48409271240234, + 104.16764831542969, + 103.87769317626953, + 104.97811126708984, + 104.40592193603516 + ], + "critical_prepare_samples_ms": [ + 2.7697039768099785, + 2.6892030145972967, + 2.602526918053627 + ], + "candidate_workspace_bytes_per_rank": [ + 1073660864, + 1073631792, + 1073737680, + 1073736096, + 1073713936, + 1073736736, + 1073659492, + 1073712188 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "85c6bf57bb0995c8feb5c0df0d3db82adea7e3a606ccee9a2dfa44315561c946", + "policies": { + "legacy-contiguous": { + "step_ms": 18.90492820739746, + "prepare_ms": 9.129596059210598, + "one_use_ms": 28.03452426660806, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.90729522705078, + 18.882495880126953, + 19.03046417236328, + 18.907712936401367, + 18.893888473510742, + 18.89472007751465, + 18.90492820739746 + ], + "critical_prepare_samples_ms": [ + 8.733341936022043, + 9.129596059210598, + 100.35485692787915 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "contiguous": { + "step_ms": 18.984800338745117, + "prepare_ms": 7.653723005205393, + "one_use_ms": 26.63852334395051, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.249396326647605e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.6476196629912486e-05, + "d_pool_score": 6.0312485136719826e-05, + "d_ape": 1.9333346650755677e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.984800338745117, + 18.974912643432617, + 18.95270347595215, + 18.997695922851562, + 19.007232666015625, + 18.9531192779541, + 19.015039443969727 + ], + "critical_prepare_samples_ms": [ + 7.653723005205393, + 7.782059954479337, + 6.843541981652379 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "zigzag": { + "step_ms": 22.576736450195312, + "prepare_ms": 48.61433804035187, + "one_use_ms": 71.19107449054718, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.3626305216121236e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.584356380441e-05, + "d_pool_score": 6.326438372102144e-05, + "d_ape": 1.884969194270893e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 22.52841567993164, + 22.75379180908203, + 22.576736450195312, + 22.577600479125977, + 22.57868766784668, + 22.550016403198242, + 22.55062484741211 + ], + "critical_prepare_samples_ms": [ + 48.61433804035187, + 47.93463693931699, + 167.71281801629812 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "planner": { + "step_ms": 18.959936141967773, + "prepare_ms": 9.468792006373405, + "one_use_ms": 28.42872814834118, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.9131160333384314e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.272192949548379e-05, + "d_pool_score": 6.528122941093681e-05, + "d_ape": 1.9596341111569527e-06 + }, + "max_loss_abs_diff": 0, + "critical_step_samples_ms": [ + 18.956384658813477, + 18.92902374267578, + 18.959936141967773, + 18.937664031982422, + 18.97270393371582, + 18.96233558654785, + 18.96553611755371 + ], + "critical_prepare_samples_ms": [ + 9.437776985578239, + 9.566316031850874, + 9.468792006373405 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + } + } + } + } + }, + "legacy_margin_sensitivity": { + "arguments": { + "tokens": [ + 131072, + 262144 + ], + "cases": null, + "warmups": 2, + "repeats": 5, + "min_gain_ms": null, + "min_chunk_tokens": 1024, + "check_only": false + }, + "cost_model": { + "candidate_ns": 0.01381271460590644, + "query_ns": 0.0, + "span_us": 9.547858289905301, + "bytes_per_query": 20608, + "intra_node_gbps": 651.5586637992105, + "inter_node_gbps": 651.5586637992105, + "rank_node_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "packing_ns_per_query": 8.499999694322469, + "exchange_us": 181.36905458868262, + "min_gain_fraction": 0.05, + "peer_gbps": 408.72149682964135, + "min_gain_ms": 20, + "peer_startup_us": 7.460149512894069e-21, + "inter_node_aggregate_gbps": null, + "overlap_intra_inter": false + }, + "cases": { + "128k-long-tail-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "d5dfbe3522b83384879cbf63c9340cc656199e93e699dd17ec6c755058085782", + "policies": { + "legacy-contiguous": { + "step_ms": 32.31264114379883, + "prepare_ms": 11.450840043835342, + "one_use_ms": 43.76348118763417, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 32.31264114379883, + 32.298912048339844, + 32.30374526977539, + 32.318016052246094, + 32.354881286621094 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "contiguous": { + "step_ms": 32.34806442260742, + "prepare_ms": 9.241741965524852, + "one_use_ms": 41.589806388132274, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015055117580929998, + "dq": 0.0004562001787174165, + "dkv": 0.00046682488364094504, + "d_index_q": 0.0003528964133666067, + "d_weights": 0.0003143728165677277, + "d_pool_kv": 0.0005616682841830009, + "d_pool_score": 0.0005502770556613789, + "d_ape": 4.118661662693684e-06 + }, + "critical_step_samples_ms": [ + 32.39360046386719, + 32.34806442260742, + 32.347774505615234, + 32.392127990722656, + 32.3389778137207 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + }, + "zigzag": { + "step_ms": 37.92006301879883, + "prepare_ms": 57.76502494700253, + "one_use_ms": 95.68508796580136, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001615737541975861, + "dq": 0.0004889801391640974, + "dkv": 0.00048574739383770417, + "d_index_q": 0.00035745246808257353, + "d_weights": 0.0003359146161968018, + "d_pool_kv": 0.0005580949946886438, + "d_pool_score": 0.0005679364170603011, + "d_ape": 4.1200106000536755e-06 + }, + "critical_step_samples_ms": [ + 38.000545501708984, + 37.92006301879883, + 37.89564895629883, + 37.883583068847656, + 37.9365119934082 + ], + "candidate_workspace_bytes_per_rank": [ + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344, + 53305344 + ] + }, + "planner": { + "step_ms": 32.34601593017578, + "prepare_ms": 11.20570907369256, + "one_use_ms": 43.55172500386834, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015106313582280778, + "dq": 0.0004573933437522137, + "dkv": 0.00046884446588082903, + "d_index_q": 0.00035414067780251005, + "d_weights": 0.00031735270241838205, + "d_pool_kv": 0.0005302273051469876, + "d_pool_score": 0.0005465640751035636, + "d_ape": 4.408512538131001e-06 + }, + "critical_step_samples_ms": [ + 32.34092712402344, + 32.34601593017578, + 32.35830307006836, + 32.395103454589844, + 32.34467315673828 + ], + "candidate_workspace_bytes_per_rank": [ + 507904, + 507904, + 507904, + 1884160, + 2080768, + 2080768, + 79151104, + 339722240 + ] + } + } + }, + "128k-mixed-1000": { + "documents": 1000, + "tokens": 131072, + "source_record_sha256": "2975aafa9ca831aa160ee84b82aefad9c2bf3c9db7670afd63a68195a68cc635", + "policies": { + "legacy-contiguous": { + "step_ms": 33.041534423828125, + "prepare_ms": 11.169639183208346, + "one_use_ms": 44.21117360703647, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 33.05654525756836, + 33.0382080078125, + 33.05062484741211, + 33.041534423828125, + 33.029502868652344 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "contiguous": { + "step_ms": 33.08902359008789, + "prepare_ms": 9.206014103256166, + "one_use_ms": 42.29503769334406, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002097984063477685, + "dq": 0.0006349000123178144, + "dkv": 0.0005296393426551206, + "d_index_q": 0.00035507980865031745, + "d_weights": 0.000322679720692784, + "d_pool_kv": 0.0005795026037481414, + "d_pool_score": 0.0005673212325304447, + "d_ape": 6.963864564318172e-06 + }, + "critical_step_samples_ms": [ + 33.08902359008789, + 33.07833480834961, + 33.089473724365234, + 33.090240478515625, + 33.07475280761719 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + }, + "zigzag": { + "step_ms": 38.74492645263672, + "prepare_ms": 57.125494931824505, + "one_use_ms": 95.87042138446122, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0021380216992278336, + "dq": 0.0006444019165939065, + "dkv": 0.0005376332039364028, + "d_index_q": 0.000355143576767433, + "d_weights": 0.00032763168537997136, + "d_pool_kv": 0.0005620423395904252, + "d_pool_score": 0.0005485233599230848, + "d_ape": 6.923240736087287e-06 + }, + "critical_step_samples_ms": [ + 38.74143981933594, + 38.77017593383789, + 38.7586555480957, + 38.720672607421875, + 38.74492645263672 + ], + "candidate_workspace_bytes_per_rank": [ + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104, + 141359104 + ] + }, + "planner": { + "step_ms": 33.06643295288086, + "prepare_ms": 11.127481004223228, + "one_use_ms": 44.19391395710409, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.002079553939769878, + "dq": 0.0006294535496018204, + "dkv": 0.0005275027195349258, + "d_index_q": 0.0003534628562349375, + "d_weights": 0.00032433872571991425, + "d_pool_kv": 0.0005567261666655464, + "d_pool_score": 0.0005407815979470878, + "d_ape": 7.193694112788869e-06 + }, + "critical_step_samples_ms": [ + 33.06643295288086, + 33.075233459472656, + 33.07798385620117, + 33.06131362915039, + 33.062496185302734 + ], + "candidate_workspace_bytes_per_rank": [ + 7045120, + 193970176, + 363913216, + 507904, + 507904, + 827392, + 147832832, + 416268288 + ] + } + } + }, + "128k-near-uniform-512": { + "documents": 512, + "tokens": 131072, + "source_record_sha256": "ffabaa10208a2f42e4baa2ad2d0f1cd618ef64f9ee8bcb4f9af41228a94be166", + "policies": { + "legacy-contiguous": { + "step_ms": 9.972160339355469, + "prepare_ms": 7.681053131818771, + "one_use_ms": 17.65321347117424, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 9.942336082458496, + 9.972352027893066, + 9.972895622253418, + 9.938719749450684, + 9.972160339355469 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "contiguous": { + "step_ms": 10.000096321105957, + "prepare_ms": 4.725359962321818, + "one_use_ms": 14.725456283427775, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.593801996048873e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.7593107187593284e-05, + "d_pool_score": 5.0368489828505864e-05, + "d_ape": 1.729257008310043e-06 + }, + "critical_step_samples_ms": [ + 9.989888191223145, + 10.002559661865234, + 9.998368263244629, + 10.021471977233887, + 10.000096321105957 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + }, + "zigzag": { + "step_ms": 11.208576202392578, + "prepare_ms": 28.68070499971509, + "one_use_ms": 39.88928120210767, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.527012691451957e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 4.41627835231595e-05, + "d_pool_score": 6.0764600335971727e-05, + "d_ape": 1.969570217598954e-06 + }, + "critical_step_samples_ms": [ + 11.20035171508789, + 11.220447540283203, + 11.208576202392578, + 11.198304176330566, + 11.610400199890137 + ], + "candidate_workspace_bytes_per_rank": [ + 1855488, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536, + 2113536 + ] + }, + "planner": { + "step_ms": 9.991231918334961, + "prepare_ms": 5.699237925000489, + "one_use_ms": 15.69046984333545, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.603275222450197e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.436856634563869e-05, + "d_pool_score": 5.375680093457066e-05, + "d_ape": 1.992918705712155e-06 + }, + "critical_step_samples_ms": [ + 9.991231918334961, + 10.003168106079102, + 10.058303833007812, + 9.991135597229004, + 9.991168022155762 + ], + "candidate_workspace_bytes_per_rank": [ + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280, + 2081280 + ] + } + } + }, + "128k-single": { + "documents": 1, + "tokens": 131072, + "source_record_sha256": "d154370b3567b56ba5abef41ff1d72e4b4dbc22ab2453867c16a07754cba7353", + "policies": { + "legacy-contiguous": { + "step_ms": 47.74051284790039, + "prepare_ms": 6.36032095644623, + "one_use_ms": 54.10083380434662, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 47.74051284790039, + 47.38595199584961, + 47.4453125, + 47.911231994628906, + 47.81068801879883 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "contiguous": { + "step_ms": 47.775840759277344, + "prepare_ms": 2.173455082811415, + "one_use_ms": 49.94929584208876, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015778303977359808, + "dq": 0.000579295779912375, + "dkv": 0.0006681889896006872, + "d_index_q": 0.0007045097134893591, + "d_weights": 0.0006651717024489214, + "d_pool_kv": 0.0008470738686227776, + "d_pool_score": 0.0008863334597355397, + "d_ape": 2.6518480232396688e-05 + }, + "critical_step_samples_ms": [ + 47.775840759277344, + 47.55440139770508, + 48.050174713134766, + 47.42118453979492, + 48.06953430175781 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + }, + "zigzag": { + "step_ms": 46.505985260009766, + "prepare_ms": 2.9064940754324198, + "one_use_ms": 49.412479335442185, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0015801558044298372, + "dq": 0.0005802450594053097, + "dkv": 0.0006717745484660001, + "d_index_q": 0.0007960406948691224, + "d_weights": 0.0006640151282144832, + "d_pool_kv": 0.0008450890931264659, + "d_pool_score": 0.0008844659218940599, + "d_ape": 2.4965860553199313e-05 + }, + "critical_step_samples_ms": [ + 46.105472564697266, + 46.51551818847656, + 46.505985260009766, + 46.52262496948242, + 46.18937683105469 + ], + "candidate_workspace_bytes_per_rank": [ + 536861644, + 536828104, + 536838528, + 536812296, + 536778684, + 536816940, + 536795936, + 536828336 + ] + }, + "planner": { + "step_ms": 47.641632080078125, + "prepare_ms": 6.449845037423074, + "one_use_ms": 54.0914771175012, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0015794720823890324, + "dq": 0.000579758894437607, + "dkv": 0.0006678818260938846, + "d_index_q": 0.0007048063939166791, + "d_weights": 0.0006638073325243364, + "d_pool_kv": 0.0008637417489227197, + "d_pool_score": 0.0008861272240136442, + "d_ape": 2.7367223437852473e-05 + }, + "critical_step_samples_ms": [ + 47.46495819091797, + 47.86592102050781, + 47.32950210571289, + 47.87481689453125, + 47.641632080078125 + ], + "candidate_workspace_bytes_per_rank": [ + 134201344, + 402636800, + 536859656, + 536840192, + 536828336, + 536826624, + 536812296, + 536844424 + ] + } + } + }, + "128k-uniform-1024": { + "documents": 1024, + "tokens": 131072, + "source_record_sha256": "796f64c340be56e9f3f4ab48e99daa85af564b2a885b5d7b0efdcf9ee0b57af0", + "policies": { + "legacy-contiguous": { + "step_ms": 9.263520240783691, + "prepare_ms": 11.319936951622367, + "one_use_ms": 20.58345719240606, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 9.272000312805176, + 9.263520240783691, + 9.285951614379883, + 9.262751579284668, + 9.24227237701416 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "contiguous": { + "step_ms": 9.281855583190918, + "prepare_ms": 8.454012917354703, + "one_use_ms": 17.73586850054562, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.268005405014033e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.299814918031266e-05, + "d_pool_score": 3.7219175723565274e-05, + "d_ape": 1.958796016299265e-06 + }, + "critical_step_samples_ms": [ + 9.310367584228516, + 9.270112037658691, + 9.281855583190918, + 9.308287620544434, + 9.251744270324707 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "zigzag": { + "step_ms": 11.09340763092041, + "prepare_ms": 53.628099034540355, + "one_use_ms": 64.72150666546077, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 2.7903255632299044e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.328703291192055e-05, + "d_pool_score": 3.499598071138621e-05, + "d_ape": 1.9553001191787064e-06 + }, + "critical_step_samples_ms": [ + 11.090751647949219, + 11.105728149414062, + 11.078559875488281, + 11.09340763092041, + 11.114175796508789 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + }, + "planner": { + "step_ms": 9.327072143554688, + "prepare_ms": 10.402858024463058, + "one_use_ms": 19.729930168017745, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.053952717562547e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.891765909667368e-05, + "d_pool_score": 3.83438621871523e-05, + "d_ape": 1.860085527233789e-06 + }, + "critical_step_samples_ms": [ + 9.677375793457031, + 9.265312194824219, + 9.342687606811523, + 9.327072143554688, + 9.311936378479004 + ], + "candidate_workspace_bytes_per_rank": [ + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192, + 1032192 + ] + } + } + }, + "256k-long-tail-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "2edb473a627497dbeee590e633990317905ccda3234a5ed584397d300307f688", + "policies": { + "legacy-contiguous": { + "step_ms": 99.59091186523438, + "prepare_ms": 15.09766501840204, + "one_use_ms": 114.68857688363642, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 98.39814758300781, + 99.7735366821289, + 100.0940170288086, + 99.59091186523438, + 98.48457336425781 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "contiguous": { + "step_ms": 99.29817962646484, + "prepare_ms": 15.911947004497051, + "one_use_ms": 115.2101266309619, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.001110904233409444, + "dq": 0.00039648636058058897, + "dkv": 0.0003898854111343413, + "d_index_q": 0.0005600748026567583, + "d_weights": 0.000507263174267189, + "d_pool_kv": 0.00066199529419305, + "d_pool_score": 0.000678040642214578, + "d_ape": 5.9787640002907704e-06 + }, + "critical_step_samples_ms": [ + 98.9152603149414, + 99.38972473144531, + 99.86335754394531, + 98.69843292236328, + 99.29817962646484 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + }, + "zigzag": { + "step_ms": 115.35395050048828, + "prepare_ms": 69.53235797118396, + "one_use_ms": 184.88630847167224, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0010856430143221802, + "dq": 0.00036992129334733367, + "dkv": 0.0004972554645909258, + "d_index_q": 0.0005008559419775389, + "d_weights": 0.0004722618115882083, + "d_pool_kv": 0.000655208868851891, + "d_pool_score": 0.000654726690762559, + "d_ape": 5.867056633821389e-06 + }, + "critical_step_samples_ms": [ + 115.24447631835938, + 115.35395050048828, + 115.61705780029297, + 115.43267059326172, + 115.2938232421875 + ], + "candidate_workspace_bytes_per_rank": [ + 1073708016, + 1073692980, + 1073648060, + 536804832, + 536760312, + 536814420, + 536809884, + 536864760 + ] + }, + "planner": { + "step_ms": 99.38566589355469, + "prepare_ms": 12.992098927497864, + "one_use_ms": 112.37776482105255, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0010568060715839816, + "dq": 0.00035934651144455286, + "dkv": 0.0004699011755092007, + "d_index_q": 0.0005465734905216791, + "d_weights": 0.0004997240432028573, + "d_pool_kv": 0.0006642917490044416, + "d_pool_score": 0.0006663724937641111, + "d_ape": 6.077476352327514e-06 + }, + "critical_step_samples_ms": [ + 98.25363159179688, + 99.62761688232422, + 99.38566589355469, + 99.48448181152344, + 98.54268646240234 + ], + "candidate_workspace_bytes_per_rank": [ + 1015808, + 2392064, + 4161536, + 418873344, + 536870852, + 536863712, + 536816672, + 1073710020 + ] + } + } + }, + "256k-mixed-1000": { + "documents": 1000, + "tokens": 262144, + "source_record_sha256": "49a55e79bd6c68bf131589a3e5cf5e565916facd5606d7dce7ce57c963f5b941", + "policies": { + "legacy-contiguous": { + "step_ms": 83.68646240234375, + "prepare_ms": 21.228261990472674, + "one_use_ms": 104.91472439281642, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 83.78521728515625, + 83.57373046875, + 83.68646240234375, + 83.2590103149414, + 84.24208068847656 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "contiguous": { + "step_ms": 83.48681640625, + "prepare_ms": 11.41117897350341, + "one_use_ms": 94.89799537975341, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017271400594987304, + "dq": 0.0005238692042413498, + "dkv": 0.0006757752642502718, + "d_index_q": 0.0003682615647156005, + "d_weights": 0.00036747700384791933, + "d_pool_kv": 0.0007684068273359155, + "d_pool_score": 0.0007428614103826032, + "d_ape": 8.689178552655459e-06 + }, + "critical_step_samples_ms": [ + 83.48681640625, + 83.37471771240234, + 84.20655822753906, + 83.2452163696289, + 84.60848236083984 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + }, + "zigzag": { + "step_ms": 95.8419189453125, + "prepare_ms": 70.37775486242026, + "one_use_ms": 166.21967380773276, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.001741600433126006, + "dq": 0.0005280536977853182, + "dkv": 0.000694620863478123, + "d_index_q": 0.00039115890285579556, + "d_weights": 0.0003701005915049805, + "d_pool_kv": 0.0013274674931419044, + "d_pool_score": 0.0013120376158842905, + "d_ape": 9.092468895337202e-06 + }, + "critical_step_samples_ms": [ + 96.7213134765625, + 95.74854278564453, + 96.22220611572266, + 95.78582763671875, + 95.8419189453125 + ], + "candidate_workspace_bytes_per_rank": [ + 536828196, + 536788788, + 518462572, + 480072156, + 441681740, + 403291324, + 364900908, + 326510492 + ] + }, + "planner": { + "step_ms": 83.3333740234375, + "prepare_ms": 12.85100297536701, + "one_use_ms": 96.18437699880451, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0017312793940356549, + "dq": 0.0005248991559417814, + "dkv": 0.0006219782668235454, + "d_index_q": 0.0003919900749740805, + "d_weights": 0.0003715013408516693, + "d_pool_kv": 0.0006638570005121069, + "d_pool_score": 0.0006602028790867173, + "d_ape": 8.978271066130069e-06 + }, + "critical_step_samples_ms": [ + 83.3333740234375, + 83.44710540771484, + 83.32115173339844, + 83.80867004394531, + 82.96355438232422 + ], + "candidate_workspace_bytes_per_rank": [ + 201015296, + 536851532, + 536841448, + 536858804, + 1335296, + 536853360, + 536859912, + 536862244 + ] + } + } + }, + "256k-near-uniform-512": { + "documents": 512, + "tokens": 262144, + "source_record_sha256": "b47a41744461e8029c8fdc96e07b896746a31b8fd492a812fb4fd6051df3b277", + "policies": { + "legacy-contiguous": { + "step_ms": 21.743040084838867, + "prepare_ms": 7.12892203591764, + "one_use_ms": 28.871962120756507, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 21.842655181884766, + 21.743040084838867, + 21.711936950683594, + 21.78006362915039, + 21.708768844604492 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "contiguous": { + "step_ms": 21.762304306030273, + "prepare_ms": 4.859174019657075, + "one_use_ms": 26.62147832568735, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.1016467473129694e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.8276387051590115e-05, + "d_pool_score": 5.540698248431048e-05, + "d_ape": 1.4415619316682367e-06 + }, + "critical_step_samples_ms": [ + 21.784767150878906, + 21.762304306030273, + 21.736831665039062, + 21.763744354248047, + 21.739456176757812 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + }, + "zigzag": { + "step_ms": 24.220352172851562, + "prepare_ms": 30.287406989373267, + "one_use_ms": 54.50775916222483, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.408343901936022e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 6.233960704879021e-05, + "d_pool_score": 4.683946617373621e-05, + "d_ape": 1.4767297408883958e-06 + }, + "critical_step_samples_ms": [ + 24.222047805786133, + 24.204736709594727, + 24.24095916748047, + 24.193023681640625, + 24.220352172851562 + ], + "candidate_workspace_bytes_per_rank": [ + 7901184, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376, + 8421376 + ] + }, + "planner": { + "step_ms": 21.748031616210938, + "prepare_ms": 6.001263973303139, + "one_use_ms": 27.749295589514077, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.143517200502267e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.689493212930183e-05, + "d_pool_score": 5.4293748387891673e-05, + "d_ape": 1.290989423691988e-06 + }, + "critical_step_samples_ms": [ + 21.748031616210938, + 21.722944259643555, + 21.850048065185547, + 21.749183654785156, + 21.720064163208008 + ], + "candidate_workspace_bytes_per_rank": [ + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352, + 8356352 + ] + } + } + }, + "256k-single": { + "documents": 1, + "tokens": 262144, + "source_record_sha256": "a0b6f467329398c2d88275ae6c33a1ac285cf76a5fdc45d076f2749ff36ae59e", + "policies": { + "legacy-contiguous": { + "step_ms": 115.62249755859375, + "prepare_ms": 6.42365706153214, + "one_use_ms": 122.04615462012589, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 116.22777557373047, + 115.62249755859375, + 116.10867309570312, + 115.35679626464844, + 114.52973175048828 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "contiguous": { + "step_ms": 115.4524154663086, + "prepare_ms": 2.2480469197034836, + "one_use_ms": 117.70046238601208, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014863891656528205, + "dq": 0.0005041585809728006, + "dkv": 0.0005403167928287846, + "d_index_q": 0.0007725595847374566, + "d_weights": 0.0006259349484387492, + "d_pool_kv": 0.0007197438097901232, + "d_pool_score": 0.000715737343426364, + "d_ape": 1.7515138877326042e-05 + }, + "critical_step_samples_ms": [ + 115.4524154663086, + 115.16633605957031, + 115.15238189697266, + 116.03148651123047, + 115.69798278808594 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + }, + "zigzag": { + "step_ms": 105.18627166748047, + "prepare_ms": 2.860206994228065, + "one_use_ms": 108.04647866170853, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0.0014852931124954782, + "dq": 0.0005038136738057022, + "dkv": 0.0004547442223581367, + "d_index_q": 0.0004776906182052158, + "d_weights": 0.0004588937679118803, + "d_pool_kv": 0.0006862380126650793, + "d_pool_score": 0.0006758094131979612, + "d_ape": 1.6993433973488575e-05 + }, + "critical_step_samples_ms": [ + 105.17571258544922, + 105.18627166748047, + 105.00962829589844, + 105.96454620361328, + 105.43711853027344 + ], + "candidate_workspace_bytes_per_rank": [ + 1073712188, + 1073660864, + 1073659492, + 1073631792, + 1073736736, + 1073737680, + 1073713936, + 1073713120 + ] + }, + "planner": { + "step_ms": 115.74569702148438, + "prepare_ms": 2.8671040199697018, + "one_use_ms": 118.61280104145408, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0.0014850858958218448, + "dq": 0.0005039087119093382, + "dkv": 0.0006199094601327424, + "d_index_q": 0.0007125454770317754, + "d_weights": 0.0005555715209788749, + "d_pool_kv": 0.0009620010607453248, + "d_pool_score": 0.0009848719645593007, + "d_ape": 1.8798434764171715e-05 + }, + "critical_step_samples_ms": [ + 115.16841888427734, + 115.74569702148438, + 116.19100952148438, + 116.41241455078125, + 114.95225524902344 + ], + "candidate_workspace_bytes_per_rank": [ + 536838144, + 536867212, + 536866384, + 536868040, + 1073713120, + 1073672900, + 1073731400, + 1073737368 + ] + } + } + }, + "256k-uniform-1024": { + "documents": 1024, + "tokens": 262144, + "source_record_sha256": "cca2fdad0f4585eca1da043c29c9689115a510d9af63c2079fd2f1abc9c36c52", + "policies": { + "legacy-contiguous": { + "step_ms": 18.9467830657959, + "prepare_ms": 12.605302035808563, + "one_use_ms": 31.55208510160446, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 0, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "critical_step_samples_ms": [ + 19.008895874023438, + 18.9467830657959, + 18.906911849975586, + 18.95804786682129, + 18.89014434814453 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "contiguous": { + "step_ms": 18.971071243286133, + "prepare_ms": 8.725322084501386, + "one_use_ms": 27.69639332778752, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.668157086472552e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 3.444373471802985e-05, + "d_pool_score": 3.4315328261516696e-05, + "d_ape": 1.2737882398570104e-06 + }, + "critical_step_samples_ms": [ + 18.968896865844727, + 19.191680908203125, + 18.971071243286133, + 18.9803524017334, + 18.957279205322266 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "zigzag": { + "step_ms": 22.57574462890625, + "prepare_ms": 59.226449113339186, + "one_use_ms": 81.80219374224544, + "selected_policy": "zigzag", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.542628572706754e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.743842138582279e-05, + "d_pool_score": 5.184653523102845e-05, + "d_ape": 1.688316934838138e-06 + }, + "critical_step_samples_ms": [ + 22.599584579467773, + 22.599424362182617, + 22.55353546142578, + 22.539520263671875, + 22.57574462890625 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + }, + "planner": { + "step_ms": 18.966144561767578, + "prepare_ms": 10.67610492464155, + "one_use_ms": 29.642249486409128, + "selected_policy": "contiguous", + "max_relative_rms": { + "output": 0, + "dq": 0, + "dkv": 3.695941042091729e-05, + "d_index_q": 0, + "d_weights": 0, + "d_pool_kv": 5.221088970557776e-05, + "d_pool_score": 4.9402061088101224e-05, + "d_ape": 1.3945307090872565e-06 + }, + "critical_step_samples_ms": [ + 19.05878448486328, + 18.966144561767578, + 19.218975067138672, + 18.93948745727539, + 18.935871124267578 + ], + "candidate_workspace_bytes_per_rank": [ + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536, + 4161536 + ] + } + } + } + } + } + }, + "software": { + "torch": "2.12.1+cu130", + "cuda_runtime": "13.0", + "cutlass_dsl": "4.8.1", + "cudnn_frontend": "1.13.0" + }, + "source_sha256": { + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "92cc6f6667e80a7b9098f614d6cddb96694a1c053bb2f03e41d9eefce94fbe36", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "a4a488a9c51821f92d9463085006c919bfb9c1c39efab1ee4b1b904b9f28b059", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "c4b1323eedd22c1f52d6db3db00059f369c998d74a897fef0ec75ab07fc5498f", + "cula/distributed/thd_planner.py": "88cfd88f0abcecd1cc09e1120248fb65a404519783736c886ceadfd6ffbbb9ec", + "cula/hybrid/__init__.py": "d1e8bd8309bc934a8c1e9d3a5d6cd057b513a48d6d9bb5772c57fa2045faa0c3", + "cula/hybrid/csa_cp.py": "92f961d637c284e7070504eee3a483f25e98a0550026488f5a72d4a9642f5113", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_cp_windows.py": "34c2d866f81c1127623ba482adce4ff365a7a1663f66bfdbf717a39cbb7c3beb", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "3b24e12397d6c2a4621f5fb7d2f36c6852b880349d9224e266183646fa36c7d3", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "0cd9570044d980c7c5f9a18fbb213ef3d1e1cf46616ed7d3f3a0de39877a45fa", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/check_csa_cp_selection.py": "b7cbd45ba221d6241dd35f616e7976c869e60d423d4907342189af188dbee936", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "b343916324b406a2d4d0b65b94345cc7c23205fdd695be8791c5f9799d020940", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "9e2c5277766075ee269e18d64ae594dce84c8414fc35ec34bedf6592de126d65", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032" + }, + "native_subgroup_checks": { + "subgroups-final": { + "checks": 64, + "all_ids_equal": true, + "all_results_retained": true, + "max_score_relative_rms": 0.0, + "source_record_sha256": "674df06728eb4c8a379f496018ad03db108b684c8a7be8adf90ea8d8571429ea" + }, + "subgroups-fp32": { + "checks": 64, + "all_ids_equal": true, + "all_results_retained": true, + "max_score_relative_rms": 0.0, + "source_record_sha256": "20991c9a3c1cd5abbb223067c4d71f345116c2a72d57f7d2b4d2e30929d9694d" + } + }, + "fp32_checks": { + "check-many": { + "max_relative_rms": { + "output": 0.001414975617080927, + "dq": 0.002863918896764517, + "dkv": 0.002098844153806567, + "teacher": 0, + "d_index_q": 0, + "d_weights": 0, + "d_kbar": 0, + "d_pool_kv": 0, + "d_pool_score": 0, + "d_ape": 0 + }, + "max_loss_abs_error": 0, + "source_record_sha256": "fee8df5ea2b612f3b5e7825d6b3a1522bd6be4fcc7b5d40fcd2fae8c0f9d182d" + }, + "check-mixed": { + "max_relative_rms": { + "output": 0.0023773587308824062, + "dq": 0.0028181811794638634, + "dkv": 0.0024023125879466534, + "teacher": 1.22032631111324e-07, + "d_index_q": 0.002322548534721136, + "d_weights": 0.0016673828940838575, + "d_kbar": 0.0019576919730752707, + "d_pool_kv": 0.002863020868971944, + "d_pool_score": 0.002832910744473338, + "d_ape": 0.0004040832573082298 + }, + "max_loss_abs_error": 7.4078343459405e-10, + "source_record_sha256": "3733c8a18429571b586faf401c2f5ba3e2b9ca2e30a70b924b06a501e1923db9" + }, + "check-single": { + "max_relative_rms": { + "output": 0.002390037290751934, + "dq": 0.0023644936736673117, + "dkv": 0.0024435562081635, + "teacher": 1.2636910184937733e-07, + "d_index_q": 0.002317695878446102, + "d_weights": 0.0016715021338313818, + "d_kbar": 0.002232952509075403, + "d_pool_kv": 0.0030496628023684025, + "d_pool_score": 0.0030512649100273848, + "d_ape": 0.0003674851031973958 + }, + "max_loss_abs_error": 1.3108092389302328e-09, + "source_record_sha256": "e2bd8016a3fdc9572c2ce76afce91cc30ed88e594916821215b5c62cd9aafd98" + } + } +} diff --git a/docs/results/thd_selection_20261115.json b/docs/results/thd_selection_20261115.json new file mode 100644 index 00000000..3de510ea --- /dev/null +++ b/docs/results/thd_selection_20261115.json @@ -0,0 +1,6544 @@ +{ + "schema_version": 1, + "date": "2026-11-15", + "cudnn_frontend_commit": "855ccd77ba9efed22441fd454f7cc0206a068e94", + "cudnn_backend": "9.20.0.48", + "cutlass_dsl": "4.8.1", + "protocol": "3 warmups, 10 measured iterations; median of per-iteration maximum rank time; full prepare counted once", + "scope": "1 KDA + 1 full CSA forward/backward, independent inputs; not a connected model", + "platforms": { + "h20_cp16": { + "world_size": 16, + "cases": { + "1m-20s2l": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1770.2546997070312, + "min_ms": 1766.39697265625, + "max_ms": 1779.2506103515625, + "allocated_gib": 40.48734903335571, + "reserved_gib": 45.47265625, + "prepare_ms": 9.66643076390028, + "layout_prepare_ms": 9.303988888859749, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.999981328845024, + "max_spans": 21, + "raw_file": "layout-1m-20s2l-contiguous.json", + "raw_sha256": "c27f0de10476d5a5227da3f8eb46c815480ae913c329c852efa1f8b8b418602f" + }, + "planner": { + "total_ms": 1326.9475708007812, + "min_ms": 1323.5238037109375, + "max_ms": 1330.18505859375, + "allocated_gib": 40.48831605911255, + "reserved_gib": 45.951171875, + "prepare_ms": 56.893481872975826, + "layout_prepare_ms": 56.678880006074905, + "policy": "selection-local-4x", + "relative_rms": { + "output": 0.0014370549510826182, + "dq": 0.0007328903431475611, + "dkv": 0.0007098334972614795, + "d_index_q": 0.0007886133158625485, + "d_weights": 0.0007656313854345311, + "d_pool_kv": 0.0009375874547660811, + "d_pool_score": 0.0009411841698845073, + "d_ape": 1.331346387614e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.9998921006917953, + "max_spans": 24, + "raw_file": "layout-1m-20s2l-planner.json", + "raw_sha256": "65c946c6186791db5b35e4c309bebf479114decc0c7ce35e7065967232d63621" + }, + "zigzag": { + "total_ms": 1361.1481323242188, + "min_ms": 1357.9793701171875, + "max_ms": 1367.4815673828125, + "allocated_gib": 40.49222230911255, + "reserved_gib": 45.970703125, + "prepare_ms": 10.714996606111526, + "layout_prepare_ms": 10.436772368848324, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014426966180279143, + "dq": 0.000736235596560469, + "dkv": 0.000665001758953214, + "d_index_q": 0.0007290876335645271, + "d_weights": 0.0007622767518445443, + "d_pool_kv": 0.0009443198974116189, + "d_pool_score": 0.0009462030090884896, + "d_ape": 1.3866521682671272e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 8.731149137020111e-11, + "workspace_gib": 1.9999244809150696, + "max_spans": 44, + "raw_file": "layout-1m-20s2l-zigzag.json", + "raw_sha256": "0baff442ce3f471a823476dc4863c99b60004430a19c311cb608e5e7f7e0847e" + } + }, + "speedup_vs_contiguous": 1.3340803650883695, + "gap_to_best_baseline_pct": -2.5126259744439827, + "prepare_plus_iteration_ms": { + "contiguous": 1779.9211304709315, + "planner": 1383.841052673757, + "zigzag": 1371.8631289303303 + }, + "prepare_inclusive_gap_pct": 0.8731136139482176, + "documents": 22, + "lengths_sha256": "1fbc78fcb26714baab3313b04d1a82e7d4c3c5f2a707fddfa67b2c0893379098" + }, + "1m-count-mixed-1000-1long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1738.2982177734375, + "min_ms": 1732.321044921875, + "max_ms": 1749.4837646484375, + "allocated_gib": 40.49317121505737, + "reserved_gib": 45.59375, + "prepare_ms": 10.79770177602768, + "layout_prepare_ms": 10.190024971961975, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999728836119175, + "max_spans": 135, + "raw_file": "layout-1m-count-mixed-1000-1long-contiguous.json", + "raw_sha256": "77a6c060f06eafec547a93790326e091c6aa01ede84b38849cfe169eb8fbf692" + }, + "planner": { + "total_ms": 1078.8523559570312, + "min_ms": 1075.45654296875, + "max_ms": 1081.654541015625, + "allocated_gib": 41.948423862457275, + "reserved_gib": 45.96875, + "prepare_ms": 57.30481201317161, + "layout_prepare_ms": 56.81393697159365, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014131674788891825, + "dq": 0.0007217496110718596, + "dkv": 0.0006838480550948861, + "d_index_q": 0.0007880740154021596, + "d_weights": 0.0008131424597783085, + "d_pool_kv": 0.0009271833921768376, + "d_pool_score": 0.0009146511551639808, + "d_ape": 3.901471804680105e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.999987006187439, + "max_spans": 137, + "raw_file": "layout-1m-count-mixed-1000-1long-planner.json", + "raw_sha256": "5041bcf57be4ba051f162f08ea08577de37d5eaad5833c4dfde3fb4db49149a7" + }, + "zigzag": { + "total_ms": 1107.8590698242188, + "min_ms": 1105.778564453125, + "max_ms": 1109.240234375, + "allocated_gib": 40.492013454437256, + "reserved_gib": 45.97265625, + "prepare_ms": 73.95509001798928, + "layout_prepare_ms": 73.50210502045229, + "policy": "zigzag", + "relative_rms": { + "output": 0.001396905567037877, + "dq": 0.0007126076884560058, + "dkv": 0.000898063044848878, + "d_index_q": 0.000842882708912927, + "d_weights": 0.0008584308868511431, + "d_pool_kv": 0.0009289185753135385, + "d_pool_score": 0.0009154616825916759, + "d_ape": 4.003750630157111e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.999989103525877, + "max_spans": 1200, + "raw_file": "layout-1m-count-mixed-1000-1long-zigzag.json", + "raw_sha256": "2dc533ddba4e51392912145142e4bd7781c883022f478efebf1d799a8faf6f34" + } + }, + "speedup_vs_contiguous": 1.611247552248633, + "gap_to_best_baseline_pct": -2.6182674906285675, + "prepare_plus_iteration_ms": { + "contiguous": 1749.0959195494652, + "planner": 1136.1571679702029, + "zigzag": 1181.814159842208 + }, + "prepare_inclusive_gap_pct": -3.863297075244143, + "documents": 1000, + "lengths_sha256": "9e67a0b0cf2c6f62c886060fcce16cad37d396e458fe8e31e2ea92de7569371d" + }, + "1m-count-mixed-256-1long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1933.6011962890625, + "min_ms": 1923.75244140625, + "max_ms": 1944.3497314453125, + "allocated_gib": 41.22977256774902, + "reserved_gib": 46.46875, + "prepare_ms": 5.177393555641174, + "layout_prepare_ms": 4.873349331319332, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.4999958872795105, + "max_spans": 52, + "raw_file": "layout-1m-count-mixed-256-1long-contiguous.json", + "raw_sha256": "e220a459b7f099102bcbfc2c44d7340a46e640a12a1032c04384e3c9223c7d20" + }, + "planner": { + "total_ms": 1197.7588500976562, + "min_ms": 1195.3802490234375, + "max_ms": 1202.0479736328125, + "allocated_gib": 41.97810459136963, + "reserved_gib": 48.919921875, + "prepare_ms": 52.76484042406082, + "layout_prepare_ms": 52.50781960785389, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014226647261483673, + "dq": 0.0007180340724526626, + "dkv": 0.0008179497186912406, + "d_index_q": 0.0008058696030484662, + "d_weights": 0.0008432608681242646, + "d_pool_kv": 0.0010930844599177977, + "d_pool_score": 0.0010590800398992991, + "d_ape": 7.770387411027729e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 4.999912668019533, + "max_spans": 53, + "raw_file": "layout-1m-count-mixed-256-1long-planner.json", + "raw_sha256": "95643a4a8536f00a772f49252efdd308acb51c87cb2c242a9297e005537d8ef6" + }, + "zigzag": { + "total_ms": 1223.2321166992188, + "min_ms": 1221.118408203125, + "max_ms": 1227.0054931640625, + "allocated_gib": 40.99202299118042, + "reserved_gib": 46.470703125, + "prepare_ms": 22.380927577614784, + "layout_prepare_ms": 22.075112909078598, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013963022789706349, + "dq": 0.0007049773454525711, + "dkv": 0.0007368037351704907, + "d_index_q": 0.0007336936166034252, + "d_weights": 0.0007946033580117142, + "d_pool_kv": 0.0009276963136814727, + "d_pool_score": 0.0009221220113713877, + "d_ape": 7.63173704818653e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 2.499992474913597, + "max_spans": 307, + "raw_file": "layout-1m-count-mixed-256-1long-zigzag.json", + "raw_sha256": "10922039f1b3ddb1b55635eb368d7ebfc2fc730d4cf0feb151c809d59e0f8543" + } + }, + "speedup_vs_contiguous": 1.6143493292755977, + "gap_to_best_baseline_pct": -2.0824556724605814, + "prepare_plus_iteration_ms": { + "contiguous": 1938.7785898447037, + "planner": 1250.523690521717, + "zigzag": 1245.6130442768335 + }, + "prepare_inclusive_gap_pct": 0.3942352938134608, + "documents": 256, + "lengths_sha256": "013b49129cf177ac7e785bf7126e9e6899c7405866ddd6a51c91d4aec66cc87e" + }, + "1m-count-mixed-512-2long-ascending": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1240.0969848632812, + "min_ms": 1234.22412109375, + "max_ms": 1242.7333984375, + "allocated_gib": 40.953940868377686, + "reserved_gib": 44.96875, + "prepare_ms": 11.49603258818388, + "layout_prepare_ms": 10.997926816344261, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999969005584717, + "max_spans": 418, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-contiguous.json", + "raw_sha256": "191f9c9e6f96e2ccf3acd63dfa1bc38bf1bf31f9e710a83fe1d84279e946e8b0" + }, + "planner": { + "total_ms": 938.5447387695312, + "min_ms": 937.6407470703125, + "max_ms": 941.2828979492188, + "allocated_gib": 42.990283489227295, + "reserved_gib": 46.3046875, + "prepare_ms": 46.396294608712196, + "layout_prepare_ms": 45.94092257320881, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014403781516027449, + "dq": 0.0007327477018683716, + "dkv": 0.0007074783258196851, + "d_index_q": 0.0007088233181552034, + "d_weights": 0.0007418597362617036, + "d_pool_kv": 0.0008261977568685192, + "d_pool_score": 0.0008087736658695805, + "d_ape": 6.546711824353377e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999847412109375, + "max_spans": 422, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-planner.json", + "raw_sha256": "4363781658c77cc3b9a7e42a822c60108c59e74f35d4925f6be61fd1b7927d7c" + }, + "zigzag": { + "total_ms": 947.3248901367188, + "min_ms": 946.660888671875, + "max_ms": 949.937255859375, + "allocated_gib": 41.866726875305176, + "reserved_gib": 50.775390625, + "prepare_ms": 58.15050844103098, + "layout_prepare_ms": 57.70234391093254, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014537235383189086, + "dq": 0.0007407989394821003, + "dkv": 0.0007096909729663449, + "d_index_q": 0.0008041080050001198, + "d_weights": 0.0007943771738968879, + "d_pool_kv": 0.0008371011910827963, + "d_pool_score": 0.0008341807726690167, + "d_ape": 6.293301240637564e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999847412109375, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-zigzag.json", + "raw_sha256": "3d28b4576e300c475c485aec206bf644e380d2a754c1cc239669c5e711197243" + } + }, + "speedup_vs_contiguous": 1.3212976788821988, + "gap_to_best_baseline_pct": -0.926836342906634, + "prepare_plus_iteration_ms": { + "contiguous": 1251.5930174514651, + "planner": 984.9410333782434, + "zigzag": 1005.4753985777497 + }, + "prepare_inclusive_gap_pct": -2.042254363314333, + "documents": 512, + "lengths_sha256": "c8ef7a47c8782c941def4feb26285b36f3ad51a989a3c052ec4e3fd7e8677f0e" + }, + "1m-count-mixed-512-2long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1235.557861328125, + "min_ms": 1233.1134033203125, + "max_ms": 1238.40869140625, + "allocated_gib": 39.73763704299927, + "reserved_gib": 44.96875, + "prepare_ms": 6.890648975968361, + "layout_prepare_ms": 6.548076868057251, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999978207051754, + "max_spans": 86, + "raw_file": "layout-1m-count-mixed-512-2long-contiguous.json", + "raw_sha256": "c5be0e549e5a89c44368a0b4a6ad8b7a56d573be301683298146b145455eea1c" + }, + "planner": { + "total_ms": 932.9089050292969, + "min_ms": 932.2576293945312, + "max_ms": 933.833984375, + "allocated_gib": 39.72488594055176, + "reserved_gib": 46.2265625, + "prepare_ms": 43.387351965066046, + "layout_prepare_ms": 43.043665995355695, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.001390238819945953, + "dq": 0.000701639014897124, + "dkv": 0.0006863240111215172, + "d_index_q": 0.000673929731405553, + "d_weights": 0.0007422031462348568, + "d_pool_kv": 0.000996414348174742, + "d_pool_score": 0.0009960602473099074, + "d_ape": 6.394037327728149e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999865889549255, + "max_spans": 90, + "raw_file": "layout-1m-count-mixed-512-2long-planner.json", + "raw_sha256": "f0a97969490f8e8aeabdcd6190e37924ae901254b2fa1f36786ca10bae0367c7" + }, + "zigzag": { + "total_ms": 942.929443359375, + "min_ms": 942.33837890625, + "max_ms": 944.03173828125, + "allocated_gib": 40.74278402328491, + "reserved_gib": 46.474609375, + "prepare_ms": 40.236313827335835, + "layout_prepare_ms": 39.91378378123045, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013902129191113775, + "dq": 0.0007078508527508515, + "dkv": 0.0007465641023219604, + "d_index_q": 0.0007212337867090088, + "d_weights": 0.0007829665474866085, + "d_pool_kv": 0.0010004758288360228, + "d_pool_score": 0.0009971238645788182, + "d_ape": 6.249562389713757e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999849684536457, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-2long-zigzag.json", + "raw_sha256": "333c20f643fc915448950bc524334b6701b9850dbd17fd0212bf99183a9c9804" + } + }, + "speedup_vs_contiguous": 1.3244142645302799, + "gap_to_best_baseline_pct": -1.062702877786692, + "prepare_plus_iteration_ms": { + "contiguous": 1242.4485103040934, + "planner": 976.2962569943629, + "zigzag": 983.1657571867108 + }, + "prepare_inclusive_gap_pct": -0.698712311950811, + "documents": 512, + "lengths_sha256": "21794dd7586d487ed826710eb069c8ec277ff51b5c88db0b0400e835d0419bcd" + }, + "1m-count-mixed-512-2long-descending": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1241.6589965820312, + "min_ms": 1240.9886474609375, + "max_ms": 1247.15087890625, + "allocated_gib": 41.22573900222778, + "reserved_gib": 45.48828125, + "prepare_ms": 12.419818027410656, + "layout_prepare_ms": 11.910784989595413, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999969005584717, + "max_spans": 418, + "raw_file": "layout-1m-count-mixed-512-2long-descending-contiguous.json", + "raw_sha256": "c1bddd9fac6471e12def2d7dc44740ce5235469ee0f34738a9eaf5ca5234372b" + }, + "planner": { + "total_ms": 933.3216857910156, + "min_ms": 932.8864135742188, + "max_ms": 933.894775390625, + "allocated_gib": 43.259296894073486, + "reserved_gib": 51.81640625, + "prepare_ms": 45.23335298290476, + "layout_prepare_ms": 44.762280012946576, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014545159003805394, + "dq": 0.000743494012154302, + "dkv": 0.0007685056085245936, + "d_index_q": 0.0008703250133279459, + "d_weights": 0.000838913170206235, + "d_pool_kv": 0.0008371936081197537, + "d_pool_score": 0.000825633756631383, + "d_ape": 6.513117921279345e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 1.9999847412109375, + "max_spans": 420, + "raw_file": "layout-1m-count-mixed-512-2long-descending-planner.json", + "raw_sha256": "a948e5b646919a94e00f9e31f6a8dd38cf4aa1b30d734c9d86efa949992ed131" + }, + "zigzag": { + "total_ms": 936.8607788085938, + "min_ms": 936.211669921875, + "max_ms": 938.4207763671875, + "allocated_gib": 43.137155055999756, + "reserved_gib": 51.8203125, + "prepare_ms": 58.16895398311317, + "layout_prepare_ms": 57.69163102377206, + "policy": "zigzag", + "relative_rms": { + "output": 0.001422822688413853, + "dq": 0.0007246148440246227, + "dkv": 0.0007254313535724738, + "d_index_q": 0.0007736511212151291, + "d_weights": 0.0007601264812653758, + "d_pool_kv": 0.0008365214053928546, + "d_pool_score": 0.0008298024805049333, + "d_ape": 6.2789767196702126e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 1.9999847412109375, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-2long-descending-zigzag.json", + "raw_sha256": "a831a51932d12b792df73be5d194891b435766a9e9adfbeea5f44922d4d966d0" + } + }, + "speedup_vs_contiguous": 1.3303655272187225, + "gap_to_best_baseline_pct": -0.37776082611535866, + "prepare_plus_iteration_ms": { + "contiguous": 1254.078814609442, + "planner": 978.5550387739204, + "zigzag": 995.0297327917069 + }, + "prepare_inclusive_gap_pct": -1.6556986665679063, + "documents": 512, + "lengths_sha256": "1355e0fb1ef1d83a855305c16e873372467e69d68007d35dcefa354ea7c6ab1b" + }, + "1m-count-mixed-512-4long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 972.0389709472656, + "min_ms": 970.7367553710938, + "max_ms": 973.3665771484375, + "allocated_gib": 39.3654465675354, + "reserved_gib": 45.3046875, + "prepare_ms": 7.157045416533947, + "layout_prepare_ms": 6.765970028936863, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 86, + "raw_file": "layout-1m-count-mixed-512-4long-contiguous.json", + "raw_sha256": "67e72a989c83401b91323ed83782e3cb11b216a0cd580612b06ed511631e9d92" + }, + "planner": { + "total_ms": 874.1318664550781, + "min_ms": 873.6409301757812, + "max_ms": 875.0445556640625, + "allocated_gib": 39.242146015167236, + "reserved_gib": 44.970703125, + "prepare_ms": 53.79614233970642, + "layout_prepare_ms": 53.46741620451212, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0014319279148353801, + "dq": 0.000730531837343021, + "dkv": 0.0006971483885515934, + "d_index_q": 0.0006813065174239298, + "d_weights": 0.0006861706654843519, + "d_pool_kv": 0.0007879833098417588, + "d_pool_score": 0.0007882555784148795, + "d_ape": 6.418386021629022e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.4999895580112934, + "max_spans": 89, + "raw_file": "layout-1m-count-mixed-512-4long-planner.json", + "raw_sha256": "0b4d24ea5cda106a16d3515fb3d8236d1efc50a131d269ebf4d0e9b3b2f22820" + }, + "zigzag": { + "total_ms": 880.24072265625, + "min_ms": 879.207763671875, + "max_ms": 880.8388671875, + "allocated_gib": 39.36966896057129, + "reserved_gib": 45.740234375, + "prepare_ms": 37.095713429152966, + "layout_prepare_ms": 36.7745952680707, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014058193089270753, + "dq": 0.0007201690410107729, + "dkv": 0.0006351635106884441, + "d_index_q": 0.0006657213660369258, + "d_weights": 0.000676296287009572, + "d_pool_kv": 0.0007949862823955126, + "d_pool_score": 0.0007856044854621893, + "d_ape": 6.393762828290671e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49999282509088516, + "max_spans": 614, + "raw_file": "layout-1m-count-mixed-512-4long-zigzag.json", + "raw_sha256": "587fc508900e40c6333631ad80d10af8a5165c2c75601bde78d44a9798f920c9" + } + }, + "speedup_vs_contiguous": 1.1120049597199062, + "gap_to_best_baseline_pct": -0.6939983624862922, + "prepare_plus_iteration_ms": { + "contiguous": 979.1960163637996, + "planner": 927.9280087947845, + "zigzag": 917.336436085403 + }, + "prepare_inclusive_gap_pct": 1.154600677869011, + "documents": 512, + "lengths_sha256": "74f528dba1eacc05b1dd3ae32df6d6e5fcba98ba8abc06fecf0c0df6bf0aa380" + }, + "1m-count-near-uniform-512": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 452.79539489746094, + "min_ms": 452.1741638183594, + "max_ms": 453.1943359375, + "allocated_gib": 37.642160415649414, + "reserved_gib": 44.06640625, + "prepare_ms": 7.008280954323709, + "layout_prepare_ms": 6.630392977967858, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.07122802734375, + "max_spans": 36, + "raw_file": "layout-1m-count-near-uniform-512-contiguous.json", + "raw_sha256": "62d05651861d7981e13bf43c83846a15306b206ca1e1b66a1f00626397fa6739" + }, + "planner": { + "total_ms": 452.81201171875, + "min_ms": 452.55108642578125, + "max_ms": 453.0105895996094, + "allocated_gib": 37.642160415649414, + "reserved_gib": 44.5390625, + "prepare_ms": 7.0143090561032295, + "layout_prepare_ms": 6.682493723928928, + "policy": "contiguous", + "relative_rms": { + "output": 0.000312570186443386, + "dq": 0.00021878066333155848, + "dkv": 0.00017310433315514652, + "d_index_q": 8.758753491107588e-05, + "d_weights": 8.747408107936404e-05, + "d_pool_kv": 0.0001562169465468653, + "d_pool_score": 0.00014843652847624337, + "d_ape": 2.1200156610156e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.07122802734375, + "max_spans": 36, + "raw_file": "layout-1m-count-near-uniform-512-planner.json", + "raw_sha256": "ae47956396ddb81e5bbbd890d2e31d31a5a5cab5e970a179ea2f3b13364d2d05" + }, + "zigzag": { + "total_ms": 489.8685302734375, + "min_ms": 488.3931884765625, + "max_ms": 490.7578430175781, + "allocated_gib": 37.648831844329834, + "reserved_gib": 45.1796875, + "prepare_ms": 64.41162247210741, + "layout_prepare_ms": 64.05988894402981, + "policy": "zigzag", + "relative_rms": { + "output": 0.0003119673592817592, + "dq": 0.00021851863380101095, + "dkv": 0.00017099683323057972, + "d_index_q": 8.753351513712065e-05, + "d_weights": 8.460601101446695e-05, + "d_pool_kv": 0.0001494269842368115, + "d_pool_score": 0.00018111080123773155, + "d_ape": 2.193984997606667e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.06634521484375, + "max_spans": 1024, + "raw_file": "layout-1m-count-near-uniform-512-zigzag.json", + "raw_sha256": "9c0ac05320b645e3d9290cc65c3e6c92fc0b703a622603743c3d114dd6cd3a92" + } + }, + "speedup_vs_contiguous": 0.9999633030466087, + "gap_to_best_baseline_pct": 0.0036698300106996484, + "prepare_plus_iteration_ms": { + "contiguous": 459.80367585178465, + "planner": 459.82632077485323, + "zigzag": 554.2801527455449 + }, + "prepare_inclusive_gap_pct": 0.004924911273618271, + "documents": 512, + "lengths_sha256": "a883d69122302c1f8f9693929fa0ea899a3e5e33acb0a13cd8340e29afc198a0" + }, + "1m-count-tiny-mix-1000": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 804.4497375488281, + "min_ms": 803.3792114257812, + "max_ms": 804.8482666015625, + "allocated_gib": 39.36701440811157, + "reserved_gib": 45.427734375, + "prepare_ms": 11.593663948588073, + "layout_prepare_ms": 11.099680326879025, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 143, + "raw_file": "layout-1m-count-tiny-mix-1000-contiguous.json", + "raw_sha256": "1dc08db67824ae58a274a6c2e945a5729ca8c097cd1d95ab55166978e3cb5181" + }, + "planner": { + "total_ms": 803.9241638183594, + "min_ms": 803.6096801757812, + "max_ms": 804.8082275390625, + "allocated_gib": 39.36701440811157, + "reserved_gib": 46.46875, + "prepare_ms": 12.909071519970894, + "layout_prepare_ms": 12.430059723556042, + "policy": "contiguous", + "relative_rms": { + "output": 0.0009725538862074621, + "dq": 0.0006188448813383469, + "dkv": 0.0003971155353896982, + "d_index_q": 0.0002914177003272136, + "d_weights": 0.0003071750406410824, + "d_pool_kv": 0.0003651129601905051, + "d_pool_score": 0.0003462786951790806, + "d_ape": 3.5353953008116053e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 143, + "raw_file": "layout-1m-count-tiny-mix-1000-planner.json", + "raw_sha256": "05addf8eabd249e3f02186e0478fd744b54c21e61b18c854750e7c80f2ab61fe" + }, + "zigzag": { + "total_ms": 819.8125, + "min_ms": 818.898193359375, + "max_ms": 820.943603515625, + "allocated_gib": 39.35548257827759, + "reserved_gib": 46.896484375, + "prepare_ms": 70.85726968944073, + "layout_prepare_ms": 70.35546749830246, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009806580554309915, + "dq": 0.0006204996956369439, + "dkv": 0.0003984549191234964, + "d_index_q": 0.00029134836348343544, + "d_weights": 0.0003059298731533832, + "d_pool_kv": 0.000370834144304363, + "d_pool_score": 0.000391578141157335, + "d_ape": 3.161593514832567e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49914050847291946, + "max_spans": 1200, + "raw_file": "layout-1m-count-tiny-mix-1000-zigzag.json", + "raw_sha256": "82adb8554d17767de8049dfbb13a48c79d2ff126bdfecfa730ecfba4629cacc8" + } + }, + "speedup_vs_contiguous": 1.0006537603347714, + "gap_to_best_baseline_pct": -0.06533332114324608, + "prepare_plus_iteration_ms": { + "contiguous": 816.0434014974162, + "planner": 816.8332353383303, + "zigzag": 890.6697696894407 + }, + "prepare_inclusive_gap_pct": 0.09678821487493217, + "documents": 1000, + "lengths_sha256": "2043af46235527c07d70653288a5fbff85cbdc1d61c2df251c4b11c9f2173662" + }, + "1m-count-tiny-mix-512": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 685.9302673339844, + "min_ms": 684.9505615234375, + "max_ms": 686.43310546875, + "allocated_gib": 38.420175552368164, + "reserved_gib": 45.21484375, + "prepare_ms": 7.0847609313204885, + "layout_prepare_ms": 6.701943988446146, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.498516321182251, + "max_spans": 59, + "raw_file": "layout-1m-count-tiny-mix-512-contiguous.json", + "raw_sha256": "a09d8c61624f4cc1af71729054dc7ddf2611dac3c7046af0ae8bd417b04ec30c" + }, + "planner": { + "total_ms": 685.8549194335938, + "min_ms": 685.347900390625, + "max_ms": 685.97705078125, + "allocated_gib": 38.420175552368164, + "reserved_gib": 45.21484375, + "prepare_ms": 6.993969029281288, + "layout_prepare_ms": 6.630199961364269, + "policy": "contiguous", + "relative_rms": { + "output": 0.0010325317851200527, + "dq": 0.0006162005011788455, + "dkv": 0.0003776748189111857, + "d_index_q": 0.0002628102942323622, + "d_weights": 0.0002676192491992476, + "d_pool_kv": 0.00033682698470347753, + "d_pool_score": 0.0003439917375613672, + "d_ape": 6.1279133614286e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.498516321182251, + "max_spans": 59, + "raw_file": "layout-1m-count-tiny-mix-512-planner.json", + "raw_sha256": "0add243c5864581013780f0e5889b80b837e719a555a8ac74d61f354649a2e00" + }, + "zigzag": { + "total_ms": 716.0761108398438, + "min_ms": 715.8760986328125, + "max_ms": 717.7543334960938, + "allocated_gib": 38.55094909667969, + "reserved_gib": 45.19921875, + "prepare_ms": 43.37962158024311, + "layout_prepare_ms": 43.03775541484356, + "policy": "zigzag", + "relative_rms": { + "output": 0.0010499063952199626, + "dq": 0.0006258225003131088, + "dkv": 0.00041073803877893905, + "d_index_q": 0.00032662600157353964, + "d_weights": 0.00031260323144411474, + "d_pool_kv": 0.00035676072352841396, + "d_pool_score": 0.0003572713035249323, + "d_ape": 6.468741565403711e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.37377477437257767, + "max_spans": 614, + "raw_file": "layout-1m-count-tiny-mix-512-zigzag.json", + "raw_sha256": "19cccd84d9f2a66d30d27386dcd71840e0b359724ec570a1bf237e3257b1ee70" + } + }, + "speedup_vs_contiguous": 1.0001098598234928, + "gap_to_best_baseline_pct": -0.010984775563771532, + "prepare_plus_iteration_ms": { + "contiguous": 693.0150282653049, + "planner": 692.848888462875, + "zigzag": 759.4557324200869 + }, + "prepare_inclusive_gap_pct": -0.023973477580374603, + "documents": 512, + "lengths_sha256": "eb5e64b29a22782c3e203c9db504474452dd18b36ac3aa9a4ad991a5af9bdbf9" + }, + "1m-count-uniform-1024": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 304.08592224121094, + "min_ms": 303.8802490234375, + "max_ms": 304.4345703125, + "allocated_gib": 38.423105239868164, + "reserved_gib": 45.3671875, + "prepare_ms": 11.142310046125203, + "layout_prepare_ms": 10.64562500687316, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.03118896484375, + "max_spans": 64, + "raw_file": "layout-1m-count-uniform-1024-contiguous.json", + "raw_sha256": "11561bae83a343dadcdb661feaa53ae03cd8f5eadfd0080792d1f6fc43d05d54" + }, + "planner": { + "total_ms": 303.88612365722656, + "min_ms": 303.5679931640625, + "max_ms": 304.477783203125, + "allocated_gib": 38.423166275024414, + "reserved_gib": 45.9609375, + "prepare_ms": 12.489238986745477, + "layout_prepare_ms": 12.01151602435857, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 4.0017354011135795e-05, + "d_index_q": 0.0, + "d_weights": 1.5643278801343326e-05, + "d_pool_kv": 3.5892032839695286e-05, + "d_pool_score": 4.1180529947898683e-05, + "d_ape": 4.6654965940994376e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.03118896484375, + "max_spans": 64, + "raw_file": "layout-1m-count-uniform-1024-planner.json", + "raw_sha256": "27317d3bcce2e740aa3113a446f2d33f3f4c1fff3fa1014172bb3fa0956a3b5b" + }, + "zigzag": { + "total_ms": 345.7340545654297, + "min_ms": 344.55413818359375, + "max_ms": 346.9438171386719, + "allocated_gib": 38.42808675765991, + "reserved_gib": 45.96484375, + "prepare_ms": 129.5411480241455, + "layout_prepare_ms": 129.09127998864278, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 4.131296766393347e-05, + "d_index_q": 0.0, + "d_weights": 1.1611570293793908e-05, + "d_pool_kv": 4.137243430716575e-05, + "d_pool_score": 7.625351996008623e-05, + "d_ape": 5.545519532233894e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.03118896484375, + "max_spans": 2048, + "raw_file": "layout-1m-count-uniform-1024-zigzag.json", + "raw_sha256": "a572931b350c86621dfd0eeb7dc1ca0e4ffab8bc02da97a47fe8195d874f4b1a" + } + }, + "speedup_vs_contiguous": 1.000657478471145, + "gap_to_best_baseline_pct": -0.06570464772318596, + "prepare_plus_iteration_ms": { + "contiguous": 315.22823228733614, + "planner": 316.37536264397204, + "zigzag": 475.2752025895752 + }, + "prepare_inclusive_gap_pct": 0.363904701146911, + "documents": 1024, + "lengths_sha256": "7d1b6ad860e729dfc1e098d7b89b3e6e85ea0e907b90de52c462e82eb70e9471" + }, + "1m-count-uniform-256": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 549.6219177246094, + "min_ms": 549.1296997070312, + "max_ms": 550.0491943359375, + "allocated_gib": 38.51983880996704, + "reserved_gib": 45.65625, + "prepare_ms": 4.39696479588747, + "layout_prepare_ms": 4.108703695237637, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.12493896484375, + "max_spans": 16, + "raw_file": "layout-1m-count-uniform-256-contiguous.json", + "raw_sha256": "bd6a3011bc09519f88cdbb431d094b4dc9d7f52ae5cc585e0538289d59d65a84" + }, + "planner": { + "total_ms": 549.4464111328125, + "min_ms": 549.179443359375, + "max_ms": 549.8413696289062, + "allocated_gib": 38.51983880996704, + "reserved_gib": 45.0625, + "prepare_ms": 4.716673865914345, + "layout_prepare_ms": 4.421164281666279, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006925854248702454, + "dq": 0.0004392233399569253, + "dkv": 0.00029740960412245063, + "d_index_q": 0.00024004049781900123, + "d_weights": 0.00019391788661221848, + "d_pool_kv": 0.00026141680733655204, + "d_pool_score": 0.0002586076167179979, + "d_ape": 4.22166115570864e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.12493896484375, + "max_spans": 16, + "raw_file": "layout-1m-count-uniform-256-planner.json", + "raw_sha256": "d2eb7ac4c21a145d7a446f6436c735525217ccb0764abc7f8deee85e4d728e6e" + }, + "zigzag": { + "total_ms": 584.0359802246094, + "min_ms": 583.1168212890625, + "max_ms": 585.0107421875, + "allocated_gib": 38.524731159210205, + "reserved_gib": 45.958984375, + "prepare_ms": 32.832130789756775, + "layout_prepare_ms": 32.5444545596838, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006908169770485342, + "dq": 0.00043303490110947876, + "dkv": 0.00027639050441138717, + "d_index_q": 0.00015210646909709564, + "d_weights": 0.00015894974404154317, + "d_pool_kv": 0.0002506036255637015, + "d_pool_score": 0.00026886913645408276, + "d_ape": 4.130315630069566e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.12493896484375, + "max_spans": 512, + "raw_file": "layout-1m-count-uniform-256-zigzag.json", + "raw_sha256": "a3209638e2436bb3bd95b42af0c6ce1f771dd7736f682c207a4df18e76810150" + } + }, + "speedup_vs_contiguous": 1.0003194244028912, + "gap_to_best_baseline_pct": -0.03193224035232811, + "prepare_plus_iteration_ms": { + "contiguous": 554.0188825204968, + "planner": 554.1630849987268, + "zigzag": 616.8681110143661 + }, + "prepare_inclusive_gap_pct": 0.026028441047709627, + "documents": 256, + "lengths_sha256": "2b66265ed16d5c91894fb143f8c654f23f801053a71d4fd69181b0642cf716a7" + }, + "1m-heldout-jittered-768-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 358.52392578125, + "min_ms": 357.597412109375, + "max_ms": 358.72125244140625, + "allocated_gib": 38.9788122177124, + "reserved_gib": 45.28515625, + "prepare_ms": 8.966105058789253, + "layout_prepare_ms": 8.518191985785961, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0449279248714447, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-contiguous.json", + "raw_sha256": "5dcb5213bc8f12b186a0ceaeed228456ac87d83946aba7bb5d5dee019ad26ce1" + }, + "planner": { + "total_ms": 358.4803161621094, + "min_ms": 357.7920837402344, + "max_ms": 358.86279296875, + "allocated_gib": 38.9788122177124, + "reserved_gib": 45.28515625, + "prepare_ms": 10.183462873101234, + "layout_prepare_ms": 9.80326347053051, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.363129604384954e-05, + "d_index_q": 0.0, + "d_weights": 1.1220725449324892e-05, + "d_pool_kv": 8.384201051467693e-05, + "d_pool_score": 5.988107855092994e-05, + "d_ape": 5.636189088678152e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0449279248714447, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-planner.json", + "raw_sha256": "7cd682aa27cc1f41a05c9af6ac44e2d9121ab9dec408343086ba737f0cdbfa91" + }, + "zigzag": { + "total_ms": 397.6351623535156, + "min_ms": 396.7493896484375, + "max_ms": 398.86016845703125, + "allocated_gib": 38.98426389694214, + "reserved_gib": 45.98828125, + "prepare_ms": 95.2249551191926, + "layout_prepare_ms": 94.84434872865677, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.3002213991950895e-05, + "d_index_q": 0.0, + "d_weights": 1.3272182524832172e-05, + "d_pool_kv": 8.348898455953135e-05, + "d_pool_score": 4.427979618573066e-05, + "d_ape": 5.096660004587656e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04387390986084938, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-zigzag.json", + "raw_sha256": "d14853409595353c0dafa9ca64ebddcee9ac78ca1611928c3256058091b1ab4d" + } + }, + "speedup_vs_contiguous": 1.0001216513631976, + "gap_to_best_baseline_pct": -0.012163656594355032, + "prepare_plus_iteration_ms": { + "contiguous": 367.49003084003925, + "planner": 368.6637790352106, + "zigzag": 492.8601174727082 + }, + "prepare_inclusive_gap_pct": 0.3193959282346448, + "documents": 768, + "lengths_sha256": "2384da57da625501af834608e8298b60467638272e4c35dc6ea261713c7eb0d3" + }, + "1m-heldout-jittered-768-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 358.77650451660156, + "min_ms": 358.13946533203125, + "max_ms": 359.1441650390625, + "allocated_gib": 39.808849811553955, + "reserved_gib": 45.8671875, + "prepare_ms": 9.590373025275767, + "layout_prepare_ms": 9.206185990478843, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04477034509181976, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-contiguous.json", + "raw_sha256": "789902a0f27ec3d80ade697ee5da1a97a3c9573fee04bc888f5934378afb9322" + }, + "planner": { + "total_ms": 358.70045471191406, + "min_ms": 358.292236328125, + "max_ms": 359.0645751953125, + "allocated_gib": 39.808849811553955, + "reserved_gib": 45.8671875, + "prepare_ms": 10.364365763962269, + "layout_prepare_ms": 9.964034892618656, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.2490964013625554e-05, + "d_index_q": 0.0, + "d_weights": 1.3651717477780841e-05, + "d_pool_kv": 3.4509005688492454e-05, + "d_pool_score": 3.3288530731895434e-05, + "d_ape": 4.501287051876582e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04477034509181976, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-planner.json", + "raw_sha256": "4c51e8a59b01dad3ad4f98ac18943960178bbe4e4f1cd4a2786e503612769735" + }, + "zigzag": { + "total_ms": 398.11195373535156, + "min_ms": 397.47796630859375, + "max_ms": 399.0572509765625, + "allocated_gib": 39.81116342544556, + "reserved_gib": 46.27734375, + "prepare_ms": 97.08238858729601, + "layout_prepare_ms": 96.70192841440439, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 4.378364012653045e-05, + "d_index_q": 0.0, + "d_weights": 1.5808015694390196e-05, + "d_pool_kv": 3.262385514096063e-05, + "d_pool_score": 3.426303838873765e-05, + "d_ape": 4.392719494088674e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.043894730508327484, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-zigzag.json", + "raw_sha256": "b38910fe602a76d46de8d8a34e7db773bcaedb881686cf6b2318b3c2d2dd98d8" + } + }, + "speedup_vs_contiguous": 1.000212014798667, + "gap_to_best_baseline_pct": -0.02119698579202689, + "prepare_plus_iteration_ms": { + "contiguous": 368.36687754187733, + "planner": 369.06482047587633, + "zigzag": 495.19434232264757 + }, + "prepare_inclusive_gap_pct": 0.18946951437555093, + "documents": 768, + "lengths_sha256": "8884d0c226a9c39788069b745f1fa9b8499da77272f0e098c192c933dea50872" + }, + "1m-heldout-jittered-768-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 359.0693359375, + "min_ms": 358.796142578125, + "max_ms": 359.2643737792969, + "allocated_gib": 40.17335319519043, + "reserved_gib": 46.28125, + "prepare_ms": 9.378077986184508, + "layout_prepare_ms": 8.974294003564864, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.044239819049835205, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-contiguous.json", + "raw_sha256": "d11bdff7eb5b9169b7cc30d74ed4caf21b5506ae4d8761eb04c9a6d7769f1274" + }, + "planner": { + "total_ms": 359.144287109375, + "min_ms": 358.93280029296875, + "max_ms": 359.4841613769531, + "allocated_gib": 40.17335319519043, + "reserved_gib": 46.28125, + "prepare_ms": 10.32171497354284, + "layout_prepare_ms": 9.934334957506508, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.280996475029592e-05, + "d_index_q": 0.0, + "d_weights": 1.2683953507657087e-05, + "d_pool_kv": 3.495714158265876e-05, + "d_pool_score": 3.7603865072635344e-05, + "d_ape": 4.496830226084333e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.044239819049835205, + "max_spans": 51, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-planner.json", + "raw_sha256": "a543fa3f4100c72630f6afd3572f5c0a197ea70b3b70f3202460326a5fe8acf4" + }, + "zigzag": { + "total_ms": 398.9141540527344, + "min_ms": 397.929931640625, + "max_ms": 399.3319091796875, + "allocated_gib": 40.17674732208252, + "reserved_gib": 46.283203125, + "prepare_ms": 97.00689493911341, + "layout_prepare_ms": 96.62403498077765, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.293799597415976e-05, + "d_index_q": 0.0, + "d_weights": 1.7251727926864844e-05, + "d_pool_kv": 3.9521198405270576e-05, + "d_pool_score": 4.615198594900765e-05, + "d_ape": 5.105090459413078e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.04397561401128769, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-zigzag.json", + "raw_sha256": "e84e9c5456c0795d9f4893ed1e9ce4e21ee13233bef2169fbd5128e3f39db425" + } + }, + "speedup_vs_contiguous": 0.9997913062393996, + "gap_to_best_baseline_pct": 0.020873732277726376, + "prepare_plus_iteration_ms": { + "contiguous": 368.4474139236845, + "planner": 369.46600208291784, + "zigzag": 495.9210489918478 + }, + "prepare_inclusive_gap_pct": 0.27645414806583, + "documents": 768, + "lengths_sha256": "484660be7f47772a4ee1c32aa13529e5d4dfa02ba770b46f67c83944d3d16782" + }, + "1m-heldout-lognormal-1000-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 684.9967041015625, + "min_ms": 684.418701171875, + "max_ms": 685.22021484375, + "allocated_gib": 39.45905113220215, + "reserved_gib": 46.2421875, + "prepare_ms": 10.71262452751398, + "layout_prepare_ms": 10.212206281721592, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 114, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-contiguous.json", + "raw_sha256": "fc53cac7b6a0143619cea370851e59a133f109d31f150b72b933cd217c1be2e1" + }, + "planner": { + "total_ms": 684.5411682128906, + "min_ms": 684.1309814453125, + "max_ms": 684.8707275390625, + "allocated_gib": 39.45905113220215, + "reserved_gib": 46.2421875, + "prepare_ms": 12.842495983932167, + "layout_prepare_ms": 12.371807999443263, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006604366951019903, + "dq": 0.0004884184178103446, + "dkv": 0.0002441307659025493, + "d_index_q": 0.00017720154893536717, + "d_weights": 0.0001965275658067716, + "d_pool_kv": 0.0002635452150096701, + "d_pool_score": 0.00025799601845620344, + "d_ape": 3.7463702901400385e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 114, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-planner.json", + "raw_sha256": "56ea84e33677dd134f4c00ed9632d537cc5df9e7dd030f54cd7be1144bed6490" + }, + "zigzag": { + "total_ms": 700.9886169433594, + "min_ms": 700.3733520507812, + "max_ms": 701.34033203125, + "allocated_gib": 39.49775457382202, + "reserved_gib": 46.5, + "prepare_ms": 120.16775365918875, + "layout_prepare_ms": 119.67392452061176, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006636288524415482, + "dq": 0.0004896882885293665, + "dkv": 0.0002431552086838662, + "d_index_q": 0.00017754175506518597, + "d_weights": 0.00019942199427568522, + "d_pool_kv": 0.0002599323951245154, + "d_pool_score": 0.00026026892084907567, + "d_ape": 3.772762659927586e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.3476196713745594, + "max_spans": 1628, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-zigzag.json", + "raw_sha256": "dcc44346b9d207f0c56919c0e0ca3ab35a5842c9b77986c69219d4bb25b3e689" + } + }, + "speedup_vs_contiguous": 1.0006654616403292, + "gap_to_best_baseline_pct": -0.0665019095631103, + "prepare_plus_iteration_ms": { + "contiguous": 695.7093286290765, + "planner": 697.3836641968228, + "zigzag": 821.1563706025481 + }, + "prepare_inclusive_gap_pct": 0.24066596477090307, + "documents": 1000, + "lengths_sha256": "f9552804d64123e178b417a374ec673ead4524cfdca92d036e2c49d2242a4462" + }, + "1m-heldout-lognormal-1000-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 763.6994323730469, + "min_ms": 763.1716918945312, + "max_ms": 764.0468139648438, + "allocated_gib": 40.053749561309814, + "reserved_gib": 46.6796875, + "prepare_ms": 12.699897983111441, + "layout_prepare_ms": 12.214038986712694, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 113, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-contiguous.json", + "raw_sha256": "a508d050d66317a33bed9a5c2ef06bf8b4089f1ebfcfedf36b64c479ab8e8527" + }, + "planner": { + "total_ms": 763.3740234375, + "min_ms": 762.8582153320312, + "max_ms": 763.90478515625, + "allocated_gib": 40.053749561309814, + "reserved_gib": 46.6796875, + "prepare_ms": 42.369666101876646, + "layout_prepare_ms": 41.90250305691734, + "policy": "contiguous", + "relative_rms": { + "output": 0.0008212079038689687, + "dq": 0.0005803233894914287, + "dkv": 0.0003178547583325437, + "d_index_q": 0.00024748702763004843, + "d_weights": 0.0002898346991169899, + "d_pool_kv": 0.00037226213041429183, + "d_pool_score": 0.0003641375152335028, + "d_ape": 3.5695850024897406e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 113, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-planner.json", + "raw_sha256": "9ba3060c3dbb7e99ec6a0205903bd122391f8254146785881fd03798dff85481" + }, + "zigzag": { + "total_ms": 760.7092590332031, + "min_ms": 760.1244506835938, + "max_ms": 761.7604370117188, + "allocated_gib": 40.262939453125, + "reserved_gib": 47.68359375, + "prepare_ms": 111.01808212697506, + "layout_prepare_ms": 110.55053118616343, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008240264857030691, + "dq": 0.000580487816087925, + "dkv": 0.00031653498036455385, + "d_index_q": 0.0002474463983232176, + "d_weights": 0.0002914637802480217, + "d_pool_kv": 0.00036201676107118724, + "d_pool_score": 0.00034605030505217594, + "d_ape": 3.512907903393745e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.4038251228630543, + "max_spans": 1617, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-zigzag.json", + "raw_sha256": "ef3337acf693424381cffe7c0f51de3ee5e65139739c476a8ce96f4e858ccbf6" + } + }, + "speedup_vs_contiguous": 1.0004262771925112, + "gap_to_best_baseline_pct": 0.3502999828980169, + "prepare_plus_iteration_ms": { + "contiguous": 776.3993303561583, + "planner": 805.7436895393766, + "zigzag": 871.7273411601782 + }, + "prepare_inclusive_gap_pct": 3.779544628117737, + "documents": 1000, + "lengths_sha256": "4860b063d4278394f20027f7e1f50de3bf6080e17af8500ea52a852f8f4840bf" + }, + "1m-heldout-lognormal-1000-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 736.0016479492188, + "min_ms": 735.2288818359375, + "max_ms": 737.14697265625, + "allocated_gib": 40.697120666503906, + "reserved_gib": 47.77734375, + "prepare_ms": 11.603533290326595, + "layout_prepare_ms": 11.127131059765816, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 99, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-contiguous.json", + "raw_sha256": "13efa21223b67e20bef071619cac3497a2dbb9ee457fc8770998898b2c556cf8" + }, + "planner": { + "total_ms": 736.3946838378906, + "min_ms": 734.807861328125, + "max_ms": 737.6834716796875, + "allocated_gib": 40.697792053222656, + "reserved_gib": 47.77734375, + "prepare_ms": 12.68920605070889, + "layout_prepare_ms": 12.206138984765857, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006433898524578946, + "dq": 0.0005110624096845733, + "dkv": 0.00026048934040463144, + "d_index_q": 0.00019598883463898154, + "d_weights": 0.000228863130253541, + "d_pool_kv": 0.000301853480114409, + "d_pool_score": 0.000297540264113885, + "d_ape": 4.091797227406095e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 99, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-planner.json", + "raw_sha256": "b4a6625eec2d393f2c092be2cadeb171c27c1f3177e3c92750bde9602a1a1490" + }, + "zigzag": { + "total_ms": 735.1014709472656, + "min_ms": 734.1682739257812, + "max_ms": 735.7579956054688, + "allocated_gib": 41.009629249572754, + "reserved_gib": 47.47265625, + "prepare_ms": 107.5354665517807, + "layout_prepare_ms": 107.07924794405699, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006420663728273947, + "dq": 0.0005102186802942892, + "dkv": 0.00027717264627597377, + "d_index_q": 0.00019555019132033196, + "d_weights": 0.0002284901651517546, + "d_pool_kv": 0.00030743783418931916, + "d_pool_score": 0.0003033758473351343, + "d_ape": 3.978015480011178e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49536293372511864, + "max_spans": 1584, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-zigzag.json", + "raw_sha256": "72dd0d69778869f26fb583147e76cf249ffa64625bf1c376320dbbca473fa59a" + } + }, + "speedup_vs_contiguous": 0.9994662700623754, + "gap_to_best_baseline_pct": 0.17592304487685784, + "prepare_plus_iteration_ms": { + "contiguous": 747.6051812395453, + "planner": 749.0838898885995, + "zigzag": 842.6369374990463 + }, + "prepare_inclusive_gap_pct": 0.19779272350712862, + "documents": 1000, + "lengths_sha256": "a790875510a4369ef17daa8104180cfe9404b90dfa3567b8652ea1435220224c" + }, + "1m-heldout-lognormal-257-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 666.2662658691406, + "min_ms": 665.9510498046875, + "max_ms": 666.6072998046875, + "allocated_gib": 39.30071496963501, + "reserved_gib": 46.08203125, + "prepare_ms": 4.745754005853087, + "layout_prepare_ms": 4.4530140003189445, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.463335856795311, + "max_spans": 26, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-contiguous.json", + "raw_sha256": "4dcabc1f4bce11d18a2d57a37fe93b11f4837c03a49ec63c70a03fae41f78a68" + }, + "planner": { + "total_ms": 666.1408996582031, + "min_ms": 665.9105224609375, + "max_ms": 666.6676635742188, + "allocated_gib": 39.30185651779175, + "reserved_gib": 46.08203125, + "prepare_ms": 4.790247010532767, + "layout_prepare_ms": 4.503867996390909, + "policy": "contiguous", + "relative_rms": { + "output": 0.0009385242732204369, + "dq": 0.0005866022976175567, + "dkv": 0.00033332166942627665, + "d_index_q": 0.0002301383322868088, + "d_weights": 0.0002399194161665796, + "d_pool_kv": 0.0002873366391359497, + "d_pool_score": 0.0002897758095958852, + "d_ape": 4.674192029871579e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.463335856795311, + "max_spans": 26, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-planner.json", + "raw_sha256": "3d2579cce44e52c4cbe0ae529d9f6cb1dc6c2cae81df1d33a5e012991896d356" + }, + "zigzag": { + "total_ms": 694.9446716308594, + "min_ms": 694.22998046875, + "max_ms": 695.3518676757812, + "allocated_gib": 39.167086124420166, + "reserved_gib": 46.083984375, + "prepare_ms": 32.833476550877094, + "layout_prepare_ms": 32.56948012858629, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009470464886583148, + "dq": 0.0005880154208656388, + "dkv": 0.00036135005526755385, + "d_index_q": 0.00022518916873369107, + "d_weights": 0.00024094943810340667, + "d_pool_kv": 0.00029319441343405016, + "d_pool_score": 0.0003009505490939196, + "d_ape": 4.417832424111279e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.3246483616530895, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-zigzag.json", + "raw_sha256": "c9d586768d9e15167c314f37e3321489cba239d2ae0aced4c94349a4e8696776" + } + }, + "speedup_vs_contiguous": 1.0001881977386493, + "gap_to_best_baseline_pct": -0.01881623269248678, + "prepare_plus_iteration_ms": { + "contiguous": 671.0120198749937, + "planner": 670.9311466687359, + "zigzag": 727.7781481817365 + }, + "prepare_inclusive_gap_pct": -0.012052422886987824, + "documents": 257, + "lengths_sha256": "22124dfa90691813499ba5fff3956732dd43551f4504a48fc46bbb35d7aa1141" + }, + "1m-heldout-lognormal-257-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 786.7605895996094, + "min_ms": 786.594482421875, + "max_ms": 787.9214477539062, + "allocated_gib": 39.929911613464355, + "reserved_gib": 46.1015625, + "prepare_ms": 4.555713036097586, + "layout_prepare_ms": 4.276246996596456, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 30, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-contiguous.json", + "raw_sha256": "01d703a50f75290adad1292fa978fa35a81b8ccd5af057a28ed4963da8cabe0f" + }, + "planner": { + "total_ms": 786.9058532714844, + "min_ms": 785.9761352539062, + "max_ms": 787.4791259765625, + "allocated_gib": 39.929789543151855, + "reserved_gib": 46.1015625, + "prepare_ms": 23.62960804020986, + "layout_prepare_ms": 23.33585696760565, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011388051659333431, + "dq": 0.0006564971734834571, + "dkv": 0.0004364387193785559, + "d_index_q": 0.000436640407897889, + "d_weights": 0.000383328993036165, + "d_pool_kv": 0.000393598589728051, + "d_pool_score": 0.00039426697683359133, + "d_ape": 5.08645882566227e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 30, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-planner.json", + "raw_sha256": "13efb3b4d74cf45df412460339bbe0a63c20173fcbbd81e46215bc1fd7d38ca2" + }, + "zigzag": { + "total_ms": 774.9671936035156, + "min_ms": 774.4342651367188, + "max_ms": 775.7385864257812, + "allocated_gib": 39.93263816833496, + "reserved_gib": 47.46875, + "prepare_ms": 33.89150928705931, + "layout_prepare_ms": 33.621191047132015, + "policy": "zigzag", + "relative_rms": { + "output": 0.001143064133573562, + "dq": 0.0006575020348302019, + "dkv": 0.00043892282645587147, + "d_index_q": 0.00035061364043188786, + "d_weights": 0.00038229657460199485, + "d_pool_kv": 0.00039741900440744445, + "d_pool_score": 0.00039195516172039574, + "d_ape": 5.082429890969267e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49996573850512505, + "max_spans": 513, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-zigzag.json", + "raw_sha256": "01378420248c6dd51aa82cdbc474318bd904cf12df08386f5a49b6ad410b73b7" + } + }, + "speedup_vs_contiguous": 0.9998153989180902, + "gap_to_best_baseline_pct": 1.5405374274561456, + "prepare_plus_iteration_ms": { + "contiguous": 791.316302635707, + "planner": 810.5354613116942, + "zigzag": 808.8587028905749 + }, + "prepare_inclusive_gap_pct": 2.428758084721916, + "documents": 257, + "lengths_sha256": "0b8ffc4c0c7294d43fa7ae5f3ee55faaef51c2f526387b1efe2d60aca24ecf20" + }, + "1m-heldout-lognormal-257-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 792.4435119628906, + "min_ms": 792.0433959960938, + "max_ms": 792.8259887695312, + "allocated_gib": 40.349995136260986, + "reserved_gib": 47.30859375, + "prepare_ms": 4.70799405593425, + "layout_prepare_ms": 4.365537024568766, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 28, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-contiguous.json", + "raw_sha256": "a5ddb06a51ad5b0301088d49f0e7592fe7c03e5e1c4bfeef1c8f61f90b7c4426" + }, + "planner": { + "total_ms": 792.43798828125, + "min_ms": 791.5970458984375, + "max_ms": 793.3018188476562, + "allocated_gib": 40.349995136260986, + "reserved_gib": 47.30859375, + "prepare_ms": 4.752930020913482, + "layout_prepare_ms": 4.501574032474309, + "policy": "contiguous", + "relative_rms": { + "output": 0.001227522930499042, + "dq": 0.0006738229944997933, + "dkv": 0.0005094287079412098, + "d_index_q": 0.000377814752024436, + "d_weights": 0.0004027883643553077, + "d_pool_kv": 0.0004238109628606652, + "d_pool_score": 0.0005288219532210354, + "d_ape": 5.122086960896457e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 28, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-planner.json", + "raw_sha256": "198812023d665581c0d6b7835fc3e313c600c914000996a05f6c4dae6ff024bd" + }, + "zigzag": { + "total_ms": 789.9468688964844, + "min_ms": 789.567626953125, + "max_ms": 790.7005615234375, + "allocated_gib": 40.58277988433838, + "reserved_gib": 46.7109375, + "prepare_ms": 33.60013302881271, + "layout_prepare_ms": 33.324415038805455, + "policy": "zigzag", + "relative_rms": { + "output": 0.0012300934864721798, + "dq": 0.0006733776529817055, + "dkv": 0.0005092419932310364, + "d_index_q": 0.0003789284595906472, + "d_weights": 0.0004030942702858978, + "d_pool_kv": 0.00042597496878129035, + "d_pool_score": 0.0005224856786887003, + "d_ape": 4.969292163654653e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.46208810433745384, + "max_spans": 513, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-zigzag.json", + "raw_sha256": "084681bc0cd40e931ce7c1d70dc1712899139a7b97c431ef9feb9066829e0b35" + } + }, + "speedup_vs_contiguous": 1.0000069704907164, + "gap_to_best_baseline_pct": 0.3153527766045272, + "prepare_plus_iteration_ms": { + "contiguous": 797.1515060188249, + "planner": 797.1909183021635, + "zigzag": 823.5470019252971 + }, + "prepare_inclusive_gap_pct": 0.004944139607210296, + "documents": 257, + "lengths_sha256": "1b0bd973931c10a52c40d767fb8a267929d55b8a7e94ed85145c4ab27033efda" + }, + "1m-heldout-tiny-tail-768-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1157.9805297851562, + "min_ms": 1154.6259765625, + "max_ms": 1161.591064453125, + "allocated_gib": 41.0916953086853, + "reserved_gib": 48.49609375, + "prepare_ms": 10.18020004266873, + "layout_prepare_ms": 9.75555699551478, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.9999936819076538, + "max_spans": 182, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-contiguous.json", + "raw_sha256": "91ed22c0ece3fdda9e0af9b39d07862c32bc0ec7844dbaddac47961b6d34899d" + }, + "planner": { + "total_ms": 887.6731567382812, + "min_ms": 886.7418823242188, + "max_ms": 888.07470703125, + "allocated_gib": 41.17432641983032, + "reserved_gib": 49.134765625, + "prepare_ms": 60.14263443648815, + "layout_prepare_ms": 59.783799573779106, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.001392233060533727, + "dq": 0.0007103945696792637, + "dkv": 0.0006237114201774019, + "d_index_q": 0.0007110733774308313, + "d_weights": 0.0007290097887728568, + "d_pool_kv": 0.0008730318487869236, + "d_pool_score": 0.0008682559631854369, + "d_ape": 6.470404000014752e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 1.8399817384779453, + "max_spans": 186, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-planner.json", + "raw_sha256": "da74a5fecf18cfdd36af78e14717511d3f5a627256de5cf71e13209e75995a9b" + }, + "zigzag": { + "total_ms": 905.5602416992188, + "min_ms": 905.1878051757812, + "max_ms": 906.0325317382812, + "allocated_gib": 42.47430372238159, + "reserved_gib": 46.845703125, + "prepare_ms": 58.144996874034405, + "layout_prepare_ms": 57.731485925614834, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013693331780748614, + "dq": 0.0007012683661039539, + "dkv": 0.0006250740418910341, + "d_index_q": 0.0006958873963356512, + "d_weights": 0.000717607521930308, + "d_pool_kv": 0.0008696228817925042, + "d_pool_score": 0.0008796289615144255, + "d_ape": 6.436690481241068e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 1.9739395715296268, + "max_spans": 894, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-zigzag.json", + "raw_sha256": "18264c92f06f169c8730c50c9612f06ba2c64c3ab8d51799b0fba94dbe51dbbc" + } + }, + "speedup_vs_contiguous": 1.304512275712052, + "gap_to_best_baseline_pct": -1.9752506942413484, + "prepare_plus_iteration_ms": { + "contiguous": 1168.160729827825, + "planner": 947.8157911747694, + "zigzag": 963.7052385732532 + }, + "prepare_inclusive_gap_pct": -1.6487870733179588, + "documents": 768, + "lengths_sha256": "759566912504d164342d1556047348e640f650e2bd969cace033c5f384cd1e9f" + }, + "1m-heldout-tiny-tail-768-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1348.0639038085938, + "min_ms": 1345.923828125, + "max_ms": 1351.31689453125, + "allocated_gib": 41.892484188079834, + "reserved_gib": 47.6171875, + "prepare_ms": 9.628218016587198, + "layout_prepare_ms": 9.187710005789995, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.49999376013875, + "max_spans": 127, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-contiguous.json", + "raw_sha256": "f23e6d23f6186c4dfeeb3396f0e4902d4f550ce4a0fecce0007f4279f8dddd96" + }, + "planner": { + "total_ms": 959.8637390136719, + "min_ms": 959.2150268554688, + "max_ms": 960.6021728515625, + "allocated_gib": 43.086970806121826, + "reserved_gib": 49.92578125, + "prepare_ms": 62.3437559697777, + "layout_prepare_ms": 61.92620098590851, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014097002039817799, + "dq": 0.0007193239552737513, + "dkv": 0.0007043678097197213, + "d_index_q": 0.0008185716297708999, + "d_weights": 0.0008139468703262711, + "d_pool_kv": 0.0009509534037469282, + "d_pool_score": 0.0009260895711974831, + "d_ape": 6.186036509744377e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 2.999843694269657, + "max_spans": 134, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-planner.json", + "raw_sha256": "fed7483a29b50600ee22b7dd13bd9565c6371f32c0b7d9867f7b9e39df31bac3" + }, + "zigzag": { + "total_ms": 970.1784362792969, + "min_ms": 968.9087524414062, + "max_ms": 971.3029174804688, + "allocated_gib": 43.80655336380005, + "reserved_gib": 49.220703125, + "prepare_ms": 52.959115244448185, + "layout_prepare_ms": 52.52271518111229, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014112870474365484, + "dq": 0.0007248772146562982, + "dkv": 0.0007172076699478802, + "d_index_q": 0.0007868982065434454, + "d_weights": 0.0007947178818619413, + "d_pool_kv": 0.0009190684436244239, + "d_pool_score": 0.000898462375877342, + "d_ape": 5.798520906604598e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 2.9999621137976646, + "max_spans": 900, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-zigzag.json", + "raw_sha256": "a5e3c36bc6ff75b804f68f235bbdc01ca47ae018f6e4a2d37f1f1527631586ce" + } + }, + "speedup_vs_contiguous": 1.4044325762256893, + "gap_to_best_baseline_pct": -1.0631752757959245, + "prepare_plus_iteration_ms": { + "contiguous": 1357.692121825181, + "planner": 1022.2074949834496, + "zigzag": 1023.1375515237451 + }, + "prepare_inclusive_gap_pct": -0.09090239517749454, + "documents": 768, + "lengths_sha256": "747a3b346f6fb67d2a1bda28b2c1431fae38be2adb70abab6f64c52541fce124" + }, + "1m-heldout-tiny-tail-768-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1590.6795043945312, + "min_ms": 1584.950927734375, + "max_ms": 1595.4150390625, + "allocated_gib": 42.541175365448, + "reserved_gib": 48.34375, + "prepare_ms": 10.44242299394682, + "layout_prepare_ms": 9.98386600986123, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999874122440815, + "max_spans": 192, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-contiguous.json", + "raw_sha256": "d40a92d0689cf9f689f3c42b6b3bdbd185f0eb0158a1d35fc12a603622060f7a" + }, + "planner": { + "total_ms": 1043.1842651367188, + "min_ms": 1041.0396728515625, + "max_ms": 1050.0198974609375, + "allocated_gib": 46.02216053009033, + "reserved_gib": 52.56640625, + "prepare_ms": 73.80954397376627, + "layout_prepare_ms": 73.403590999078, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0013978769968494877, + "dq": 0.0007126548832640084, + "dkv": 0.0005994592647928282, + "d_index_q": 0.0006814399394547405, + "d_weights": 0.0007170806907142871, + "d_pool_kv": 0.0008913555060706143, + "d_pool_score": 0.0008884188541226811, + "d_ape": 6.953670537953373e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999817803502083, + "max_spans": 194, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-planner.json", + "raw_sha256": "5780aacb4562ec565e2a2a011af8572cd0f482563ff53094b55f7d10c9a5f0f5" + }, + "zigzag": { + "total_ms": 1054.1594848632812, + "min_ms": 1053.013916015625, + "max_ms": 1055.3916015625, + "allocated_gib": 45.27772665023804, + "reserved_gib": 50.9765625, + "prepare_ms": 62.31703143566847, + "layout_prepare_ms": 61.90841551870108, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013928374079413585, + "dq": 0.0007163195568851283, + "dkv": 0.0006183839712719321, + "d_index_q": 0.0007805775784666594, + "d_weights": 0.0007601769275057709, + "d_pool_kv": 0.0008918821804395749, + "d_pool_score": 0.0008784295268673588, + "d_ape": 7.135205817860119e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.9999601282179356, + "max_spans": 892, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-zigzag.json", + "raw_sha256": "f7c6e53ffc1874b39853489c2911068ce53115b45f331148f965adda8dbba092" + } + }, + "speedup_vs_contiguous": 1.5248308065555976, + "gap_to_best_baseline_pct": -1.041134656013265, + "prepare_plus_iteration_ms": { + "contiguous": 1601.121927388478, + "planner": 1116.993809110485, + "zigzag": 1116.4765162989497 + }, + "prepare_inclusive_gap_pct": 0.04633261909081643, + "documents": 768, + "lengths_sha256": "e879dc84790636c8379bc654c1bf98f0f184183c998e485cd8cf5a1df8e1e686" + }, + "1m-heldout-trimodal-513-seed20261114": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1590.93896484375, + "min_ms": 1587.810302734375, + "max_ms": 1597.840576171875, + "allocated_gib": 42.64446830749512, + "reserved_gib": 48.48828125, + "prepare_ms": 7.761615968775004, + "layout_prepare_ms": 7.369155006017536, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.999986007809639, + "max_spans": 158, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-contiguous.json", + "raw_sha256": "b79765dd4a8f05f94432c920fec00478564e03860cc14e4701403733835e6341" + }, + "planner": { + "total_ms": 1123.5488891601562, + "min_ms": 1121.196533203125, + "max_ms": 1124.972412109375, + "allocated_gib": 44.79515075683594, + "reserved_gib": 51.78125, + "prepare_ms": 72.7891989517957, + "layout_prepare_ms": 72.46607198612764, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0014105699808921169, + "dq": 0.0007211853655796001, + "dkv": 0.00076317319296445, + "d_index_q": 0.0008273633359754639, + "d_weights": 0.000839848530016898, + "d_pool_kv": 0.0010063930178328682, + "d_pool_score": 0.0009932757448149157, + "d_ape": 7.00794864677479e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.999897550791502, + "max_spans": 162, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-planner.json", + "raw_sha256": "819ca7d68179708b71824ddac20148db07f896b4f26e0964df375bcb645ad7d6" + }, + "zigzag": { + "total_ms": 1143.2892456054688, + "min_ms": 1139.7706298828125, + "max_ms": 1147.6728515625, + "allocated_gib": 44.624155044555664, + "reserved_gib": 50.60546875, + "prepare_ms": 45.48390599666163, + "layout_prepare_ms": 45.11003097286448, + "policy": "zigzag", + "relative_rms": { + "output": 0.001423907749754376, + "dq": 0.0007267030659625826, + "dkv": 0.0008249438553013025, + "d_index_q": 0.0007623419003029103, + "d_weights": 0.0007973129116156198, + "d_pool_kv": 0.0010020190272064045, + "d_pool_score": 0.0009955530954732558, + "d_ape": 6.664383819341363e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999913454055786, + "max_spans": 616, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-zigzag.json", + "raw_sha256": "28325435bf8b7964b07d4f4716e6f170883b247d4e3cc4408f22a533473c1c24" + } + }, + "speedup_vs_contiguous": 1.4159944264045012, + "gap_to_best_baseline_pct": -1.726628368209504, + "prepare_plus_iteration_ms": { + "contiguous": 1598.700580812525, + "planner": 1196.338088111952, + "zigzag": 1188.7731516021304 + }, + "prepare_inclusive_gap_pct": 0.636365020494134, + "documents": 513, + "lengths_sha256": "3e4264b20b69bf872e0287259db21e68128815011e3a225f7f002b5e26266a7b" + }, + "1m-heldout-trimodal-513-seed20261115": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1565.771240234375, + "min_ms": 1562.936767578125, + "max_ms": 1571.0311279296875, + "allocated_gib": 42.291651248931885, + "reserved_gib": 47.84375, + "prepare_ms": 7.783631037455052, + "layout_prepare_ms": 7.3381830006837845, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999883584678173, + "max_spans": 140, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-contiguous.json", + "raw_sha256": "865ef22abb24e72f9ac9bc8df39d87636f0b8b2c00487563c5ee7d5517132236" + }, + "planner": { + "total_ms": 1138.5536499023438, + "min_ms": 1137.5244140625, + "max_ms": 1141.59814453125, + "allocated_gib": 43.47452640533447, + "reserved_gib": 50.501953125, + "prepare_ms": 65.18473598407581, + "layout_prepare_ms": 64.8464719997719, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0014311802780475353, + "dq": 0.0007300040993747967, + "dkv": 0.0007274725527564618, + "d_index_q": 0.0007625024247133749, + "d_weights": 0.0007744764822715035, + "d_pool_kv": 0.001020879848512074, + "d_pool_score": 0.0009976675758443746, + "d_ape": 7.196140335950982e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 2.9999768882989883, + "max_spans": 144, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-planner.json", + "raw_sha256": "6dca6f101f5597b0bdda06a3fcb03153c2ffa2db0bb9f1d338a6b9df97968853" + }, + "zigzag": { + "total_ms": 1154.8492431640625, + "min_ms": 1152.441650390625, + "max_ms": 1156.752685546875, + "allocated_gib": 44.93105983734131, + "reserved_gib": 50.974609375, + "prepare_ms": 44.415701006073505, + "layout_prepare_ms": 44.05278398189694, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014446751397546181, + "dq": 0.0007365494400272126, + "dkv": 0.0006853989864517794, + "d_index_q": 0.0008626266028998927, + "d_weights": 0.0009449086875199775, + "d_pool_kv": 0.0009599962727863646, + "d_pool_score": 0.0009406345768126211, + "d_ape": 6.54012337861862e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 3.9999019987881184, + "max_spans": 616, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-zigzag.json", + "raw_sha256": "e84e2afa237164871da19461e39333b654d6d8e7d12c39de24ff5e995b28035c" + } + }, + "speedup_vs_contiguous": 1.3752283349745307, + "gap_to_best_baseline_pct": -1.4110580543891582, + "prepare_plus_iteration_ms": { + "contiguous": 1573.55487127183, + "planner": 1203.7383858864196, + "zigzag": 1199.264944170136 + }, + "prepare_inclusive_gap_pct": 0.3730152989153801, + "documents": 513, + "lengths_sha256": "d11985e121377244a83f6693db589940ae4899c1bf48610d0e00341a06f525bd" + }, + "1m-heldout-trimodal-513-seed20261116": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1917.0702514648438, + "min_ms": 1912.1605224609375, + "max_ms": 1927.1201171875, + "allocated_gib": 43.041377544403076, + "reserved_gib": 49.6796875, + "prepare_ms": 13.605939981061965, + "layout_prepare_ms": 13.198564003687352, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.4999979436397552, + "max_spans": 125, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-contiguous.json", + "raw_sha256": "757713ac12769cf75f84e27b9fc0bad211974344e489f5dd20775b2b8514074a" + }, + "planner": { + "total_ms": 1195.0348510742188, + "min_ms": 1192.755615234375, + "max_ms": 1196.207275390625, + "allocated_gib": 45.410380840301514, + "reserved_gib": 49.587890625, + "prepare_ms": 62.280201993416995, + "layout_prepare_ms": 61.91658799070865, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.001453102960327942, + "dq": 0.0007394295376428531, + "dkv": 0.0006929703357612946, + "d_index_q": 0.0009285016240026516, + "d_weights": 0.0008356778914921855, + "d_pool_kv": 0.0009733506318998237, + "d_pool_score": 0.0009755384220336541, + "d_ape": 5.943132432689433e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 4.99997153878212, + "max_spans": 129, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-planner.json", + "raw_sha256": "313bc655ca0f1021f66224cbb9a4a96431c19bb13b3a695417a065b5ed3b5c6a" + }, + "zigzag": { + "total_ms": 1225.0924682617188, + "min_ms": 1222.8372802734375, + "max_ms": 1226.3341064453125, + "allocated_gib": 46.19820690155029, + "reserved_gib": 51.845703125, + "prepare_ms": 48.36773098213598, + "layout_prepare_ms": 47.98452300019562, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014586838423307257, + "dq": 0.000749034668781768, + "dkv": 0.0008060485699208124, + "d_index_q": 0.0008707924287472155, + "d_weights": 0.0008722376455450429, + "d_pool_kv": 0.0013020207402422673, + "d_pool_score": 0.0012741966886479401, + "d_ape": 6.281001290077127e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 4.999932385981083, + "max_spans": 616, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-zigzag.json", + "raw_sha256": "a444f890fab0b9c9e3b8c69cccefebca5a7b9c905700e58d5da5d6b9e9f9e078" + } + }, + "speedup_vs_contiguous": 1.6041961033534597, + "gap_to_best_baseline_pct": -2.453497835159224, + "prepare_plus_iteration_ms": { + "contiguous": 1930.6761914459057, + "planner": 1257.3150530676357, + "zigzag": 1273.4601992438547 + }, + "prepare_inclusive_gap_pct": -1.267817100668367, + "documents": 513, + "lengths_sha256": "30e8a9b80449ecc324bba6134d04db805dac074f3b7693579d6f2afb153135f7" + }, + "1m-near-uniform": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 657.8585815429688, + "min_ms": 657.422119140625, + "max_ms": 658.462158203125, + "allocated_gib": 38.75661849975586, + "reserved_gib": 45.703125, + "prepare_ms": 2.901871979702264, + "layout_prepare_ms": 2.6385170058347285, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.3607950210571289, + "max_spans": 4, + "raw_file": "layout-1m-near-uniform-contiguous.json", + "raw_sha256": "4839fa9922f01fd5680c089e3b287c4e3dfea81d886b85f0b416cfde52d871d7" + }, + "planner": { + "total_ms": 657.8680419921875, + "min_ms": 657.4669189453125, + "max_ms": 658.1314697265625, + "allocated_gib": 38.75661849975586, + "reserved_gib": 45.703125, + "prepare_ms": 2.4814242497086525, + "layout_prepare_ms": 2.266555093228817, + "policy": "contiguous", + "relative_rms": { + "output": 0.0012094462774270827, + "dq": 0.0006690625087143198, + "dkv": 0.00050289701248222, + "d_index_q": 0.00048048109350015526, + "d_weights": 0.0003585406867666776, + "d_pool_kv": 0.000397641522095392, + "d_pool_score": 0.00037759271103066154, + "d_ape": 8.173075437996835e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.3607950210571289, + "max_spans": 4, + "raw_file": "layout-1m-near-uniform-planner.json", + "raw_sha256": "7c0b7122a26409e447f407526a46a39586a9b82b65095a27c193733633e0167c" + }, + "zigzag": { + "total_ms": 691.0231323242188, + "min_ms": 690.33544921875, + "max_ms": 692.6437377929688, + "allocated_gib": 38.89805316925049, + "reserved_gib": 45.986328125, + "prepare_ms": 14.919791370630264, + "layout_prepare_ms": 14.696252532303333, + "policy": "zigzag", + "relative_rms": { + "output": 0.0012156282361116832, + "dq": 0.0006701063181955824, + "dkv": 0.0004998448482890624, + "d_index_q": 0.0003623673282831218, + "d_weights": 0.0003377786590599061, + "d_pool_kv": 0.0003491324172495882, + "d_pool_score": 0.0003654630032104385, + "d_ape": 7.500653892418262e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.4973450005054474, + "max_spans": 128, + "raw_file": "layout-1m-near-uniform-zigzag.json", + "raw_sha256": "8e09ad7be40af8ce81ae3a2b8b76cfdab794b1a3e02270d70ee82dbc5c85ea16" + } + }, + "speedup_vs_contiguous": 0.9999856195336833, + "gap_to_best_baseline_pct": 0.001438067311765323, + "prepare_plus_iteration_ms": { + "contiguous": 660.760453522671, + "planner": 660.3494662418962, + "zigzag": 705.942923694849 + }, + "prepare_inclusive_gap_pct": -0.06219913413155753, + "documents": 64, + "lengths_sha256": "f7b3e262fd896136c35c22e4766cae5a4b416484124973745b17cbc8ceac3850" + }, + "1m-power-law": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 1788.4425659179688, + "min_ms": 1784.1978759765625, + "max_ms": 1796.81005859375, + "allocated_gib": 42.91158723831177, + "reserved_gib": 47.84375, + "prepare_ms": 3.3689523115754128, + "layout_prepare_ms": 3.1191427260637283, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999989978969097, + "max_spans": 4, + "raw_file": "layout-1m-power-law-contiguous.json", + "raw_sha256": "ce4146f1ee3890865780f6784326c03488e5affca35da0a224ee79f45d3043fa" + }, + "planner": { + "total_ms": 1195.981201171875, + "min_ms": 1193.8533935546875, + "max_ms": 1202.4031982421875, + "allocated_gib": 42.91142702102661, + "reserved_gib": 47.951171875, + "prepare_ms": 64.25684876739979, + "layout_prepare_ms": 64.0103267505765, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.00141111190665047, + "dq": 0.0007189686564165684, + "dkv": 0.000768608027980026, + "d_index_q": 0.000982448407481547, + "d_weights": 0.0009287907382984111, + "d_pool_kv": 0.0010326563504508956, + "d_pool_score": 0.0010357392149652606, + "d_ape": 1.2755589527582849e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 3.999922826886177, + "max_spans": 11, + "raw_file": "layout-1m-power-law-planner.json", + "raw_sha256": "ed160493d20ca4d75fedbdf8083b23940a454980ff880cff8ca5c6925f5a428a" + }, + "zigzag": { + "total_ms": 1200.6126708984375, + "min_ms": 1199.58984375, + "max_ms": 1202.20849609375, + "allocated_gib": 42.416353702545166, + "reserved_gib": 47.84765625, + "prepare_ms": 10.712767951190472, + "layout_prepare_ms": 10.503863915801048, + "policy": "zigzag", + "relative_rms": { + "output": 0.0013937638936558827, + "dq": 0.0007097413346802662, + "dkv": 0.0008191421111825079, + "d_index_q": 0.0008236355513542162, + "d_weights": 0.0008566196450316358, + "d_pool_kv": 0.001021948285677823, + "d_pool_score": 0.0010267541408195688, + "d_ape": 1.1541836768590658e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 1.9999937117099762, + "max_spans": 14, + "raw_file": "layout-1m-power-law-zigzag.json", + "raw_sha256": "ecf17cd4721ee756c606f9097f902764309154dcb89f9046d88de584c1b3f6ac" + } + }, + "speedup_vs_contiguous": 1.4953768204429752, + "gap_to_best_baseline_pct": -0.38575885785852027, + "prepare_plus_iteration_ms": { + "contiguous": 1791.8115182295442, + "planner": 1260.2380499392748, + "zigzag": 1211.325438849628 + }, + "prepare_inclusive_gap_pct": 4.03794137569653, + "documents": 7, + "lengths_sha256": "b787057eb34fe32408b2fe31d07c9bc6642c12e889e2e2a42e790f32a7ce58d8" + }, + "1m-single": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 2908.836669921875, + "min_ms": 2895.02099609375, + "max_ms": 2922.881103515625, + "allocated_gib": 44.911484241485596, + "reserved_gib": 49.34375, + "prepare_ms": 5.531359056476504, + "layout_prepare_ms": 5.292983027175069, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999951247125864, + "max_spans": 1, + "raw_file": "layout-1m-single-contiguous.json", + "raw_sha256": "bdbbfaac7bef04b46a9356c86ea45d5723b298e95ccd426c7ea2b748e7af47cd" + }, + "planner": { + "total_ms": 1926.761962890625, + "min_ms": 1924.40087890625, + "max_ms": 1931.9456787109375, + "allocated_gib": 44.916353702545166, + "reserved_gib": 49.97265625, + "prepare_ms": 60.06631493801251, + "layout_prepare_ms": 59.86356601351872, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014038787207327147, + "dq": 0.0007154462314895009, + "dkv": 0.0006833642438317521, + "d_index_q": 0.0007211015679207741, + "d_weights": 0.0008126837022277005, + "d_pool_kv": 0.0009996668460791477, + "d_pool_score": 0.0009953564906126419, + "d_ape": 1.471933089292079e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 3.2014213502407074e-10, + "workspace_gib": 3.999988541007042, + "max_spans": 2, + "raw_file": "layout-1m-single-planner.json", + "raw_sha256": "916b072e7cf7a7079caa13195995c67c587bba2a0b6715a365ff577f812e822f" + }, + "zigzag": { + "total_ms": 1934.275390625, + "min_ms": 1925.960205078125, + "max_ms": 1942.73828125, + "allocated_gib": 44.816391944885254, + "reserved_gib": 49.865234375, + "prepare_ms": 8.57052201172337, + "layout_prepare_ms": 8.357521030120552, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014390151905128753, + "dq": 0.000735736599938945, + "dkv": 0.0006618938605150542, + "d_index_q": 0.000861860733717636, + "d_weights": 0.0008535259313058091, + "d_pool_kv": 0.0010403120797250875, + "d_pool_score": 0.0010515623317991771, + "d_ape": 1.4940338041906002e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.4551915228366852e-10, + "workspace_gib": 3.999988541007042, + "max_spans": 2, + "raw_file": "layout-1m-single-zigzag.json", + "raw_sha256": "93948f98a3acecba7d0deaabfaf4f6c8e589d19e6247ffbf4a55316ad41596e9" + } + }, + "speedup_vs_contiguous": 1.5097021458519413, + "gap_to_best_baseline_pct": -0.3884362987189416, + "prepare_plus_iteration_ms": { + "contiguous": 2914.3680289783515, + "planner": 1986.8282778286375, + "zigzag": 1942.8459126367234 + }, + "prepare_inclusive_gap_pct": 2.263811293826379, + "documents": 1, + "lengths_sha256": "df57f4ef94807b0499d37da5e83d5c5333272a5ebcad42ac3dbec6ac94a30bf2" + }, + "1m-uniform": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 653.5117797851562, + "min_ms": 653.2141723632812, + "max_ms": 658.2883911132812, + "allocated_gib": 38.895822048187256, + "reserved_gib": 45.234375, + "prepare_ms": 3.467416390776634, + "layout_prepare_ms": 3.2254476100206375, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49993896484375, + "max_spans": 4, + "raw_file": "layout-1m-uniform-contiguous.json", + "raw_sha256": "6e865c7f586f9301303f2978436088bbabe9721f239ecfc52fe9935174fcd326" + }, + "planner": { + "total_ms": 657.158935546875, + "min_ms": 656.90869140625, + "max_ms": 657.7572021484375, + "allocated_gib": 38.895822048187256, + "reserved_gib": 45.234375, + "prepare_ms": 2.456046990118921, + "layout_prepare_ms": 2.2205039858818054, + "policy": "contiguous", + "relative_rms": { + "output": 0.001222877084560653, + "dq": 0.0006775747367604897, + "dkv": 0.0005307056800977869, + "d_index_q": 0.00037191161926160743, + "d_weights": 0.00033510128340201917, + "d_pool_kv": 0.000352034253727824, + "d_pool_score": 0.000378997248118142, + "d_ape": 7.817866314212561e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49993896484375, + "max_spans": 4, + "raw_file": "layout-1m-uniform-planner.json", + "raw_sha256": "5a4b2455ed9656af314a3a2f0fc539935d551041e550d382f50644f680352a1f" + }, + "zigzag": { + "total_ms": 690.3211975097656, + "min_ms": 689.617431640625, + "max_ms": 690.9261474609375, + "allocated_gib": 38.90070724487305, + "reserved_gib": 45.986328125, + "prepare_ms": 90.23720677942038, + "layout_prepare_ms": 90.0002857670188, + "policy": "zigzag", + "relative_rms": { + "output": 0.0012208882109481808, + "dq": 0.0006700987030479299, + "dkv": 0.0005020949017264413, + "d_index_q": 0.0003544317603985004, + "d_weights": 0.0003364373695151482, + "d_pool_kv": 0.0003600490455313742, + "d_pool_score": 0.00036589665566245076, + "d_ape": 7.339361484699418e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49993896484375, + "max_spans": 128, + "raw_file": "layout-1m-uniform-zigzag.json", + "raw_sha256": "9e4cd5238a8c4b8f0b44732c138bbe8e870fe81f8ed269602153d5853f90e48a" + } + }, + "speedup_vs_contiguous": 0.9944501161523678, + "gap_to_best_baseline_pct": 0.5580856955505364, + "prepare_plus_iteration_ms": { + "contiguous": 656.9791961759329, + "planner": 659.6149825369939, + "zigzag": 780.558404289186 + }, + "prepare_inclusive_gap_pct": 0.40119784255012725, + "documents": 64, + "lengths_sha256": "dd1b660aba10a40fca5d0224dbbcfdec6e38447a6ae6c9aadae9aa252935fd2e" + } + } + }, + "b300_cp8": { + "world_size": 8, + "cases": { + "1m-20s2l": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 606.7195434570312, + "min_ms": 603.2216186523438, + "max_ms": 607.1049194335938, + "allocated_gib": 68.95169401168823, + "reserved_gib": 71.357421875, + "prepare_ms": 8.157755946740508, + "layout_prepare_ms": 7.820633007213473, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999997615814209, + "max_spans": 21, + "raw_file": "layout-1m-20s2l-contiguous.json", + "raw_sha256": "6e60bf1ead83ddf7d64e883c230ed1e7808be48a09e885988971479496baf23c" + }, + "planner": { + "total_ms": 523.0918579101562, + "min_ms": 521.7444458007812, + "max_ms": 524.9149780273438, + "allocated_gib": 71.21805715560913, + "reserved_gib": 81.74609375, + "prepare_ms": 25.480400072410703, + "layout_prepare_ms": 25.222072028554976, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.00114268760664425, + "dq": 0.0003795518734018198, + "dkv": 0.0004977505257510024, + "d_index_q": 0.0006354303346022228, + "d_weights": 0.0005348152030303236, + "d_pool_kv": 0.0006389127848179412, + "d_pool_score": 0.0006350322727140332, + "d_ape": 1.0398483891015381e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999911297112703, + "max_spans": 24, + "raw_file": "layout-1m-20s2l-planner.json", + "raw_sha256": "0d057d4d9bb3eae1c4ec23e927636eb0251e442a5592de852e5c8bf16c3384eb" + }, + "zigzag": { + "total_ms": 525.017578125, + "min_ms": 523.6755981445312, + "max_ms": 528.3817138671875, + "allocated_gib": 69.14970111846924, + "reserved_gib": 72.060546875, + "prepare_ms": 8.265939075499773, + "layout_prepare_ms": 7.958356058225036, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011220897239465242, + "dq": 0.0003615562187011278, + "dkv": 0.0005358599327513749, + "d_index_q": 0.0006301410117376193, + "d_weights": 0.0005489266029995818, + "d_pool_kv": 0.0006605039288327338, + "d_pool_score": 0.0006744354766970948, + "d_ape": 1.1430774781154517e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.9999856054782867, + "max_spans": 44, + "raw_file": "layout-1m-20s2l-zigzag.json", + "raw_sha256": "231c5cd7327bd55911a5ee839f891ca84e62b0004a55a2886176e160f17e915a" + } + }, + "speedup_vs_contiguous": 1.1598718930189094, + "gap_to_best_baseline_pct": -0.3667915694786972, + "prepare_plus_iteration_ms": { + "contiguous": 614.8772994037718, + "planner": 548.572257982567, + "zigzag": 533.2835172004998 + }, + "prepare_inclusive_gap_pct": 2.8669066807701427, + "documents": 22, + "lengths_sha256": "1fbc78fcb26714baab3313b04d1a82e7d4c3c5f2a707fddfa67b2c0893379098" + }, + "1m-count-mixed-1000-1long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 564.9183959960938, + "min_ms": 562.6470947265625, + "max_ms": 567.5447387695312, + "allocated_gib": 69.90887594223022, + "reserved_gib": 72.322265625, + "prepare_ms": 9.918121038936079, + "layout_prepare_ms": 9.384725010022521, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999988861382008, + "max_spans": 262, + "raw_file": "layout-1m-count-mixed-1000-1long-contiguous.json", + "raw_sha256": "09eddcebc906090d96c39180c67440f1a35211d78349812b6a948624cc0a4ab9" + }, + "planner": { + "total_ms": 466.26866149902344, + "min_ms": 465.1852722167969, + "max_ms": 466.9646301269531, + "allocated_gib": 72.88241815567017, + "reserved_gib": 85.33203125, + "prepare_ms": 37.03177708666772, + "layout_prepare_ms": 36.51334100868553, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0009270768706657761, + "dq": 0.00031215269029789, + "dkv": 0.00046040820270757207, + "d_index_q": 0.0004667214906071487, + "d_weights": 0.00046785144097795687, + "d_pool_kv": 0.0007079749914338807, + "d_pool_score": 0.0006910569760609635, + "d_ape": 3.5272129210754033e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 5.999798700213432, + "max_spans": 264, + "raw_file": "layout-1m-count-mixed-1000-1long-planner.json", + "raw_sha256": "bcb863f548a77b3da2675aa1f1476a4964e4226890939bdeeb4eb9d49a178e92" + }, + "zigzag": { + "total_ms": 480.1496124267578, + "min_ms": 479.004638671875, + "max_ms": 482.7588195800781, + "allocated_gib": 72.79625606536865, + "reserved_gib": 74.830078125, + "prepare_ms": 85.75332898180932, + "layout_prepare_ms": 85.18884400837123, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009499978231453273, + "dq": 0.00035834556965688485, + "dkv": 0.0005074644981803778, + "d_index_q": 0.0006145701352548918, + "d_weights": 0.0005349551113659254, + "d_pool_kv": 0.0006670691629931602, + "d_pool_score": 0.0006652888839264348, + "d_ape": 3.2772513683446074e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 5.999901540577412, + "max_spans": 2000, + "raw_file": "layout-1m-count-mixed-1000-1long-zigzag.json", + "raw_sha256": "8cd57f99a5c1ef4180c616f58a86bcc4b8e82c272b2bfdab18ca97f7d8215054" + } + }, + "speedup_vs_contiguous": 1.211572731866469, + "gap_to_best_baseline_pct": -2.890963684751857, + "prepare_plus_iteration_ms": { + "contiguous": 574.8365170350298, + "planner": 503.30043858569115, + "zigzag": 565.9029414085671 + }, + "prepare_inclusive_gap_pct": -11.062409866090206, + "documents": 1000, + "lengths_sha256": "9e67a0b0cf2c6f62c886060fcce16cad37d396e458fe8e31e2ea92de7569371d" + }, + "1m-count-mixed-256-1long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 646.4150695800781, + "min_ms": 639.7410278320312, + "max_ms": 655.7701416015625, + "allocated_gib": 70.46625900268555, + "reserved_gib": 72.87890625, + "prepare_ms": 6.630808115005493, + "layout_prepare_ms": 6.256341002881527, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 4.999974433332682, + "max_spans": 82, + "raw_file": "layout-1m-count-mixed-256-1long-contiguous.json", + "raw_sha256": "00e0cf7d510294d34b536ba15a711f2c242982c64dae404f8df89b142b09c14b" + }, + "planner": { + "total_ms": 494.0136413574219, + "min_ms": 492.4333190917969, + "max_ms": 495.97784423828125, + "allocated_gib": 74.01397752761841, + "reserved_gib": 85.20703125, + "prepare_ms": 18.05205293931067, + "layout_prepare_ms": 17.699307994917035, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0008858106682604503, + "dq": 0.00045696975731111025, + "dkv": 0.0006497224056940322, + "d_index_q": 0.0006953071842500477, + "d_weights": 0.0006144128886141594, + "d_pool_kv": 0.0008463621505370647, + "d_pool_score": 0.0008385926850077365, + "d_ape": 7.137320680575466e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 7.999998513609171, + "max_spans": 84, + "raw_file": "layout-1m-count-mixed-256-1long-planner.json", + "raw_sha256": "839a52e070e1e1f5f24fbfac53fcd848de76b3cad5141e8afd574f262278abf3" + }, + "zigzag": { + "total_ms": 509.91094970703125, + "min_ms": 507.9485168457031, + "max_ms": 510.84820556640625, + "allocated_gib": 70.80725765228271, + "reserved_gib": 73.716796875, + "prepare_ms": 21.465071011334658, + "layout_prepare_ms": 21.089443005621433, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008941719266047247, + "dq": 0.00045941777187373734, + "dkv": 0.0005930797558927764, + "d_index_q": 0.0006706491690614671, + "d_weights": 0.0005955728663222536, + "d_pool_kv": 0.0008320355873393238, + "d_pool_score": 0.0008225265210542579, + "d_ape": 6.629130868855286e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 4.999996654689312, + "max_spans": 512, + "raw_file": "layout-1m-count-mixed-256-1long-zigzag.json", + "raw_sha256": "8e8b0a3e7aafb5f925681b06acf0c4ede83d0cc9e2f163a550714ec0753f757b" + } + }, + "speedup_vs_contiguous": 1.3084963965851155, + "gap_to_best_baseline_pct": -3.1176636545544145, + "prepare_plus_iteration_ms": { + "contiguous": 653.0458776950836, + "planner": 512.0656942967325, + "zigzag": 531.3760207183659 + }, + "prepare_inclusive_gap_pct": -3.6340229270277913, + "documents": 256, + "lengths_sha256": "013b49129cf177ac7e785bf7126e9e6899c7405866ddd6a51c91d4aec66cc87e" + }, + "1m-count-mixed-512-2long-ascending": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 466.8989715576172, + "min_ms": 466.17083740234375, + "max_ms": 468.3266906738281, + "allocated_gib": 69.80380439758301, + "reserved_gib": 84.18359375, + "prepare_ms": 22.160860942676663, + "layout_prepare_ms": 21.723491954617202, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999922811985016, + "max_spans": 432, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-contiguous.json", + "raw_sha256": "9e87c991926de8af250f166e40a77b1f38cce04235dbcc5d3e3595c64f6c959d" + }, + "planner": { + "total_ms": 421.4059600830078, + "min_ms": 419.9235534667969, + "max_ms": 423.1059875488281, + "allocated_gib": 73.5907506942749, + "reserved_gib": 88.091796875, + "prepare_ms": 26.00089693441987, + "layout_prepare_ms": 25.468059931881726, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0011625036828467048, + "dq": 0.00048191146093208825, + "dkv": 0.0005925134624210264, + "d_index_q": 0.0005005708418548132, + "d_weights": 0.0004610351395769133, + "d_pool_kv": 0.0006021968088168659, + "d_pool_score": 0.0005987209876557188, + "d_ape": 5.704240967664213e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.778191454708576, + "max_spans": 435, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-planner.json", + "raw_sha256": "ea633766e5684664cc4b4390e982912be90d03e5688c48cf1d2c410af16394a7" + }, + "zigzag": { + "total_ms": 434.58563232421875, + "min_ms": 432.279052734375, + "max_ms": 437.4580993652344, + "allocated_gib": 71.56598663330078, + "reserved_gib": 73.9140625, + "prepare_ms": 55.45506288763136, + "layout_prepare_ms": 54.934649961069226, + "policy": "zigzag", + "relative_rms": { + "output": 0.001145584907702924, + "dq": 0.0004733690371043385, + "dkv": 0.0005199240863022013, + "d_index_q": 0.0005926175683857409, + "d_weights": 0.0005237273746309723, + "d_pool_kv": 0.0006443768894823862, + "d_pool_score": 0.0006404921576780437, + "d_ape": 5.340775707345109e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.9998051151633263, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-2long-ascending-zigzag.json", + "raw_sha256": "46185171c19639efdff42f646abb228b268b7538d5f55ab168009f12267a3ec6" + } + }, + "speedup_vs_contiguous": 1.1079553109919191, + "gap_to_best_baseline_pct": -3.032698566384806, + "prepare_plus_iteration_ms": { + "contiguous": 489.05983250029385, + "planner": 447.4068570174277, + "zigzag": 490.0406952118501 + }, + "prepare_inclusive_gap_pct": -8.516948789254975, + "documents": 512, + "lengths_sha256": "c8ef7a47c8782c941def4feb26285b36f3ad51a989a3c052ec4e3fd7e8677f0e" + }, + "1m-count-mixed-512-2long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 441.9642791748047, + "min_ms": 439.4814758300781, + "max_ms": 444.86492919921875, + "allocated_gib": 67.59718465805054, + "reserved_gib": 70.009765625, + "prepare_ms": 6.5072940196841955, + "layout_prepare_ms": 6.10797805711627, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999815858900547, + "max_spans": 128, + "raw_file": "layout-1m-count-mixed-512-2long-contiguous.json", + "raw_sha256": "6da77918492109977a8174da04e87b5773ed494d66935b745ac20e5a1651341b" + }, + "planner": { + "total_ms": 407.34478759765625, + "min_ms": 404.76300048828125, + "max_ms": 410.2059326171875, + "allocated_gib": 69.54749727249146, + "reserved_gib": 81.146484375, + "prepare_ms": 32.38745790440589, + "layout_prepare_ms": 32.009240007027984, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0009130715838757859, + "dq": 0.0003953446079186865, + "dkv": 0.0005196069178357984, + "d_index_q": 0.0004475866982705158, + "d_weights": 0.0004382092502930036, + "d_pool_kv": 0.0006609964862132346, + "d_pool_score": 0.0006553244279760129, + "d_ape": 5.201641577139356e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 2.999816305935383, + "max_spans": 132, + "raw_file": "layout-1m-count-mixed-512-2long-planner.json", + "raw_sha256": "684838bb2153ab31197ba4fd95b7a86a2568fc9c8b8a2d26e1440f56b0318aa4" + }, + "zigzag": { + "total_ms": 421.9330749511719, + "min_ms": 420.48248291015625, + "max_ms": 424.0693359375, + "allocated_gib": 68.32417440414429, + "reserved_gib": 71.236328125, + "prepare_ms": 40.53927294444293, + "layout_prepare_ms": 40.1057000271976, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008742053584959008, + "dq": 0.00038871127807538956, + "dkv": 0.0006647450765884712, + "d_index_q": 0.0005054744827599346, + "d_weights": 0.00046845562206280025, + "d_pool_kv": 0.0007495320656450234, + "d_pool_score": 0.0007417334133141131, + "d_ape": 5.2839636515700575e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 2.9998051151633263, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-2long-zigzag.json", + "raw_sha256": "fc7f6bc7a78e939c6e007c5d7d6d844e526b54be053c23ba56a787fa19d3beb4" + } + }, + "speedup_vs_contiguous": 1.084988178641782, + "gap_to_best_baseline_pct": -3.45748845482281, + "prepare_plus_iteration_ms": { + "contiguous": 448.4715731944889, + "planner": 439.73224550206214, + "zigzag": 462.4723478956148 + }, + "prepare_inclusive_gap_pct": -1.9486915592388576, + "documents": 512, + "lengths_sha256": "21794dd7586d487ed826710eb069c8ec277ff51b5c88db0b0400e835d0419bcd" + }, + "1m-count-mixed-512-2long-descending": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 466.68701171875, + "min_ms": 464.418212890625, + "max_ms": 470.1160583496094, + "allocated_gib": 69.80540466308594, + "reserved_gib": 84.1796875, + "prepare_ms": 10.136506985872984, + "layout_prepare_ms": 9.61173593532294, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999922811985016, + "max_spans": 432, + "raw_file": "layout-1m-count-mixed-512-2long-descending-contiguous.json", + "raw_sha256": "a1e5aff7a16615d0379729e41f357707511f7f08670af3e2ae65f0f2f5826a1b" + }, + "planner": { + "total_ms": 412.10789489746094, + "min_ms": 410.8602600097656, + "max_ms": 413.2406005859375, + "allocated_gib": 73.81171321868896, + "reserved_gib": 88.30859375, + "prepare_ms": 35.53258301690221, + "layout_prepare_ms": 34.98343599494547, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0011776489787716086, + "dq": 0.0004764408764781611, + "dkv": 0.0005078191487493045, + "d_index_q": 0.0005257317147264598, + "d_weights": 0.0004880231426663937, + "d_pool_kv": 0.0006782394860014693, + "d_pool_score": 0.0006898723741323015, + "d_ape": 5.235521354946445e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.998665437102318, + "max_spans": 435, + "raw_file": "layout-1m-count-mixed-512-2long-descending-planner.json", + "raw_sha256": "38e8a91ba4d35806c6dff4952478cfee49fe069d45b658e879f0141d21d2ddb6" + }, + "zigzag": { + "total_ms": 430.64112854003906, + "min_ms": 427.5303955078125, + "max_ms": 433.921630859375, + "allocated_gib": 71.83758449554443, + "reserved_gib": 87.01953125, + "prepare_ms": 68.68545711040497, + "layout_prepare_ms": 68.15863598603755, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011748686063494781, + "dq": 0.00047520037171146624, + "dkv": 0.0005415161789643414, + "d_index_q": 0.0005674451612003116, + "d_weights": 0.00048600187424832343, + "d_pool_kv": 0.0005817740228488663, + "d_pool_score": 0.0005603234157210047, + "d_ape": 5.432171671313229e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.9998051151633263, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-2long-descending-zigzag.json", + "raw_sha256": "8055ef0fefe8d48eb4a8e5f0ef8a6632ed7eac920f7deb986023511dc0b57a1b" + } + }, + "speedup_vs_contiguous": 1.132438901309739, + "gap_to_best_baseline_pct": -4.303637626394297, + "prepare_plus_iteration_ms": { + "contiguous": 476.823518704623, + "planner": 447.64047791436315, + "zigzag": 499.32658565044403 + }, + "prepare_inclusive_gap_pct": -6.120302301686131, + "documents": 512, + "lengths_sha256": "1355e0fb1ef1d83a855305c16e873372467e69d68007d35dcefa354ea7c6ab1b" + }, + "1m-count-mixed-512-4long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 338.35548400878906, + "min_ms": 337.0683898925781, + "max_ms": 339.7387390136719, + "allocated_gib": 66.59222507476807, + "reserved_gib": 69.00390625, + "prepare_ms": 8.868525037541986, + "layout_prepare_ms": 8.5072279907763, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 128, + "raw_file": "layout-1m-count-mixed-512-4long-contiguous.json", + "raw_sha256": "2dc7e20d38e92e28ab857def38a7a1baa59daa25e1114930adc86faa455c401a" + }, + "planner": { + "total_ms": 338.45623779296875, + "min_ms": 336.9595031738281, + "max_ms": 341.4984436035156, + "allocated_gib": 66.59222507476807, + "reserved_gib": 69.00390625, + "prepare_ms": 19.420458003878593, + "layout_prepare_ms": 19.02695291209966, + "policy": "contiguous", + "relative_rms": { + "output": 0.001108400347265694, + "dq": 0.00041324557296966425, + "dkv": 0.0004567346009868542, + "d_index_q": 0.0002833367986887155, + "d_weights": 0.00024968591781364885, + "d_pool_kv": 0.00032218307925626635, + "d_pool_score": 0.0003184380093125898, + "d_ape": 5.236217222337733e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 128, + "raw_file": "layout-1m-count-mixed-512-4long-planner.json", + "raw_sha256": "6f84a09f175260bf25c2595c7071ee4c324b11e42965c5d7fedd606ef3207465" + }, + "zigzag": { + "total_ms": 338.5984802246094, + "min_ms": 337.65234375, + "max_ms": 342.5445556640625, + "allocated_gib": 66.80190896987915, + "reserved_gib": 69.716796875, + "prepare_ms": 37.792065064422786, + "layout_prepare_ms": 37.35221491660923, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011215964857729552, + "dq": 0.00041829052447210673, + "dkv": 0.0004561462148145632, + "d_index_q": 0.00033356936908467654, + "d_weights": 0.0002592224993963174, + "d_pool_kv": 0.0003434961985130709, + "d_pool_score": 0.0003701823394787622, + "d_ape": 5.452835218376787e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.49999136850237846, + "max_spans": 1024, + "raw_file": "layout-1m-count-mixed-512-4long-zigzag.json", + "raw_sha256": "84ee23ee7a5bc2e3bc5ad747b70f63d8b668af58cd1869aefb51165e623fe8b8" + } + }, + "speedup_vs_contiguous": 0.9997023137028388, + "gap_to_best_baseline_pct": 0.02977749406807284, + "prepare_plus_iteration_ms": { + "contiguous": 347.22400904633105, + "planner": 357.87669579684734, + "zigzag": 376.39054528903216 + }, + "prepare_inclusive_gap_pct": 3.0679579962729076, + "documents": 512, + "lengths_sha256": "74f528dba1eacc05b1dd3ae32df6d6e5fcba98ba8abc06fecf0c0df6bf0aa380" + }, + "1m-count-near-uniform-512": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 174.68292236328125, + "min_ms": 173.8264923095703, + "max_ms": 177.15196228027344, + "allocated_gib": 65.79301404953003, + "reserved_gib": 68.0859375, + "prepare_ms": 6.28169986885041, + "layout_prepare_ms": 5.795156001113355, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.1400146484375, + "max_spans": 70, + "raw_file": "layout-1m-count-near-uniform-512-contiguous.json", + "raw_sha256": "46b66a22406b73c9ade46c9389b8981bb27ec8a8f1bfe6b3e1fa4d17215ad402" + }, + "planner": { + "total_ms": 174.54737854003906, + "min_ms": 173.66610717773438, + "max_ms": 175.65142822265625, + "allocated_gib": 65.79301404953003, + "reserved_gib": 68.0859375, + "prepare_ms": 6.889444077387452, + "layout_prepare_ms": 6.43392198253423, + "policy": "contiguous", + "relative_rms": { + "output": 0.0003621722687541866, + "dq": 0.0001877717766341684, + "dkv": 0.00015745879408379858, + "d_index_q": 5.0233157871382964e-05, + "d_weights": 4.737857482285738e-05, + "d_pool_kv": 0.0001252604413295691, + "d_pool_score": 0.0001366667434959041, + "d_ape": 2.009941592126655e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.1400146484375, + "max_spans": 70, + "raw_file": "layout-1m-count-near-uniform-512-planner.json", + "raw_sha256": "913cceeccdd10800dd4e912a1d564ef8bb61dad70ae81a422a862a906e3a8c9b" + }, + "zigzag": { + "total_ms": 185.5495834350586, + "min_ms": 185.0115203857422, + "max_ms": 186.0485382080078, + "allocated_gib": 65.8086576461792, + "reserved_gib": 68.587890625, + "prepare_ms": 40.83809698931873, + "layout_prepare_ms": 40.41551903355867, + "policy": "zigzag", + "relative_rms": { + "output": 0.00035880839521131567, + "dq": 0.00018598617770927155, + "dkv": 0.00015672983164195206, + "d_index_q": 4.980354544334212e-05, + "d_weights": 4.6031241434262136e-05, + "d_pool_kv": 0.00012127384059513735, + "d_pool_score": 0.0001278846890799026, + "d_ape": 2.0326038418112683e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.1326904296875, + "max_spans": 1024, + "raw_file": "layout-1m-count-near-uniform-512-zigzag.json", + "raw_sha256": "1cc9d1f19a5a539dc40fe26386924bd4ac6896b9d1eca72fe3f782ccd62ae421" + } + }, + "speedup_vs_contiguous": 1.0007765445942294, + "gap_to_best_baseline_pct": -0.07759420406323603, + "prepare_plus_iteration_ms": { + "contiguous": 180.96462223213166, + "planner": 181.43682261742651, + "zigzag": 226.38768042437732 + }, + "prepare_inclusive_gap_pct": 0.26093519245389274, + "documents": 512, + "lengths_sha256": "a883d69122302c1f8f9693929fa0ea899a3e5e33acb0a13cd8340e29afc198a0" + }, + "1m-count-tiny-mix-1000": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 277.5030212402344, + "min_ms": 276.7580871582031, + "max_ms": 278.6227111816406, + "allocated_gib": 67.28772163391113, + "reserved_gib": 69.443359375, + "prepare_ms": 14.695362886413932, + "layout_prepare_ms": 14.116181991994381, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 174, + "raw_file": "layout-1m-count-tiny-mix-1000-contiguous.json", + "raw_sha256": "80d08f5ab5f1e35f442698c25b83751be87ef5ce4224b955890777d7182d1219" + }, + "planner": { + "total_ms": 277.97694396972656, + "min_ms": 276.58367919921875, + "max_ms": 281.081787109375, + "allocated_gib": 67.28772163391113, + "reserved_gib": 69.443359375, + "prepare_ms": 12.267046957276762, + "layout_prepare_ms": 11.748468037694693, + "policy": "contiguous", + "relative_rms": { + "output": 0.0008147533423000863, + "dq": 0.0003699549868243877, + "dkv": 0.00028025793158283253, + "d_index_q": 0.00018789359275136517, + "d_weights": 0.00016652942081256042, + "d_pool_kv": 0.00023713204488080765, + "d_pool_score": 0.00023688382513071412, + "d_ape": 2.7803123769608203e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 174, + "raw_file": "layout-1m-count-tiny-mix-1000-planner.json", + "raw_sha256": "3eac9c25237afd5472f5a365b4f857ee8e947e2718dfb2c3611e8688666e20aa" + }, + "zigzag": { + "total_ms": 288.2996520996094, + "min_ms": 287.6239318847656, + "max_ms": 289.6219177246094, + "allocated_gib": 67.24870872497559, + "reserved_gib": 78.861328125, + "prepare_ms": 67.15863605495542, + "layout_prepare_ms": 66.5940580656752, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008172251772034926, + "dq": 0.00036548175905332905, + "dkv": 0.0002639633962520787, + "d_index_q": 0.00017265337760804814, + "d_weights": 0.00016465928857432128, + "d_pool_kv": 0.00023364198757436886, + "d_pool_score": 0.00023719059082038686, + "d_ape": 2.881065687905103e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49526814743876457, + "max_spans": 2000, + "raw_file": "layout-1m-count-tiny-mix-1000-zigzag.json", + "raw_sha256": "ca6895e139050eaba280929dc0e0e527168019b7b4e68b3b07fbbd3644138080" + } + }, + "speedup_vs_contiguous": 0.9982951005837959, + "gap_to_best_baseline_pct": 0.17078110622872344, + "prepare_plus_iteration_ms": { + "contiguous": 292.1983841266483, + "planner": 290.2439909270033, + "zigzag": 355.4582881545648 + }, + "prepare_inclusive_gap_pct": -0.6688583188050323, + "documents": 1000, + "lengths_sha256": "2043af46235527c07d70653288a5fbff85cbdc1d61c2df251c4b11c9f2173662" + }, + "1m-count-tiny-mix-512": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 252.4097137451172, + "min_ms": 251.78445434570312, + "max_ms": 253.2755126953125, + "allocated_gib": 66.40801906585693, + "reserved_gib": 68.822265625, + "prepare_ms": 11.81349204853177, + "layout_prepare_ms": 11.319566983729601, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49916257709264755, + "max_spans": 92, + "raw_file": "layout-1m-count-tiny-mix-512-contiguous.json", + "raw_sha256": "470b9156895dfef6f001c6566a52ff9fe9156a7aebfe0d9e953cec31588205b4" + }, + "planner": { + "total_ms": 252.61129760742188, + "min_ms": 251.07884216308594, + "max_ms": 253.76771545410156, + "allocated_gib": 66.40801906585693, + "reserved_gib": 68.822265625, + "prepare_ms": 10.621751076541841, + "layout_prepare_ms": 10.238889022730291, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011379571872195018, + "dq": 0.0005142828500427043, + "dkv": 0.00032494544441751173, + "d_index_q": 0.00016323910386101733, + "d_weights": 0.00014454080721691223, + "d_pool_kv": 0.00024713382526628155, + "d_pool_score": 0.00024822032949893725, + "d_ape": 5.576403384703898e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49916257709264755, + "max_spans": 92, + "raw_file": "layout-1m-count-tiny-mix-512-planner.json", + "raw_sha256": "542b74951fd33de1c6536f234290815cf3368ed74b1a311ca939395424d0cce1" + }, + "zigzag": { + "total_ms": 260.91038513183594, + "min_ms": 260.1084899902344, + "max_ms": 261.8586730957031, + "allocated_gib": 66.41342687606812, + "reserved_gib": 69.32421875, + "prepare_ms": 35.87489610072225, + "layout_prepare_ms": 35.48457403667271, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011495718791550967, + "dq": 0.000523029028005613, + "dkv": 0.0003364792880244074, + "d_index_q": 0.00021014620007773496, + "d_weights": 0.0001769941445596639, + "d_pool_kv": 0.0002674930321212269, + "d_pool_score": 0.0002657953944677991, + "d_ape": 5.380611858644655e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.4989374950528145, + "max_spans": 1024, + "raw_file": "layout-1m-count-tiny-mix-512-zigzag.json", + "raw_sha256": "25ff432deedf54ecca7b914719d4cc1415c96127bd9f5acb375435664074a70d" + } + }, + "speedup_vs_contiguous": 0.9992019998146798, + "gap_to_best_baseline_pct": 0.07986374981916455, + "prepare_plus_iteration_ms": { + "contiguous": 264.22320579364896, + "planner": 263.2330486839637, + "zigzag": 296.7852812325582 + }, + "prepare_inclusive_gap_pct": -0.3747426751223859, + "documents": 512, + "lengths_sha256": "eb5e64b29a22782c3e203c9db504474452dd18b36ac3aa9a4ad991a5af9bdbf9" + }, + "1m-count-uniform-1024": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 119.41489791870117, + "min_ms": 119.24073791503906, + "max_ms": 119.6561279296875, + "allocated_gib": 66.34996938705444, + "reserved_gib": 68.70703125, + "prepare_ms": 8.79430107306689, + "layout_prepare_ms": 8.168380009010434, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0623779296875, + "max_spans": 128, + "raw_file": "layout-1m-count-uniform-1024-contiguous.json", + "raw_sha256": "cb2b198cea9cd3cc1c3eaf1a11d285567036530e09b0bb6e49b1e711424adf27" + }, + "planner": { + "total_ms": 119.48390579223633, + "min_ms": 119.34534454345703, + "max_ms": 119.6822738647461, + "allocated_gib": 66.34996938705444, + "reserved_gib": 68.70703125, + "prepare_ms": 11.64705806877464, + "layout_prepare_ms": 11.14371104631573, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.345404721876349e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.14921621395609e-05, + "d_pool_score": 7.218511403644321e-05, + "d_ape": 9.385957743902641e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0623779296875, + "max_spans": 128, + "raw_file": "layout-1m-count-uniform-1024-planner.json", + "raw_sha256": "27c35dcd661256741e720d8db490b9e8e83568b814699ed745b9ab2bbc01fc1d" + }, + "zigzag": { + "total_ms": 133.69087982177734, + "min_ms": 133.57740783691406, + "max_ms": 133.81129455566406, + "allocated_gib": 66.35586500167847, + "reserved_gib": 69.208984375, + "prepare_ms": 96.13504004664719, + "layout_prepare_ms": 95.60391097329557, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.0615702213174114e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.488063479448549e-05, + "d_pool_score": 7.075341797691834e-05, + "d_ape": 9.443483387805362e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0623779296875, + "max_spans": 2048, + "raw_file": "layout-1m-count-uniform-1024-zigzag.json", + "raw_sha256": "0dfd44fe1fc27144d82b9d43d49510d9504ce3ee909f4a66712e0e47b687967a" + } + }, + "speedup_vs_contiguous": 0.9994224504708178, + "gap_to_best_baseline_pct": 0.0577883285401537, + "prepare_plus_iteration_ms": { + "contiguous": 128.20919899176806, + "planner": 131.13096386101097, + "zigzag": 229.82591986842453 + }, + "prepare_inclusive_gap_pct": 2.278904237932644, + "documents": 1024, + "lengths_sha256": "7d1b6ad860e729dfc1e098d7b89b3e6e85ea0e907b90de52c462e82eb70e9471" + }, + "1m-count-uniform-256": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 212.9967041015625, + "min_ms": 212.8686065673828, + "max_ms": 213.72950744628906, + "allocated_gib": 65.50574588775635, + "reserved_gib": 67.923828125, + "prepare_ms": 3.961967071518302, + "layout_prepare_ms": 3.575184033252299, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.2498779296875, + "max_spans": 32, + "raw_file": "layout-1m-count-uniform-256-contiguous.json", + "raw_sha256": "06822967569423fd2d1a139db17c3413ae46207130da1cfde995a274d5d45d78" + }, + "planner": { + "total_ms": 213.0918197631836, + "min_ms": 212.87930297851562, + "max_ms": 213.86585998535156, + "allocated_gib": 65.50574588775635, + "reserved_gib": 67.923828125, + "prepare_ms": 4.589442978613079, + "layout_prepare_ms": 4.242056980729103, + "policy": "contiguous", + "relative_rms": { + "output": 0.0008983001869912695, + "dq": 0.0004206017803911536, + "dkv": 0.00029550489189777144, + "d_index_q": 0.00012622909156508592, + "d_weights": 0.00011351108154107496, + "d_pool_kv": 0.00020215864138665445, + "d_pool_score": 0.0002058701846442034, + "d_ape": 3.364103427754296e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.2498779296875, + "max_spans": 32, + "raw_file": "layout-1m-count-uniform-256-planner.json", + "raw_sha256": "f2b892081f4956336269dce71261727b62a18bd3c2a902b38e6846a72bd0e6c7" + }, + "zigzag": { + "total_ms": 220.54678344726562, + "min_ms": 220.18325805664062, + "max_ms": 221.15260314941406, + "allocated_gib": 65.51161479949951, + "reserved_gib": 68.42578125, + "prepare_ms": 22.930420003831387, + "layout_prepare_ms": 22.559862001799047, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009009673640706196, + "dq": 0.0004183338551494578, + "dkv": 0.0002940296337405208, + "d_index_q": 0.00010926371136128527, + "d_weights": 9.464661783500973e-05, + "d_pool_kv": 0.00020544086367790907, + "d_pool_score": 0.00021230314881310683, + "d_ape": 3.6113839509353313e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.2498779296875, + "max_spans": 512, + "raw_file": "layout-1m-count-uniform-256-zigzag.json", + "raw_sha256": "bebe21ddd1346db24c9d611e1a1a05d400b8ec7f65ace3032d45430e6888efbf" + } + }, + "speedup_vs_contiguous": 0.9995536400143056, + "gap_to_best_baseline_pct": 0.044655931190251685, + "prepare_plus_iteration_ms": { + "contiguous": 216.9586711730808, + "planner": 217.68126274179667, + "zigzag": 243.477203451097 + }, + "prepare_inclusive_gap_pct": 0.3330549384400472, + "documents": 256, + "lengths_sha256": "2b66265ed16d5c91894fb143f8c654f23f801053a71d4fd69181b0642cf716a7" + }, + "1m-heldout-jittered-768-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 139.59954833984375, + "min_ms": 139.31805419921875, + "max_ms": 140.8841552734375, + "allocated_gib": 66.06588745117188, + "reserved_gib": 68.396484375, + "prepare_ms": 7.3116159765049815, + "layout_prepare_ms": 6.80255691986531, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08821307122707367, + "max_spans": 99, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-contiguous.json", + "raw_sha256": "c61c210bfbe957150e8897a19dae3ec5aefa62d1b97f3dbc0c3c74a13421bbf6" + }, + "planner": { + "total_ms": 139.54326629638672, + "min_ms": 139.3474884033203, + "max_ms": 139.70681762695312, + "allocated_gib": 66.06588745117188, + "reserved_gib": 68.396484375, + "prepare_ms": 9.15238680317998, + "layout_prepare_ms": 8.70554894208908, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.534552172078891e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 8.384488122997325e-05, + "d_pool_score": 0.00012568695483731863, + "d_ape": 9.973954559775284e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08821307122707367, + "max_spans": 99, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-planner.json", + "raw_sha256": "84b9744b6aa02b91671350f67939997a9415d18cd770df78214da079e1e72be8" + }, + "zigzag": { + "total_ms": 152.68809509277344, + "min_ms": 152.55538940429688, + "max_ms": 153.68234252929688, + "allocated_gib": 66.0734510421753, + "reserved_gib": 68.90234375, + "prepare_ms": 64.65917301829904, + "layout_prepare_ms": 64.22794598620385, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.467940785081395e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.491533441383689e-05, + "d_pool_score": 6.582463735603153e-05, + "d_ape": 8.561384532654269e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08652793243527412, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261114-zigzag.json", + "raw_sha256": "8a93d233b6c7707cf3595479a5e33dfa2e64b450083591db987e93dc77794549" + } + }, + "speedup_vs_contiguous": 1.0004033304146507, + "gap_to_best_baseline_pct": -0.040316780481275405, + "prepare_plus_iteration_ms": { + "contiguous": 146.91116431634873, + "planner": 148.6956530995667, + "zigzag": 217.34726811107248 + }, + "prepare_inclusive_gap_pct": 1.2146720036711134, + "documents": 768, + "lengths_sha256": "2384da57da625501af834608e8298b60467638272e4c35dc6ea261713c7eb0d3" + }, + "1m-heldout-jittered-768-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 139.19557189941406, + "min_ms": 138.859619140625, + "max_ms": 140.72189331054688, + "allocated_gib": 66.05843544006348, + "reserved_gib": 68.38671875, + "prepare_ms": 7.30161985848099, + "layout_prepare_ms": 6.851075915619731, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08720123767852783, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-contiguous.json", + "raw_sha256": "281825c0dfd3827c5cc80ef8b905d5396badd31ecbada0b521049755e9fae486" + }, + "planner": { + "total_ms": 138.96669006347656, + "min_ms": 138.5364532470703, + "max_ms": 139.24237060546875, + "allocated_gib": 66.05843544006348, + "reserved_gib": 68.38671875, + "prepare_ms": 9.032243047840893, + "layout_prepare_ms": 8.597164996899664, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.083760511669527e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.48209891306436e-05, + "d_pool_score": 6.628093591786182e-05, + "d_ape": 8.97144166007357e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08720123767852783, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-planner.json", + "raw_sha256": "a2229866121576881acd3547ce8ad2ba875e0536e87568b6d4b5b6425ad2d497" + }, + "zigzag": { + "total_ms": 152.07752227783203, + "min_ms": 151.6907501220703, + "max_ms": 152.71705627441406, + "allocated_gib": 66.06595468521118, + "reserved_gib": 68.890625, + "prepare_ms": 67.99968285486102, + "layout_prepare_ms": 67.59183399844915, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.278026323711552e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 7.544563700271164e-05, + "d_pool_score": 7.063303606390711e-05, + "d_ape": 9.006565520482359e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.0866801105439663, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261115-zigzag.json", + "raw_sha256": "5ef0a181cfe1c6fafec89e9ac1539b46788a6868f57d4dd07a2bee0ff28bb0cd" + } + }, + "speedup_vs_contiguous": 1.0016470266063973, + "gap_to_best_baseline_pct": -0.16443183702919972, + "prepare_plus_iteration_ms": { + "contiguous": 146.49719175789505, + "planner": 147.99893311131746, + "zigzag": 220.07720513269305 + }, + "prepare_inclusive_gap_pct": 1.0250990721407227, + "documents": 768, + "lengths_sha256": "8884d0c226a9c39788069b745f1fa9b8499da77272f0e098c192c933dea50872" + }, + "1m-heldout-jittered-768-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 139.4260025024414, + "min_ms": 138.7571258544922, + "max_ms": 140.87948608398438, + "allocated_gib": 66.05876111984253, + "reserved_gib": 68.38671875, + "prepare_ms": 7.246268913149834, + "layout_prepare_ms": 6.824985961429775, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08736230432987213, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-contiguous.json", + "raw_sha256": "880cadfe68a2931bee30456d4c858043bad4546227d18fe035b62b8ea3f8b7ec" + }, + "planner": { + "total_ms": 139.19524383544922, + "min_ms": 138.77737426757812, + "max_ms": 140.00262451171875, + "allocated_gib": 66.05876111984253, + "reserved_gib": 68.38671875, + "prepare_ms": 9.026253945194185, + "layout_prepare_ms": 8.594213984906673, + "policy": "contiguous", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.93177721660562e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.695678676610094e-05, + "d_pool_score": 5.4488619542700796e-05, + "d_ape": 8.187076335588852e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08736230432987213, + "max_spans": 98, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-planner.json", + "raw_sha256": "840688ba09cf768b72512d250f48fe5d9f33bd023b6ee5b7103d19f9b8335802" + }, + "zigzag": { + "total_ms": 152.10254669189453, + "min_ms": 151.87277221679688, + "max_ms": 152.91253662109375, + "allocated_gib": 66.06567144393921, + "reserved_gib": 68.890625, + "prepare_ms": 71.22155895922333, + "layout_prepare_ms": 70.7582599716261, + "policy": "zigzag", + "relative_rms": { + "output": 0.0, + "dq": 0.0, + "dkv": 3.1343651448698765e-05, + "d_index_q": 0.0, + "d_weights": 0.0, + "d_pool_kv": 5.659607648562089e-05, + "d_pool_score": 5.1206900409261796e-05, + "d_ape": 8.198515312622374e-07, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.08655709028244019, + "max_spans": 1536, + "raw_file": "layout-1m-heldout-jittered-768-seed20261116-zigzag.json", + "raw_sha256": "3aebfa0afb6dc800f3e22106a7f55c050272ff51d1bce0726e29d3fee735c395" + } + }, + "speedup_vs_contiguous": 1.0016578056881382, + "gap_to_best_baseline_pct": -0.16550619170778713, + "prepare_plus_iteration_ms": { + "contiguous": 146.67227141559124, + "planner": 148.2214977806434, + "zigzag": 223.32410565111786 + }, + "prepare_inclusive_gap_pct": 1.0562503396858602, + "documents": 768, + "lengths_sha256": "484660be7f47772a4ee1c32aa13529e5d4dfa02ba770b46f67c83944d3d16782" + }, + "1m-heldout-lognormal-1000-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 242.74365997314453, + "min_ms": 241.8260498046875, + "max_ms": 243.4641571044922, + "allocated_gib": 67.14972496032715, + "reserved_gib": 69.392578125, + "prepare_ms": 23.308494011871517, + "layout_prepare_ms": 22.75374298915267, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 186, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-contiguous.json", + "raw_sha256": "c72415467035a1b2320593612913f0d9156c3b9f87dfd0b2bab580825435d73d" + }, + "planner": { + "total_ms": 242.79873657226562, + "min_ms": 242.02297973632812, + "max_ms": 243.5925750732422, + "allocated_gib": 67.14972496032715, + "reserved_gib": 69.392578125, + "prepare_ms": 18.588110920973122, + "layout_prepare_ms": 17.979708965867758, + "policy": "contiguous", + "relative_rms": { + "output": 0.0006639286414964582, + "dq": 0.0003421409964573097, + "dkv": 0.00019721484937837802, + "d_index_q": 0.00011000092475129282, + "d_weights": 0.0001068132442141927, + "d_pool_kv": 0.00018391022308396582, + "d_pool_score": 0.000179099258457748, + "d_ape": 3.408956283145262e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 186, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-planner.json", + "raw_sha256": "31fe420fe090d09c73296c247e5e8a6243701726a0ea69e87df03d963bc8a78d" + }, + "zigzag": { + "total_ms": 255.3045883178711, + "min_ms": 253.9938507080078, + "max_ms": 256.24114990234375, + "allocated_gib": 67.42135286331177, + "reserved_gib": 79.7109375, + "prepare_ms": 73.20481608621776, + "layout_prepare_ms": 72.67806399613619, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006729130295918793, + "dq": 0.000346805284697457, + "dkv": 0.00019871221130998912, + "d_index_q": 0.00011022225577992002, + "d_weights": 0.00010803303612996262, + "d_pool_kv": 0.00018681248298492773, + "d_pool_score": 0.00017741006842812262, + "d_ape": 3.0188399219256512e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49985016882419586, + "max_spans": 1790, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261114-zigzag.json", + "raw_sha256": "9021d15661632a997b26c2d03add080ed503ddb322430f01d16390e8079eedc8" + } + }, + "speedup_vs_contiguous": 0.9997731594492679, + "gap_to_best_baseline_pct": 0.02268920190426904, + "prepare_plus_iteration_ms": { + "contiguous": 266.05215398501605, + "planner": 261.38684749323875, + "zigzag": 328.50940440408885 + }, + "prepare_inclusive_gap_pct": -1.7535308103688751, + "documents": 1000, + "lengths_sha256": "f9552804d64123e178b417a374ec673ead4524cfdca92d036e2c49d2242a4462" + }, + "1m-heldout-lognormal-1000-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 238.08277130126953, + "min_ms": 237.04464721679688, + "max_ms": 240.6115264892578, + "allocated_gib": 67.31743335723877, + "reserved_gib": 69.40234375, + "prepare_ms": 10.576636996120214, + "layout_prepare_ms": 10.025152005255222, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 205, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-contiguous.json", + "raw_sha256": "73d12f484cc6a6c5781d76c844be01adbcc0bb83c62e714ff358b96cdf452c3f" + }, + "planner": { + "total_ms": 238.69235229492188, + "min_ms": 236.03648376464844, + "max_ms": 239.77203369140625, + "allocated_gib": 67.31743335723877, + "reserved_gib": 69.40234375, + "prepare_ms": 14.750760048627853, + "layout_prepare_ms": 14.257603092119098, + "policy": "contiguous", + "relative_rms": { + "output": 0.000627483968379822, + "dq": 0.0003440353669091771, + "dkv": 0.00019056774527328763, + "d_index_q": 0.00012198431218257316, + "d_weights": 0.00012043062385770405, + "d_pool_kv": 0.00018010804743631497, + "d_pool_score": 0.00017470367552733856, + "d_ape": 3.126033191671862e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 205, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-planner.json", + "raw_sha256": "268cbe29bd9f8bbfb74658d4ec75f8412f70a70d7a8090fd2764979bf02fba83" + }, + "zigzag": { + "total_ms": 248.48849487304688, + "min_ms": 246.69786071777344, + "max_ms": 250.46841430664062, + "allocated_gib": 67.61479187011719, + "reserved_gib": 69.91015625, + "prepare_ms": 69.25003707874566, + "layout_prepare_ms": 68.7437430024147, + "policy": "zigzag", + "relative_rms": { + "output": 0.0006277650198511363, + "dq": 0.00034722313626910604, + "dkv": 0.00019309980305575396, + "d_index_q": 0.00011586721853121804, + "d_weights": 0.00011320829699045079, + "d_pool_kv": 0.00019337419199732237, + "d_pool_score": 0.0001837468057090465, + "d_ape": 3.3393516739202175e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49977222830057144, + "max_spans": 1756, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261115-zigzag.json", + "raw_sha256": "462ccac477b21d252cce91a052b7ebeedc598fc61058bf33fe9395890da44b82" + } + }, + "speedup_vs_contiguous": 0.9974461645386142, + "gap_to_best_baseline_pct": 0.25603742359037174, + "prepare_plus_iteration_ms": { + "contiguous": 248.65940829738975, + "planner": 253.44311234354973, + "zigzag": 317.73853195179254 + }, + "prepare_inclusive_gap_pct": 1.9237977275482043, + "documents": 1000, + "lengths_sha256": "4860b063d4278394f20027f7e1f50de3bf6080e17af8500ea52a852f8f4840bf" + }, + "1m-heldout-lognormal-1000-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 243.51073455810547, + "min_ms": 242.13845825195312, + "max_ms": 245.1307830810547, + "allocated_gib": 67.10111045837402, + "reserved_gib": 69.26171875, + "prepare_ms": 9.780772146768868, + "layout_prepare_ms": 9.23799304291606, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 165, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-contiguous.json", + "raw_sha256": "85762a7b6a5337bbff4590396f9f87426434440d07a330772a0b23ac03910407" + }, + "planner": { + "total_ms": 244.25122833251953, + "min_ms": 242.3041229248047, + "max_ms": 245.93923950195312, + "allocated_gib": 67.10111045837402, + "reserved_gib": 69.26171875, + "prepare_ms": 11.766831041313708, + "layout_prepare_ms": 11.266377987340093, + "policy": "contiguous", + "relative_rms": { + "output": 0.0005700379404832603, + "dq": 0.000319132883698665, + "dkv": 0.00017456473994610376, + "d_index_q": 0.00011017834715051893, + "d_weights": 9.883468029904095e-05, + "d_pool_kv": 0.00017233705959233462, + "d_pool_score": 0.00017515985135813642, + "d_ape": 3.255570361937096e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 165, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-planner.json", + "raw_sha256": "066d43150dee8501d7f08bc63475b30c1f33486a98044180dc550a0c599da0f0" + }, + "zigzag": { + "total_ms": 255.27643585205078, + "min_ms": 254.18067932128906, + "max_ms": 256.8161315917969, + "allocated_gib": 67.19186353683472, + "reserved_gib": 69.892578125, + "prepare_ms": 65.90253801550716, + "layout_prepare_ms": 65.40064502041787, + "policy": "zigzag", + "relative_rms": { + "output": 0.0005658249552824931, + "dq": 0.00032052362678565844, + "dkv": 0.00017358126670537923, + "d_index_q": 0.00010731651292044749, + "d_weights": 9.582209699794192e-05, + "d_pool_kv": 0.00017146293516180915, + "d_pool_score": 0.0001825117182559775, + "d_ape": 3.296560839970807e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49838995188474655, + "max_spans": 1743, + "raw_file": "layout-1m-heldout-lognormal-1000-seed20261116-zigzag.json", + "raw_sha256": "d898c9cf7a64ce0e3dc8bbc7f34d48e71c1bac6dcccc8b6008a9ecd44aeb7b41" + } + }, + "speedup_vs_contiguous": 0.99696831094988, + "gap_to_best_baseline_pct": 0.30409081380244896, + "prepare_plus_iteration_ms": { + "contiguous": 253.29150670487434, + "planner": 256.01805937383324, + "zigzag": 321.17897386755794 + }, + "prepare_inclusive_gap_pct": 1.076448517531925, + "documents": 1000, + "lengths_sha256": "a790875510a4369ef17daa8104180cfe9404b90dfa3567b8652ea1435220224c" + }, + "1m-heldout-lognormal-257-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 241.6246566772461, + "min_ms": 240.7863311767578, + "max_ms": 242.99119567871094, + "allocated_gib": 65.84781551361084, + "reserved_gib": 68.2734375, + "prepare_ms": 4.298857995308936, + "layout_prepare_ms": 3.948985948227346, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49603627994656563, + "max_spans": 41, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-contiguous.json", + "raw_sha256": "9f80e24a6b4b20a2697560effb1c4ce1ea3384ddd71ec8131fd60f255b096e43" + }, + "planner": { + "total_ms": 241.78618621826172, + "min_ms": 241.03724670410156, + "max_ms": 242.47116088867188, + "allocated_gib": 65.84781551361084, + "reserved_gib": 68.2734375, + "prepare_ms": 16.895215027034283, + "layout_prepare_ms": 16.556818038225174, + "policy": "contiguous", + "relative_rms": { + "output": 0.001082461469725319, + "dq": 0.00047458198967384186, + "dkv": 0.00032668096235836366, + "d_index_q": 0.00016693420591685536, + "d_weights": 0.0001401149820797931, + "d_pool_kv": 0.0002270806698028004, + "d_pool_score": 0.0002237423415161784, + "d_ape": 4.043788132397096e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49603627994656563, + "max_spans": 41, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-planner.json", + "raw_sha256": "2627c6242e0e931d47e104db31fdd7f534a82a2e567533cd631d8a1ebca7aaca" + }, + "zigzag": { + "total_ms": 249.37068939208984, + "min_ms": 248.5408935546875, + "max_ms": 250.9279327392578, + "allocated_gib": 65.8637466430664, + "reserved_gib": 68.78125, + "prepare_ms": 22.184235975146294, + "layout_prepare_ms": 21.849395940080285, + "policy": "zigzag", + "relative_rms": { + "output": 0.001088782543496487, + "dq": 0.0004730404072428335, + "dkv": 0.00031546559266904786, + "d_index_q": 0.00016690660493783074, + "d_weights": 0.00014110205381527205, + "d_pool_kv": 0.0002280621593994692, + "d_pool_score": 0.0002186074713902558, + "d_ape": 3.871855939807412e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49925995245575905, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261114-zigzag.json", + "raw_sha256": "77d01364b5020f7f3f07344f74d5e5c49a00d484dbec869d72f66e5812e4068b" + } + }, + "speedup_vs_contiguous": 0.9993319323012531, + "gap_to_best_baseline_pct": 0.06685143115645609, + "prepare_plus_iteration_ms": { + "contiguous": 245.92351467255503, + "planner": 258.681401245296, + "zigzag": 271.55492536723614 + }, + "prepare_inclusive_gap_pct": 5.187745706110292, + "documents": 257, + "lengths_sha256": "22124dfa90691813499ba5fff3956732dd43551f4504a48fc46bbb35d7aa1141" + }, + "1m-heldout-lognormal-257-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 265.89527893066406, + "min_ms": 264.75982666015625, + "max_ms": 268.54608154296875, + "allocated_gib": 65.86178588867188, + "reserved_gib": 68.294921875, + "prepare_ms": 4.706783103756607, + "layout_prepare_ms": 4.385833046399057, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 43, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-contiguous.json", + "raw_sha256": "0e96976fed2028414081cc39e060cf53f598d02e636f8fa3f134d0f65c0451cf" + }, + "planner": { + "total_ms": 266.5758819580078, + "min_ms": 264.8830261230469, + "max_ms": 267.7908020019531, + "allocated_gib": 65.86178588867188, + "reserved_gib": 68.294921875, + "prepare_ms": 5.949159967713058, + "layout_prepare_ms": 5.652374937199056, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011651272922486198, + "dq": 0.00046214190754927277, + "dkv": 0.00033944054288377243, + "d_index_q": 0.00018021932141442865, + "d_weights": 0.00016083575052146454, + "d_pool_kv": 0.00026245941371384453, + "d_pool_score": 0.00027060036694570604, + "d_ape": 3.819325031365223e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 43, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-planner.json", + "raw_sha256": "a6a02d026ba3e2910be4548d73f88cfb1ead501e4292eb973941dfac8667150b" + }, + "zigzag": { + "total_ms": 270.3296203613281, + "min_ms": 269.5667724609375, + "max_ms": 272.5, + "allocated_gib": 65.86807107925415, + "reserved_gib": 68.802734375, + "prepare_ms": 29.988013091497123, + "layout_prepare_ms": 29.641884961165488, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011900939859345021, + "dq": 0.0004752948227896761, + "dkv": 0.0003530750774232262, + "d_index_q": 0.0001982857983837737, + "d_weights": 0.00016784374447724294, + "d_pool_kv": 0.00026278078754386075, + "d_pool_score": 0.0002623528065003457, + "d_ape": 3.922395870611972e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.4999861344695091, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261115-zigzag.json", + "raw_sha256": "3f4742ba6a41a71ac42df4190de2bedd3f97ce2acfdcd640f44e3148b04dd391" + } + }, + "speedup_vs_contiguous": 0.9974468694529126, + "gap_to_best_baseline_pct": 0.25596657077961993, + "prepare_plus_iteration_ms": { + "contiguous": 270.60206203442067, + "planner": 272.52504192572087, + "zigzag": 300.31763345282525 + }, + "prepare_inclusive_gap_pct": 0.710630169202342, + "documents": 257, + "lengths_sha256": "0b8ffc4c0c7294d43fa7ae5f3ee55faaef51c2f526387b1efe2d60aca24ecf20" + }, + "1m-heldout-lognormal-257-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 260.8553009033203, + "min_ms": 259.1219482421875, + "max_ms": 261.8494567871094, + "allocated_gib": 65.91541194915771, + "reserved_gib": 68.26171875, + "prepare_ms": 5.178843857720494, + "layout_prepare_ms": 4.816529923118651, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 54, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-contiguous.json", + "raw_sha256": "195dddebf12e65361a3e617322fbf574c7c9c3eed81643a4e319b1fd093c307d" + }, + "planner": { + "total_ms": 261.18247985839844, + "min_ms": 260.1310119628906, + "max_ms": 263.29669189453125, + "allocated_gib": 65.91541194915771, + "reserved_gib": 68.26171875, + "prepare_ms": 5.219785147346556, + "layout_prepare_ms": 4.866166040301323, + "policy": "contiguous", + "relative_rms": { + "output": 0.0011647173346943536, + "dq": 0.00047342276965134394, + "dkv": 0.00036562031615802835, + "d_index_q": 0.00021551581252850696, + "d_weights": 0.00016625701885295734, + "d_pool_kv": 0.0002632888196715603, + "d_pool_score": 0.000255773612521125, + "d_ape": 3.971134246049848e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.5, + "max_spans": 54, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-planner.json", + "raw_sha256": "8d936534214d4549c1acb1225671cf5c54859c2056d5b334d7cc1ad492298d73" + }, + "zigzag": { + "total_ms": 265.91778564453125, + "min_ms": 263.9945983886719, + "max_ms": 267.6018371582031, + "allocated_gib": 65.97511625289917, + "reserved_gib": 68.794921875, + "prepare_ms": 21.29810012411326, + "layout_prepare_ms": 20.93889401294291, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011734780735060887, + "dq": 0.00047267021844590634, + "dkv": 0.0003651486194391658, + "d_index_q": 0.00019081633609087836, + "d_weights": 0.0001568154855097009, + "d_pool_kv": 0.0002506288295072141, + "d_pool_score": 0.0002447311671719562, + "d_ape": 3.5682579411177634e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.4963202215731144, + "max_spans": 514, + "raw_file": "layout-1m-heldout-lognormal-257-seed20261116-zigzag.json", + "raw_sha256": "abea4ee11e6a7073940a93bf00e5942008bb99412477d1c56c274fe4044782f4" + } + }, + "speedup_vs_contiguous": 0.998747316606935, + "gap_to_best_baseline_pct": 0.12542545769440583, + "prepare_plus_iteration_ms": { + "contiguous": 266.0341447610408, + "planner": 266.402265005745, + "zigzag": 287.2158857686445 + }, + "prepare_inclusive_gap_pct": 0.13837330731918396, + "documents": 257, + "lengths_sha256": "1b0bd973931c10a52c40d767fb8a267929d55b8a7e94ed85145c4ab27033efda" + }, + "1m-heldout-tiny-tail-768-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 436.30291748046875, + "min_ms": 433.5298767089844, + "max_ms": 439.7943115234375, + "allocated_gib": 68.20519065856934, + "reserved_gib": 70.546875, + "prepare_ms": 10.106927016749978, + "layout_prepare_ms": 9.562587947584689, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.0, + "max_spans": 263, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-contiguous.json", + "raw_sha256": "8a8ffeec6eec8a264804f99b37d66ec9a4b009a10313c251fc50f63ae72185af" + }, + "planner": { + "total_ms": 397.61912536621094, + "min_ms": 393.9225769042969, + "max_ms": 398.3644714355469, + "allocated_gib": 69.3592586517334, + "reserved_gib": 82.36328125, + "prepare_ms": 42.66655596438795, + "layout_prepare_ms": 42.218683985993266, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0008605439165391654, + "dq": 0.0003796395940608871, + "dkv": 0.0004054932800869999, + "d_index_q": 0.00046272970168726135, + "d_weights": 0.0004308179609189755, + "d_pool_kv": 0.0006452094646867585, + "d_pool_score": 0.0006358833131884315, + "d_ape": 5.545222900579305e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 2.9356987215578556, + "max_spans": 267, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-planner.json", + "raw_sha256": "784f57c9706e909b3737814020733d26552a436f7be44459872241000b3725c8" + }, + "zigzag": { + "total_ms": 406.08197021484375, + "min_ms": 404.5019226074219, + "max_ms": 407.4826965332031, + "allocated_gib": 70.75117206573486, + "reserved_gib": 73.427734375, + "prepare_ms": 48.15629299264401, + "layout_prepare_ms": 47.64960997272283, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008973106239127031, + "dq": 0.00038915186357727963, + "dkv": 0.0005022528545868634, + "d_index_q": 0.0005194953800697627, + "d_weights": 0.0004678227992383376, + "d_pool_kv": 0.0006610885127278837, + "d_pool_score": 0.0006449522590483258, + "d_ape": 5.407920341001154e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 2.9998814575374126, + "max_spans": 1260, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261114-zigzag.json", + "raw_sha256": "7d6cbc1cb6b8c8507589e766a4faf98800549fc3398bf84b122e09175a2c8179" + } + }, + "speedup_vs_contiguous": 1.0972885599469837, + "gap_to_best_baseline_pct": -2.0840237856793786, + "prepare_plus_iteration_ms": { + "contiguous": 446.4098444972187, + "planner": 440.2856813305989, + "zigzag": 454.23826320748776 + }, + "prepare_inclusive_gap_pct": -1.3718700969772124, + "documents": 768, + "lengths_sha256": "759566912504d164342d1556047348e640f650e2bd969cace033c5f384cd1e9f" + }, + "1m-heldout-tiny-tail-768-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 498.76747131347656, + "min_ms": 496.61907958984375, + "max_ms": 499.63848876953125, + "allocated_gib": 68.21028566360474, + "reserved_gib": 70.623046875, + "prepare_ms": 9.636490955017507, + "layout_prepare_ms": 9.107938036322594, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 2.99999725446105, + "max_spans": 176, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-contiguous.json", + "raw_sha256": "751af802d0941f5e890120e8eb2cb43fbcdd5b47b8f92461fdb1fa7de6da76f2" + }, + "planner": { + "total_ms": 416.4952392578125, + "min_ms": 412.5445251464844, + "max_ms": 419.02020263671875, + "allocated_gib": 71.1473913192749, + "reserved_gib": 80.89453125, + "prepare_ms": 29.679763014428318, + "layout_prepare_ms": 29.224156984128058, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0007987615295327556, + "dq": 0.00039969776861594446, + "dkv": 0.0005472686255923894, + "d_index_q": 0.0004013407150718083, + "d_weights": 0.00039543349130716937, + "d_pool_kv": 0.0006895992619430324, + "d_pool_score": 0.0006990540356347693, + "d_ape": 5.209086564608632e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 4.499835789203644, + "max_spans": 177, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-planner.json", + "raw_sha256": "30332df67d347bcfa79a52ba3f9690159d7c4efc61181cd7792ad349b4bf31bd" + }, + "zigzag": { + "total_ms": 423.7662811279297, + "min_ms": 422.471435546875, + "max_ms": 425.5033874511719, + "allocated_gib": 71.05869436264038, + "reserved_gib": 73.974609375, + "prepare_ms": 48.98834810592234, + "layout_prepare_ms": 48.50035498384386, + "policy": "zigzag", + "relative_rms": { + "output": 0.0007981808335954349, + "dq": 0.00040108914815335767, + "dkv": 0.0005455319088188014, + "d_index_q": 0.00040511993368696884, + "d_weights": 0.00040129581062005136, + "d_pool_kv": 0.000683351457010383, + "d_pool_score": 0.0006834509121711517, + "d_ape": 5.314517142166228e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 4.499952055513859, + "max_spans": 1233, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261115-zigzag.json", + "raw_sha256": "6a2beb285d81a216825fd74f33c3b1cc372c41ee64d142f80226c6e7de0b5a5b" + } + }, + "speedup_vs_contiguous": 1.1975346277721488, + "gap_to_best_baseline_pct": -1.7158141631193558, + "prepare_plus_iteration_ms": { + "contiguous": 508.40396226849407, + "planner": 446.1750022722408, + "zigzag": 472.75462923385203 + }, + "prepare_inclusive_gap_pct": -5.622288036541545, + "documents": 768, + "lengths_sha256": "747a3b346f6fb67d2a1bda28b2c1431fae38be2adb70abab6f64c52541fce124" + }, + "1m-heldout-tiny-tail-768-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 559.7161865234375, + "min_ms": 555.4302978515625, + "max_ms": 560.9325561523438, + "allocated_gib": 69.61964559555054, + "reserved_gib": 72.03125, + "prepare_ms": 9.729802957735956, + "layout_prepare_ms": 9.215633966960013, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999971453100443, + "max_spans": 225, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-contiguous.json", + "raw_sha256": "34fb5f5d012997932088d2de86f6c14a8a809f74ed42e02a14fa7eb70040d2f8" + }, + "planner": { + "total_ms": 460.2198486328125, + "min_ms": 457.4324951171875, + "max_ms": 461.8628234863281, + "allocated_gib": 71.91085004806519, + "reserved_gib": 84.1875, + "prepare_ms": 33.673338941298425, + "layout_prepare_ms": 33.20295491721481, + "policy": "selection-local-5x", + "relative_rms": { + "output": 0.0009838151945294016, + "dq": 0.0003731228063005086, + "dkv": 0.0004893858809695903, + "d_index_q": 0.000563171314835403, + "d_weights": 0.000552436597671842, + "d_pool_kv": 0.0006584843852002015, + "d_pool_score": 0.000660263037667876, + "d_ape": 6.329523728349438e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 5.999660342931747, + "max_spans": 226, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-planner.json", + "raw_sha256": "87ded17b209d211147131a5fc1862e21ff1f67971640652625b880fb27055cb5" + }, + "zigzag": { + "total_ms": 466.84815979003906, + "min_ms": 465.43475341796875, + "max_ms": 468.9653625488281, + "allocated_gib": 72.83784294128418, + "reserved_gib": 82.203125, + "prepare_ms": 49.553396995179355, + "layout_prepare_ms": 49.08921697642654, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009946083346030021, + "dq": 0.00038457455793733337, + "dkv": 0.0005425483112289231, + "d_index_q": 0.0006348606318300278, + "d_weights": 0.000538693682633851, + "d_pool_kv": 0.0006613934471291266, + "d_pool_score": 0.0006581440262921791, + "d_ape": 5.958685410178528e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 5.999810948967934, + "max_spans": 1248, + "raw_file": "layout-1m-heldout-tiny-tail-768-seed20261116-zigzag.json", + "raw_sha256": "fd5cbe18c33e009862cd9cdab3a5a6d72d1be1f2db6e9201e6cd706880b2efbb" + } + }, + "speedup_vs_contiguous": 1.2161930611776122, + "gap_to_best_baseline_pct": -1.419800210888178, + "prepare_plus_iteration_ms": { + "contiguous": 569.4459894811735, + "planner": 493.8931875741109, + "zigzag": 516.4015567852184 + }, + "prepare_inclusive_gap_pct": -4.358695072731777, + "documents": 768, + "lengths_sha256": "e879dc84790636c8379bc654c1bf98f0f184183c998e485cd8cf5a1df8e1e686" + }, + "1m-heldout-trimodal-513-seed20261114": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 528.0744018554688, + "min_ms": 522.5753784179688, + "max_ms": 529.0509643554688, + "allocated_gib": 70.32668733596802, + "reserved_gib": 72.73828125, + "prepare_ms": 8.3013930125162, + "layout_prepare_ms": 7.790984003804624, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999376945197582, + "max_spans": 233, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-contiguous.json", + "raw_sha256": "8a3fc72747754bc7bf98ae1deb8ea22ee20008aa3b6703f2d087437b21883ccb" + }, + "planner": { + "total_ms": 471.4351348876953, + "min_ms": 470.94830322265625, + "max_ms": 474.5873107910156, + "allocated_gib": 73.74629783630371, + "reserved_gib": 87.171875, + "prepare_ms": 44.60881382692605, + "layout_prepare_ms": 44.13713398389518, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.000961612511327622, + "dq": 0.0003228331744803191, + "dkv": 0.000675315032828408, + "d_index_q": 0.0006175052651110469, + "d_weights": 0.0005102943346481259, + "d_pool_kv": 0.0007424000145656556, + "d_pool_score": 0.0007213206164765067, + "d_ape": 5.763866208010157e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999752186238766, + "max_spans": 237, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-planner.json", + "raw_sha256": "5441b436adfd01760c603c170f255191312cae6630d20e5ac4b7616c6600aa09" + }, + "zigzag": { + "total_ms": 482.18006896972656, + "min_ms": 480.6382141113281, + "max_ms": 484.48101806640625, + "allocated_gib": 73.42837762832642, + "reserved_gib": 75.80078125, + "prepare_ms": 34.889695001766086, + "layout_prepare_ms": 34.403237979859114, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009026663219808964, + "dq": 0.00029800123046863397, + "dkv": 0.0005882285438040862, + "d_index_q": 0.0004945744903112981, + "d_weights": 0.0004758399315457316, + "d_pool_kv": 0.0006676745367674404, + "d_pool_score": 0.0006603297039606205, + "d_ape": 5.349247680815768e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 5.999862506985664, + "max_spans": 835, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261114-zigzag.json", + "raw_sha256": "5f55ccd78b5bdd22737e144e4239d13ccba51f553282e2a54b8e49fbc808dce0" + } + }, + "speedup_vs_contiguous": 1.1201422269497712, + "gap_to_best_baseline_pct": -2.2284069320803623, + "prepare_plus_iteration_ms": { + "contiguous": 536.375794867985, + "planner": 516.0439487146214, + "zigzag": 517.0697639714926 + }, + "prepare_inclusive_gap_pct": -0.19839010678022184, + "documents": 513, + "lengths_sha256": "3e4264b20b69bf872e0287259db21e68128815011e3a225f7f002b5e26266a7b" + }, + "1m-heldout-trimodal-513-seed20261115": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 550.4004211425781, + "min_ms": 547.5628051757812, + "max_ms": 553.02490234375, + "allocated_gib": 69.8761420249939, + "reserved_gib": 72.076171875, + "prepare_ms": 33.551717875525355, + "layout_prepare_ms": 33.11699989717454, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999942742288113, + "max_spans": 260, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-contiguous.json", + "raw_sha256": "3dee930bb2ff3e79a9e051ed87ccaa077dee76d566fb3f1bb0d6e015c51dd7b4" + }, + "planner": { + "total_ms": 475.1640930175781, + "min_ms": 471.80645751953125, + "max_ms": 477.1839904785156, + "allocated_gib": 74.11927461624146, + "reserved_gib": 85.970703125, + "prepare_ms": 28.17985403817147, + "layout_prepare_ms": 27.757827076129615, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0009194823642314863, + "dq": 0.0002948452198108362, + "dkv": 0.00043585622989332116, + "d_index_q": 0.0007520857577959455, + "d_weights": 0.0005999834445308111, + "d_pool_kv": 0.0006288663704566335, + "d_pool_score": 0.0006258680549696073, + "d_ape": 5.887180290091932e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999993029981852, + "max_spans": 265, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-planner.json", + "raw_sha256": "924d83bdefd5a1ee078e10521aee743d31584790a0f0ae9d1f6e24569a585005" + }, + "zigzag": { + "total_ms": 479.2586669921875, + "min_ms": 477.9535827636719, + "max_ms": 481.8824768066406, + "allocated_gib": 71.65527057647705, + "reserved_gib": 74.568359375, + "prepare_ms": 35.18300596624613, + "layout_prepare_ms": 34.71819101832807, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008696921284109005, + "dq": 0.0003016373452874105, + "dkv": 0.0005305068105551274, + "d_index_q": 0.0006150898262228985, + "d_weights": 0.0005590287009485252, + "d_pool_kv": 0.0007163439171067582, + "d_pool_score": 0.0007367160338217451, + "d_ape": 5.756793869761055e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999988615512848, + "max_spans": 817, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261115-zigzag.json", + "raw_sha256": "27be3e9d0a28de7930f03c3651bd2d9858cf830d7ba19b8ff0487e2815828021" + } + }, + "speedup_vs_contiguous": 1.1583375706005141, + "gap_to_best_baseline_pct": -0.8543557491211939, + "prepare_plus_iteration_ms": { + "contiguous": 583.9521390181035, + "planner": 503.3439470557496, + "zigzag": 514.4416729584336 + }, + "prepare_inclusive_gap_pct": -2.1572369592190332, + "documents": 513, + "lengths_sha256": "d11985e121377244a83f6693db589940ae4899c1bf48610d0e00341a06f525bd" + }, + "1m-heldout-trimodal-513-seed20261116": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 606.0254821777344, + "min_ms": 603.9866943359375, + "max_ms": 608.9718017578125, + "allocated_gib": 71.13527822494507, + "reserved_gib": 73.546875, + "prepare_ms": 20.425522001460195, + "layout_prepare_ms": 20.0147470459342, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 4.999916918575764, + "max_spans": 240, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-contiguous.json", + "raw_sha256": "e787b7d95ec05a91a114359d1ab494d627217bdd711ce034667b52a13bc9434b" + }, + "planner": { + "total_ms": 498.80828857421875, + "min_ms": 495.22216796875, + "max_ms": 500.7232971191406, + "allocated_gib": 75.91035795211792, + "reserved_gib": 85.07421875, + "prepare_ms": 28.75487704295665, + "layout_prepare_ms": 28.326248051598668, + "policy": "selection-local-6x", + "relative_rms": { + "output": 0.0009195983309898245, + "dq": 0.0003224729936723083, + "dkv": 0.0005545372026871616, + "d_index_q": 0.0004788406854099131, + "d_weights": 0.0004916242112471586, + "d_pool_kv": 0.0007596727225373355, + "d_pool_score": 0.0007537076166265762, + "d_ape": 5.542803593958258e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 7.999753892421722, + "max_spans": 243, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-planner.json", + "raw_sha256": "a9d6fb26a731bfdea6c8af827da8d3fb53f15d16fd7edcc6b1d54c02c71082c5" + }, + "zigzag": { + "total_ms": 509.7503662109375, + "min_ms": 507.0311584472656, + "max_ms": 511.49639892578125, + "allocated_gib": 72.71457099914551, + "reserved_gib": 75.626953125, + "prepare_ms": 38.119480945169926, + "layout_prepare_ms": 37.66501497011632, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009651923554078656, + "dq": 0.0003559762003050189, + "dkv": 0.0004964050388006907, + "d_index_q": 0.0006082251240356637, + "d_weights": 0.0005475287924962972, + "d_pool_kv": 0.0007222207414950428, + "d_pool_score": 0.0007242622401541285, + "d_ape": 5.483300095764783e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 7.499938875436783, + "max_spans": 847, + "raw_file": "layout-1m-heldout-trimodal-513-seed20261116-zigzag.json", + "raw_sha256": "a7707b4d3b17d281581ddee91ba773fdbb2646e2054a68873f5bb0aebb316db2" + } + }, + "speedup_vs_contiguous": 1.2149466960743227, + "gap_to_best_baseline_pct": -2.146556110994702, + "prepare_plus_iteration_ms": { + "contiguous": 626.4510041791946, + "planner": 527.5631656171754, + "zigzag": 547.8698471561074 + }, + "prepare_inclusive_gap_pct": -3.706479129001228, + "documents": 513, + "lengths_sha256": "30e8a9b80449ecc324bba6134d04db805dac074f3b7693579d6f2afb153135f7" + }, + "1m-near-uniform": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 261.5732116699219, + "min_ms": 261.14141845703125, + "max_ms": 262.1058654785156, + "allocated_gib": 65.3901014328003, + "reserved_gib": 67.666015625, + "prepare_ms": 3.4267649753019214, + "layout_prepare_ms": 3.1357220141217113, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.3920431137084961, + "max_spans": 8, + "raw_file": "layout-1m-near-uniform-contiguous.json", + "raw_sha256": "0341b198790d2b51903794067c2aecc73345790bc795019ab793fd43fa3e4991" + }, + "planner": { + "total_ms": 261.5891571044922, + "min_ms": 261.0076599121094, + "max_ms": 262.2134704589844, + "allocated_gib": 65.3901014328003, + "reserved_gib": 67.666015625, + "prepare_ms": 3.440934931859374, + "layout_prepare_ms": 3.165671951137483, + "policy": "contiguous", + "relative_rms": { + "output": 0.0015901644171629613, + "dq": 0.0005902133591829151, + "dkv": 0.00047656249471409655, + "d_index_q": 0.00026291229043044637, + "d_weights": 0.000199747845851943, + "d_pool_kv": 0.00029679011974003083, + "d_pool_score": 0.00028698171976176143, + "d_ape": 6.356026201118735e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 0.3920431137084961, + "max_spans": 8, + "raw_file": "layout-1m-near-uniform-planner.json", + "raw_sha256": "9798272f912bf142eb9127388a691e04d11e776a1c7423d3f2f5f9df8619fac8" + }, + "zigzag": { + "total_ms": 266.84141540527344, + "min_ms": 266.42767333984375, + "max_ms": 267.36077880859375, + "allocated_gib": 65.50305652618408, + "reserved_gib": 68.41796875, + "prepare_ms": 10.221436969004571, + "layout_prepare_ms": 9.95832693297416, + "policy": "zigzag", + "relative_rms": { + "output": 0.0016123834602235457, + "dq": 0.000596027757416673, + "dkv": 0.0004744482659872333, + "d_index_q": 0.0003024936753378096, + "d_weights": 0.00022482807025107514, + "d_pool_kv": 0.00029945938494957544, + "d_pool_score": 0.0002934684618935451, + "d_ape": 6.307271516024825e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 0.4991369843482971, + "max_spans": 128, + "raw_file": "layout-1m-near-uniform-zigzag.json", + "raw_sha256": "3f1624f6a8368598fe04d11064f2689db4a6f15a9c78f9e45f8bdcb05df1a07c" + } + }, + "speedup_vs_contiguous": 0.9999390439774078, + "gap_to_best_baseline_pct": 0.006095973845532399, + "prepare_plus_iteration_ms": { + "contiguous": 264.9999766452238, + "planner": 265.03009203635156, + "zigzag": 277.062852374278 + }, + "prepare_inclusive_gap_pct": 0.011364299540339573, + "documents": 64, + "lengths_sha256": "f7b3e262fd896136c35c22e4766cae5a4b416484124973745b17cbc8ceac3850" + }, + "1m-power-law": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 612.9796752929688, + "min_ms": 610.664306640625, + "max_ms": 614.4771118164062, + "allocated_gib": 68.94974088668823, + "reserved_gib": 71.298828125, + "prepare_ms": 4.962046048603952, + "layout_prepare_ms": 4.639261984266341, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999988242983818, + "max_spans": 5, + "raw_file": "layout-1m-power-law-contiguous.json", + "raw_sha256": "6429bf4696405989be5de571ee173b6ca4acde3df0aaaa4c786cf941014001e3" + }, + "planner": { + "total_ms": 492.8457794189453, + "min_ms": 490.5483093261719, + "max_ms": 493.98095703125, + "allocated_gib": 71.04951000213623, + "reserved_gib": 81.416015625, + "prepare_ms": 17.59058702737093, + "layout_prepare_ms": 17.31892500538379, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0013432263284307595, + "dq": 0.00048809261707209576, + "dkv": 0.0005241199919794483, + "d_index_q": 0.0005870922763788422, + "d_weights": 0.0005419402939198647, + "d_pool_kv": 0.0006357484486274489, + "d_pool_score": 0.0006410094965115505, + "d_ape": 1.1075007383756194e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 8.731149137020111e-11, + "workspace_gib": 5.999798443168402, + "max_spans": 11, + "raw_file": "layout-1m-power-law-planner.json", + "raw_sha256": "a6abe46aecd4b94aff2226f824224590238014f009c4258ae1b44969ec53371a" + }, + "zigzag": { + "total_ms": 493.15296936035156, + "min_ms": 491.9137878417969, + "max_ms": 497.9830627441406, + "allocated_gib": 68.97773599624634, + "reserved_gib": 71.888671875, + "prepare_ms": 7.186377886682749, + "layout_prepare_ms": 6.911287899129093, + "policy": "zigzag", + "relative_rms": { + "output": 0.001332787454565865, + "dq": 0.0004750203554706029, + "dkv": 0.0005353831251106813, + "d_index_q": 0.0005565436942807204, + "d_weights": 0.0005130840726434935, + "d_pool_kv": 0.0006657684779061981, + "d_pool_score": 0.0006483417130847936, + "d_ape": 1.1188895558466485e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.9999998174607754, + "max_spans": 14, + "raw_file": "layout-1m-power-law-zigzag.json", + "raw_sha256": "2e25f55ea691125c1c3adb2d2e7bf27af6d94db7c98b2aa0f4070eef7dd2aaaa" + } + }, + "speedup_vs_contiguous": 1.24375555374677, + "gap_to_best_baseline_pct": -0.062291005122550036, + "prepare_plus_iteration_ms": { + "contiguous": 617.9417213415727, + "planner": 510.43636644631624, + "zigzag": 500.3393472470343 + }, + "prepare_inclusive_gap_pct": 2.018034211148434, + "documents": 7, + "lengths_sha256": "b787057eb34fe32408b2fe31d07c9bc6642c12e889e2e2a42e790f32a7ce58d8" + }, + "1m-single": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 879.3570251464844, + "min_ms": 876.3743286132812, + "max_ms": 884.391357421875, + "allocated_gib": 72.9497447013855, + "reserved_gib": 75.298828125, + "prepare_ms": 6.599797983653843, + "layout_prepare_ms": 6.2927789986133575, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 7.9999911189079285, + "max_spans": 1, + "raw_file": "layout-1m-single-contiguous.json", + "raw_sha256": "de07cbd27df21fd95be7f54d41bd223ef072cd85db0e59944fbc0776603a1610" + }, + "planner": { + "total_ms": 664.8384704589844, + "min_ms": 663.4234008789062, + "max_ms": 669.0645141601562, + "allocated_gib": 72.95559072494507, + "reserved_gib": 75.802734375, + "prepare_ms": 15.023829997517169, + "layout_prepare_ms": 14.757112949155271, + "policy": "selection-local-2x", + "relative_rms": { + "output": 0.0011529975036186093, + "dq": 0.0003767490804819508, + "dkv": 0.0005946098839724117, + "d_index_q": 0.0006048463254406639, + "d_weights": 0.0005030932802858819, + "d_pool_kv": 0.0008104081461652777, + "d_pool_score": 0.0007695389939561969, + "d_ape": 1.2836426006061245e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.9103830456733704e-11, + "workspace_gib": 7.9999911189079285, + "max_spans": 2, + "raw_file": "layout-1m-single-planner.json", + "raw_sha256": "dface57613b8b8bdec4b67f29b41e3a4814f217e27e436fbd73114e1b6f7db98" + }, + "zigzag": { + "total_ms": 667.4992980957031, + "min_ms": 662.85498046875, + "max_ms": 671.0496826171875, + "allocated_gib": 72.95510244369507, + "reserved_gib": 75.80078125, + "prepare_ms": 6.756177055649459, + "layout_prepare_ms": 6.48829503916204, + "policy": "zigzag", + "relative_rms": { + "output": 0.001182160242463775, + "dq": 0.0004037382914331386, + "dkv": 0.0005202209349556262, + "d_index_q": 0.0006295794340313956, + "d_weights": 0.0005169888293507331, + "d_pool_kv": 0.0007736514408215355, + "d_pool_score": 0.0007362455606161046, + "d_ape": 1.444483038268409e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 7.9999911189079285, + "max_spans": 2, + "raw_file": "layout-1m-single-zigzag.json", + "raw_sha256": "ae689331fefbba2baca40beac27367e73c7758b5e7833cc80dfe10fe1c55d506" + } + }, + "speedup_vs_contiguous": 1.3226626680303306, + "gap_to_best_baseline_pct": -0.3986262823511222, + "prepare_plus_iteration_ms": { + "contiguous": 885.9568231301382, + "planner": 679.8623004565015, + "zigzag": 674.2554751513526 + }, + "prepare_inclusive_gap_pct": 0.8315579942291818, + "documents": 1, + "lengths_sha256": "df57f4ef94807b0499d37da5e83d5c5333272a5ebcad42ac3dbec6ac94a30bf2" + }, + "1m-uniform": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 10, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "total_ms": 261.7406768798828, + "min_ms": 261.2946472167969, + "max_ms": 262.6141052246094, + "allocated_gib": 65.49799585342407, + "reserved_gib": 67.916015625, + "prepare_ms": 21.517536020837724, + "layout_prepare_ms": 21.237667999230325, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49993896484375, + "max_spans": 8, + "raw_file": "layout-1m-uniform-contiguous.json", + "raw_sha256": "1be5b4895931639b8ffdae843a7316effeb9f792a8fdb2cd7344db3d5933f273" + }, + "planner": { + "total_ms": 261.7190704345703, + "min_ms": 261.2510070800781, + "max_ms": 262.2694091796875, + "allocated_gib": 65.49799585342407, + "reserved_gib": 67.916015625, + "prepare_ms": 4.200693103484809, + "layout_prepare_ms": 3.913125954568386, + "policy": "contiguous", + "relative_rms": { + "output": 0.0015967309781703524, + "dq": 0.0006004709784135087, + "dkv": 0.0005059608205209192, + "d_index_q": 0.00029251105906248924, + "d_weights": 0.00022732957796148034, + "d_pool_kv": 0.00029126673820099435, + "d_pool_score": 0.0002934495651972289, + "d_ape": 6.876366322072345e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 0.49993896484375, + "max_spans": 8, + "raw_file": "layout-1m-uniform-planner.json", + "raw_sha256": "0c3dee88bee4c9bd715e8e224758cbef62730da61683099646ce4092be28962f" + }, + "zigzag": { + "total_ms": 267.4135284423828, + "min_ms": 266.8036193847656, + "max_ms": 267.57830810546875, + "allocated_gib": 65.5038571357727, + "reserved_gib": 68.41796875, + "prepare_ms": 9.938867064192891, + "layout_prepare_ms": 9.64121101424098, + "policy": "zigzag", + "relative_rms": { + "output": 0.0016095293393893419, + "dq": 0.0005985745463675235, + "dkv": 0.0004895944404512913, + "d_index_q": 0.0002713512548711526, + "d_weights": 0.0002148084702065561, + "d_pool_kv": 0.0002933053408000937, + "d_pool_score": 0.00029111625679209704, + "d_ape": 6.765475676396151e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.49993896484375, + "max_spans": 128, + "raw_file": "layout-1m-uniform-zigzag.json", + "raw_sha256": "763cd6a9c47b94fed97a8ac9aac15ece2cc739a8a320973118e87c63096ddd2d" + } + }, + "speedup_vs_contiguous": 1.0000825558690722, + "gap_to_best_baseline_pct": -0.00825490541632723, + "prepare_plus_iteration_ms": { + "contiguous": 283.25821290072054, + "planner": 265.9197635380551, + "zigzag": 277.3523955065757 + }, + "prepare_inclusive_gap_pct": -4.122059933046273, + "documents": 64, + "lengths_sha256": "dd1b660aba10a40fca5d0224dbbcfdec6e38447a6ae6c9aadae9aa252935fd2e" + } + } + } + }, + "measured_source_sha256": { + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "9100c6359659a85d024340878a7ff1c395ba20b51ecf2668d308a9066304a231", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "0fe29b40204b5f59976517353c63a5ed5e2ea1033be1bb58b68aeb1c703a3aa3", + "cula/distributed/thd_planner.py": "6b10315a430627461d5ecb7941b8bd1e7bf5411cf4afa3057317070fda80867c", + "cula/hybrid/__init__.py": "74934cf8db62dd1f237a30ee6aca283aa7a38483c5ed921da61aee09ea7ebbf5", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "6b4a64ed35b2b70e41fc978fb4db3a8fd60b85fe33160c5d4fa02afb0ca4b52e", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "benchmarks/bench_full_csa_cp.py": "39bdd101b52253d9da338b94f259cbde4a0f1f07dda9d137231c2b1dfe653bfa" + } +} diff --git a/docs/results/thd_selection_review_20261115.json b/docs/results/thd_selection_review_20261115.json new file mode 100644 index 00000000..7c8e97a4 --- /dev/null +++ b/docs/results/thd_selection_review_20261115.json @@ -0,0 +1,1038 @@ +{ + "cpu_tests_passed": 337, + "dsa_rank_plans_metadata_equal": 2232, + "profiles_and_planner_decisions_unchanged_cases": 62, + "source_sha256": { + "benchmarks/bench_cake_vs_flashkda_cp.py": "22f4eb718c2bb6a741deeff3f4c8627ade3c158267a1c3c635055ef93806fe9d", + "benchmarks/bench_chunk_delta_h.py": "9bb20f7275e2616aaa7ed86cc30624a8c5ebf8caac1edc6cd8677373faea173d", + "benchmarks/bench_csa_layout_matrix.py": "774f54d9032b3321543df3aba1554dba1d093b802fea1952c3b03ecf8fdd7a5d", + "benchmarks/bench_full_csa_cp.py": "2acb5c13043c7776ce6b029620310335f2bbb032ce58118b39fe8db23dfb70d2", + "benchmarks/bench_fused_kda_conv_mtp.py": "bc5654425e0094d3cbef84afc92c2fca7d2a31a2b8658892f83ed5b5ad1b3ccc", + "benchmarks/bench_fwd_o.py": "3492461eec0444a399019b18a39322fefe2a05d99adcd497ca34047ed3da616d", + "benchmarks/bench_hybrid_cp_emulation.py": "518956297d26eec2ab6ceec8adacdf504dd5fa83d92b7ba552fe48acffdcc3dc", + "benchmarks/bench_hybrid_kda_dsa_e2e.py": "31c08ed37213e0d034f310f2c527ef5d1986fcbd3fc660eb07be3f16af76e5fa", + "benchmarks/bench_index_pool_cp.py": "943c955d09d141f74854c399688b2828d3d8f74f5d0f5ecbe9fe172cfdf7b606", + "benchmarks/bench_kda_bwd_wy_dqkg_sm100.py": "83af62eab1c1ca48065544a9fd671d74a6ae3d5207b97866e466347b5034214d", + "benchmarks/bench_kda_bwd_wy_dqkg_sm90.py": "94d5bf6f4ba39984153870ba356ad1eae5d741eb3d77de2ff4e4c886f066a50b", + "benchmarks/bench_kda_chunk_intra.py": "776f1e731c3af1165f9dfd5b7b381b1954706db51ec4091a104968526be8db70", + "benchmarks/bench_kda_decode.py": "370129509fc915c4bc3e869e3e5a4475b033a5c7cc2265d27f22d88da82f1938", + "benchmarks/bench_kda_decode_mtp.py": "0fb2d356360d5adbb690e4a2b81c4ca8ef33be9606be13e4b28b241083abe053", + "benchmarks/bench_kda_independent_shards.py": "a7d06bb6f9cd7c14c2ee792baf64e094a35e745f81a22c948d6b8608ed16ac1f", + "benchmarks/bench_kda_packed_decode.py": "54aa9bf143dbcfcd3e560812c5c2366601596fd2f5c2cbdf979c0bdf5e1bbb32", + "benchmarks/bench_kda_sm90_cp.py": "0a4e4e3eff0bd1b85e0340b6e516a0e1724011d49eea845e30ac37b0ea910e32", + "benchmarks/bench_kda_sm90_prefill.py": "70e5fac3ea7dec7ffb32b0ad9080387987f3db8a045412c76dfe5120fd9ba841", + "benchmarks/bench_kimi_cp_e2e.py": "1505dc676af3f4fa1a0cbaea2545d819b349ccc3810c895911cca18f6faa78e7", + "benchmarks/bench_la_decode_mtp.py": "e333402d00ac90f96662266f31a07093bca73d02d2c29d22dfafbc2e5427adf5", + "benchmarks/bench_la_decode_vs_fla.py": "6c84b7bc1d621c653707beb8b3fb98fe07c3093f6506e9e8d7f4c4f3339c4230", + "benchmarks/bench_la_kvbuffer.py": "4ff99a098e58baa9927ae442735b0b50994068c4171933414ad02b47dc46bb21", + "benchmarks/bench_lightning_attn_prefill.py": "e47cc45cf6ef1d8ab8a1c567640cc6e92d76dafb14df700c0d8e2be425d89ab5", + "benchmarks/bench_linear_attn.py": "a86cdb6a7a69dabb7de553309b9a61783b54419bfc10a74eb36128dc32ddb327", + "benchmarks/bench_recompute_wu.py": "4306d27d1fe747a8b226ea13adb70a861a7f20047582835d6dee5db47cc3c28f", + "benchmarks/bench_thd_transport.py": "4d50912e6c143d3c68889cb14722afde283d552c56fc575758a48ab3211d5c40", + "benchmarks/calibrate_thd_selection.py": "90cbb34c0cc341b972692d8081758de48d1b8a34fe82e92c86d10bc9d22f2792", + "benchmarks/cp_layouts.py": "10cfe493d9563e93f6e5eed56a81db805b1543979d7e42cd2fc17adaaced8894", + "benchmarks/fit_thd_transport.py": "20ce22c2a71a747492004a459379f2d902220e37fab79ba05f7889ccc18f3c0a", + "benchmarks/large_cp_emulation/__init__.py": "62e0fbc6c5113bb7b59ee1a3c0ceaa60143c893aa8103801c6c82fcb8d8e43b1", + "benchmarks/large_cp_emulation/bench_components.py": "567fe03bbb9b95b6049a8f7337b665499991215ca7a86ab3aabcbb1d48327600", + "benchmarks/large_cp_emulation/bench_dsa_layout.py": "410cd55e9be6c015848454f474e48d23f85556f9aa324a05dc81c1299d5418c9", + "benchmarks/large_cp_emulation/bench_hybrid.py": "1e99ca51e4ba27336d268494298948613f2aab87ca067104377cd0deb644db2f", + "benchmarks/large_cp_emulation/bench_kda.py": "1c78c021bada79c2edbc6af5e1bc37c441d24d70e1651b9ab623ddba6ab3c15a", + "benchmarks/large_cp_emulation/bench_thd_planner.py": "ca0ca5b93401240988c168848820f4ae35bd3591636bf44469bd6acdf1c9c779", + "benchmarks/large_cp_emulation/query_layout.py": "62318e810f1c22f46da9c5033640e721b13dfa7688fe568bdbb7d5654ca02e4f", + "benchmarks/large_cp_emulation/topology.py": "e570cc80c1b7855d72af324e633b3250c55968a3215d137b0f8a9b70469e0aba", + "benchmarks/large_cp_emulation/transport.py": "c9a762491e150881fd3360a127f69ce2ebc580cb739d061cc883ff49264d87fe", + "benchmarks/large_cp_emulation/workload.py": "383f23380d7bb340ba2c03c98cecf63b520fb455e89de345389b97a4a3be9f8b", + "benchmarks/profiles/README.md": "4b15ea28c35d6dbcb65fb4be02d233fd82a93bedfecacbc9df08926ac290f0a6", + "benchmarks/profiles/thd_selection_b300_cp8.json": "d166704b094b2c6aef277290e0d77672b4ea06cbd8f24bdd110ebb316fe1eb72", + "benchmarks/profiles/thd_selection_h20_cp16.json": "63839a024e0120c256576a159e555441c1f4850524af07d10ea0b1cc5fc774de", + "benchmarks/stress_kda_sm100_cutedsl_determinism.py": "42f5449481c5880556a7adaff6b559197f9ada5c21d405987235598c37df6d06", + "benchmarks/summarize_csa_layout_matrix.py": "058a536d36c0117f70ebf6309d290909a537bfa7e816d9cf04ea58b2495c8d0f", + "benchmarks/thd_1m_cases.py": "de6364aad2b098281c2241b231f64d645f3a91aab53f5fd106cdcec097c843c5", + "benchmarks/utils.py": "a6ae5cc9081297f890db3f58de507117b7720aa748fc68d2c560bf6d48c1e032", + "cula/__init__.py": "5a25fdb2904d863a4ba4065c5d7ad66ef393044db74dc7e859aded1cb17bea51", + "cula/_metadata.py": "990bd5ed2a5fb559ad65020d140356d995f83ff84c4363148f24800dcf122a6a", + "cula/distributed/__init__.py": "958ae5ab187f57e8eefe9aeee665ff6c33f5a70e9e37828ad3b20fc1a02ab0fb", + "cula/distributed/kda.py": "9100c6359659a85d024340878a7ff1c395ba20b51ecf2668d308a9066304a231", + "cula/distributed/plan.py": "ebfdbbdf89de090eec4d5f3b761f7ea9c8e1efaa984dc33699a6a1bcb3ebec61", + "cula/distributed/redistribute.py": "b923570e51067143cf99919ca3b11e56b47308dd7f44ba6d3daa9fc34d7a0e65", + "cula/distributed/thd_planner.py": "6b10315a430627461d5ecb7941b8bd1e7bf5411cf4afa3057317070fda80867c", + "cula/hybrid/__init__.py": "74934cf8db62dd1f237a30ee6aca283aa7a38483c5ed921da61aee09ea7ebbf5", + "cula/hybrid/distributed_pool.py": "eed4426ef2ba6aa8975694a77dee94ac246ae932e6ab3dd2813f948b3eb1e39d", + "cula/hybrid/dsa.py": "fdf845764b609034cb9ed45170dc5e1aa4b3dcecaf268671b6897c5aceb321c7", + "cula/hybrid/index_pool.py": "2c9774f806aba003bbddbbcb8a39999a4ae7ed1724d212270f54e4007d9e9f76", + "cula/hybrid/plan.py": "136d0c7dcb35ad17b0508fe46cb122adc8f20fa0374e90ccfe926118e7ffb033", + "cula/kda/__init__.py": "10d8b5ce99d3d142a30fe62ac73f2107ef259e56dbfdd6435ed2b6117525b894", + "cula/kda/_flashkda_arch.py": "a81ec06759874dfb4e3360afbd7731799b64ce7d10055cb8c39066d82eecb64e", + "cula/kda/chunk_intra.py": "0f2d798d13bcbaf4274f137ab9ba51e986ba06bac0dd2a3e4e8d0cf13ec74bf5", + "cula/kda/flashkda.py": "6e05a366b6a96a22b9547332c57d9d9e3d93f5708340b11c890ca47bec998037", + "cula/kda/prefill.py": "0ff98b5116a5675b61aff245a4b6dde5e6c77f22ee75b43dfd6d94341249034e", + "cula/kernels/__init__.py": "c6fb97b107d4892906ada76bbb757aa7230b93809c858157a89d157ba3595bd5", + "cula/kernels/_lazy.py": "b277af0c363be2982db4fc05a8d5df2af3022a1fb6b7210bb756eae62ade72d6", + "cula/kernels/kda.py": "16b98bca551287748457276222544d123640d28773af2b13e30b129e6fcd86bb", + "cula/kernels/lightning.py": "95c7ac09305fec1e6512c4e708e6a327db81c89ad5fe37b8875d6718cb5155c0", + "cula/kernels/registry.py": "9e5a56255d38ac48e50f6801dddbd6e0f3e110baa4a06b3c335c83e8d6c49d34", + "cula/lightning/__init__.py": "76be8323c8b30977e440e88bb92af94e8c973e055b3d2bded103f949032e103c", + "cula/lightning/la_decode_mtp.py": "e343baead10558469ee2fd2f8bf229c9155a32ab8f39577cf50f1476d24f5fa2", + "cula/lightning/la_state_update_kvbuffer.py": "8b0f1408ee292ccd0deb51798f53f8e3206551294d8d45a54ed32431a4f58fb5", + "cula/lightning/la_verify_kvbuffer.py": "18835bde1d1536e2238a31063848024cbd54ff22c2f6e2005c6fee08da7a241c", + "cula/ops/__init__.py": "50c4eb7787b88701db989bd4c109932466c30a4d2db48d76ef07c3e867b51876", + "cula/ops/_mlir_compat.py": "65bf6568071a4b0d4d947e85f652b53b952c21926f6c78196123f9692af86605", + "cula/ops/_nvvm_compat.py": "08a524afc6f536682d5f561de98223ef4e3ecfeff8fdbe18419042a988a31296", + "cula/ops/experimental/__init__.py": "e40e284b3eb0c05e7fe4ddf9cf2479ebaed03f41fbcc64b7799afae149925d72", + "cula/ops/experimental/linear_attn_prototype.py": "0fee460e7b535c5d479de1e83344a3ff86185b62c082711553fcdf7ecb995a14", + "cula/ops/inv.py": "c18a7431829e2b3b4a4074f83a719929c5ef8a52332b405a65659767f5e6af43", + "cula/ops/kda/__init__.py": "3da85398d754e7167aa65f9d4b09e0535081d2974aceaa722b99222ebf5ba164", + "cula/ops/kda/cp_mode.py": "ea4e3f8ed797d026b5cc0a9c81b5cd9c7a0c6b1562d3fd7b87351a93263c2a46", + "cula/ops/kda/decode/__init__.py": "fbc3cbb3c04d7ed8bae38e0564db6d0ae65e104e1f8e08d55b761ac4d3a8cdcd", + "cula/ops/kda/decode/cute.py": "01f15dd6b395d90ebb6c99add31bcf03214ddf4c68aa629a3870eea40db66507", + "cula/ops/kda/decode/mtp.py": "3cbf38ece95914d342f4438fb95106e419997fa8be82ae709ee9bfbe6a2ec3f5", + "cula/ops/kda/decode/mtp_conv.py": "b2ce72a65bfe350ad2220f547458bf12f20362f183a80d6c8deba2f8ce68c227", + "cula/ops/kda/decode/mtp_conv_kvbuffer.py": "b93a9949cc12f34c21460d804601c88ba9051621190b25d5f072531d8d03f83e", + "cula/ops/kda/decode/mtp_kvbuffer.py": "8c747c0eae35876c3c13baa3055c0d8d7fddf4243f4cf65d387ad9b8f51ef170", + "cula/ops/kda/experimental/__init__.py": "327d1c662d8fa038752823f09828e09a4f67d8629e21dcc97dbb98fa918beacb", + "cula/ops/kda/index.py": "0d1f9df62e1919da5377591d28b9ba8ac9873401b2b70a06931c78dc761a1d37", + "cula/ops/kda/sm100/__init__.py": "90ac67656a2e0208a55caf6682e4e1264be9bdc96b31e2f1cffe2e4239cb1c36", + "cula/ops/kda/sm100/akk_inv_fp32.py": "ba7517097f0334e2fe14f89841c19423ba9a5afbdf73d7559ef9a5d62de25660", + "cula/ops/kda/sm100/akk_inv_tf32.py": "2de2ff81ce7b8532fae2a0a57a061d4ba0fa5bb31c5d505d07083ad350a49879", + "cula/ops/kda/sm100/bwd_wy_dqkg.py": "105eb23c9739e2a035185b695b8b3dc9e757c7d3ede2da61e1f33c8c9ce1da8a", + "cula/ops/kda/sm100/cp/__init__.py": "06b0d65744bb8c26d0eff4945b58e7b0af8169e2da8ef4a49df9da7ff6dd2784", + "cula/ops/kda/sm100/cp/chunk_delta_h.py": "2e185452d15ab6f5d564f8f799c086ecd97eab08a23a8b0860d5d0682b49c7e9", + "cula/ops/kda/sm100/cp/merge.py": "cede190c72506984ee52dd336ee4c9c1215f07bc0b2254a9b8376dc9d60366b1", + "cula/ops/kda/sm100/cp/pre_scan.py": "b429c8a9a3bbd69082705f0efcb6ef0473580952bb0105c503986fecc9c63f36", + "cula/ops/kda/sm100/delta_h.py": "9b97d709e1617d8645da011083143101c0d621c722e6fc7bad1839a43287579e", + "cula/ops/kda/sm100/full_port.py": "12088a83c2d3f06b2f9e01f38d37750e8e1d3c2b2254f79a9554091b079ddf4f", + "cula/ops/kda/sm100/fwd_o.py": "6b324c1233d77ac11a7b5f8b75ce987c5e333d85f63220121bb4233fda954012", + "cula/ops/kda/sm100/intra_fused.py": "4ca489f46a7a01a542786f976a808a8e0de918aba6d0be497b017df43f0041be", + "cula/ops/kda/sm100/policy.py": "9ff71246e28f7b2e13b4e0e2b7affafd3bd4f67377221523433159730583d325", + "cula/ops/kda/sm100/recompute_wu.py": "6f633dc8d24adf138f89282a9a58102ab87f1d4e93b7d9cc08af219dee1a424b", + "cula/ops/kda/sm90/__init__.py": "3211cfbeb1372e6dc2a3a1ea2f39e590096eb31e6bedaa4c756dcd03f6b67e16", + "cula/ops/kda/sm90/_common.py": "44c37985bb9ccc381beddcb783c47bebaea6a96616fed8ba1449ee29872c51ac", + "cula/ops/kda/sm90/bwd_wy_dqkg.py": "c80e66da0a9029178f64bb6d1c9e55a08f7a00aaa01e44d14f4de636c6ec29ac", + "cula/ops/kda/sm90/cp/__init__.py": "319dec8e1bcc10d79412c714b9ac356f4aad93cb46a7034da04a98059f9c08a2", + "cula/ops/kda/sm90/cp/driver.py": "c2151d77e80f63fb0c34e81b4508663e6abb6b2e8f7c8bcb369d1abf3e9ea159", + "cula/ops/kda/sm90/cp/merge.py": "0021d5ff717329acbd9f514e26c334f7500edd8e84dfb1d4073f904e76392882", + "cula/ops/kda/sm90/cp/plan.py": "f4651f616e99097d9d44f84d5da8a7049c16d1c6a43e45c049580faf11bfbacc", + "cula/ops/kda/sm90/cp/pre_scan.py": "8135b11ff230f7064806860501a38abdcb86084777a9f2f4ef532cd012cdc54a", + "cula/ops/kda/sm90/fwd.py": "ff332993ccfcd786404a8e393cd4396d0f11debf18ac5a764b1849e52be00362", + "cula/ops/kda/sm90/k1.py": "3bf83a45c027a2ddf72330493e66b568ac4881a6d18513367b226cc59ec8ae34", + "cula/ops/kda/sm90/k2.py": "6b85f5decbeff196bd75a5d6aa1358f10dee19cab41ed6ef8138ff1d87a16044", + "cula/ops/lightning/__init__.py": "6caf8450686e1f336db810e01b40e9a26b69dc73036a4e657bf87ea9547d27b7", + "cula/ops/lightning/decode.py": "7a0ddacd68a5c7b08f70bfa514b13cc708e9eb6d8d686365e87b16b4b846af45", + "cula/ops/lightning/prefill.py": "68b153478488342ddc3f47f6d6b7c7249c47f1d56b6b1630a1320ac305c4332b", + "cula/ops/lightning/prefill_sm100.py": "d324cdd1752ee7e687eaaef2f5255c72dc3ceadfdfda2fa0f89d45e959d55b43", + "cula/ops/lightning/prefill_sm90.py": "13be17f006d59a32d8ddbc9d1248681b5fac522cee3062186f0b97cda9a3ccf6", + "cula/ops/lightning/sm90/__init__.py": "0c4020d171149e44a2f8db52d9bb99956b8d425461313761711d23e4f9125f75", + "cula/ops/lightning/sm90/prefill_kernel.py": "9801dec0b0df3d4f678fe28190012398c307017e7e984f60b99e57541f9de4eb", + "cula/ops/lightning/sm90/schedule.py": "8ad5a96fe5190410c84420febc0005ed47de739a779fbcc42b17b5436123c6ac", + "cula/ops/ptx.py": "184889154fb9d9823a2dafdb34f12795270dc2001580dcb274e9b1e6601db5e4", + "cula/ops/sm100/__init__.py": "0ea3c69d1b9d035399c3f533ac35877cb4808be9967d1c17915eb119ccb11aae", + "cula/ops/sm100/ptx.py": "fb9e5913fdc18e00d7481a7243d2367f82f8de0bc1692148d31bbce5419518ee", + "cula/utils.py": "0c4d8632316f7b0d6fd6e500ebe51a25967c3d0fa2902323f05935fed899ee34", + "pyproject.toml": "56a112afeea1bfd5f334853f3f34e92afe3dd26ad025a73ada3075bb1aff7f25", + "scripts/modal_validate.py": "048d89dc199da77a4a38b390fe6f02275ea4ec4123a8cb05fe9778fa23bd773a", + "setup.py": "6b65a6be394a519d1a1f79db18c60a20479e6065c9fe4c0f9a27474d8f58d6b3", + "tests/__init__.py": "0f0f8fcb798517a7b8cf062cf3bcbcc7f3b245a3431590f85a40b4150e6aa857", + "tests/conftest.py": "ce2c8122dc096d503d5904e612d4656f592293d96456955856899acfd2199127", + "tests/lightning_attn_reference.py": "5072c4000e3e51a6887e24b5820e8364d8f0ca5216f282506f11cc366321f29d", + "tests/test_chunk_wy_dqkg_sm90.py": "b3275ae3e718ceace6a315dceefd19c4bc0386dd8a3f453ae1b40af4791d8f17", + "tests/test_compare_with_fla.py": "a32febc8fc73cab1e751cd2daa9ee31ae022136b8cf37613836cdf72f416512c", + "tests/test_csa_matrix_report.py": "35b201871efed3e85ec540724d2f8e9f24b71eae0a471143183ccde2777f1411", + "tests/test_cutedsl_compat.py": "1abe8b60c888f1bce045c4b881fb201976b3c80004f32ed3c1a519fb4cb7f573", + "tests/test_cutedsl_mbarrier_init.py": "e408a02c3d1f547576277f7d73e019f951c54202c69354ed81083f946f7008d6", + "tests/test_full_csa_cp_metadata.py": "c44aca48dff4765fb850b6203cccb9269e203fffc9d87b0635ca5a23db411e48", + "tests/test_hybrid_distributed_pool.py": "071ece1d2f11154500486c5948d11e51d5f53bdcc561c76066841cacb2af5fd2", + "tests/test_hybrid_dsa_cudnn.py": "25d14e60911a4dfee55cf561c39db180bfbb59eb3996cc94bd16067a6d5d4602", + "tests/test_hybrid_dsa_helpers.py": "642c55c8536011c8afa65cc101d55c2b0c56968cef30c91c2e42e719e7154d6c", + "tests/test_hybrid_dsa_plan.py": "647102932147049e6569e4896e094c867de6211124548d97d98d704e9464d0ed", + "tests/test_hybrid_index_pool_helpers.py": "d3eb6cc43cd023872039e94c3928b4f253a51e6582a75d5c07519d79130fe6da", + "tests/test_hybrid_index_pool_plan.py": "aa47c241c3c55e386da4f716ab7b83626e170b7a9c3dd81020bdba2916444f2f", + "tests/test_kda_conv_mtp.py": "484743cdc3b7c50d4a196caaa2c00119f3f1631124b5d5999f0c1d200310f053", + "tests/test_kda_conv_mtp_kvbuffer.py": "6fa813cf8f9c25caa79756dd2b58070036c83610313db4529bf3dfd202df2527", + "tests/test_kda_decode.py": "fcdaca68b9773dea3a8822ec571daa7a6898b993ec78fab259a3969a02276afc", + "tests/test_kda_decode_mtp.py": "45224002328dbace1a9250f4944c7e5c91f6804efd26f221ca278c42f293be9c", + "tests/test_kda_flashkda_arch.py": "7d0b125df694743b97429434537608cd09ce288897cba75cdd2489fea2e27374", + "tests/test_kda_flashkda_pipeline.py": "4598953061b94e593da29775b85692632e41558f39fb0916e5bb2d3ceab3e9c6", + "tests/test_kda_flashkda_sm100.py": "427cf2782a86b2143b2315b31e77dc94d4495f762d05ce688606f515e01a8599", + "tests/test_kda_packed_decode.py": "1eed4f70b8ca7152a7205a12faaeec8e9c605a5790b215e8016e447cde029242", + "tests/test_kda_prefill_api.py": "10c913455885e763c4b64f64b222aba01a4ad853d24f2a5293854c83d5f76b85", + "tests/test_kda_sm100_csrc_golden.py": "e0b97255a6b76430cd432bbe2a97dcae225885d2a5bf6f464ae9691a569e7e29", + "tests/test_kda_sm100_full_port_cutedsl.py": "b7ff7a5b1b05c48be0861bf0cfac36dc0c53383fd514f3676a66ab74ef1f68ac", + "tests/test_kda_sm100_intra_fused_cutedsl.py": "7bdccf9e9213ce59b85e011517a53a789634fa3ecdf614c8624f4d15c591fcc6", + "tests/test_kda_sm100_intracard_cp.py": "ac55728c850d864514956fab7b9f6a030ed36c10a24a6645ceb418a573ef83e0", + "tests/test_kda_sm100_output.py": "986b89eb1654907df89d683e06552dd377766dbccebfeeb0a1e9efe845095ab0", + "tests/test_kda_sm100_recompute_wu_cutedsl.py": "a436cc24f8bb00693f4efb93d631bfe43fc598e127012f5c9ec17454df976397", + "tests/test_kda_sm100_recurrence.py": "f129df6556f6ee3717a62d7a384263726f2ae99e7993fd0883a3fd47c3f781e6", + "tests/test_kda_sm90_intracard_cp.py": "a29ba5ed1c5e7c20a44fbfbb4863dc6b5d084a4194370d9c1616d318e797e6b6", + "tests/test_kda_sm90_prefill_vs_fla.py": "c592157326564935b2db6bf270f8b6f3c464170a2bc0a24b7624bb881665d255", + "tests/test_kimi_cp_distributed.py": "24ec2a82ad164c38365cba0840935ec0262659bc2a9b6c931dbe48ef5649684c", + "tests/test_kimi_cp_helpers.py": "c2374b4267b86be6003c1687b1b787c153fac5214a7485ba11850d78a92f116e", + "tests/test_kimi_cp_plan.py": "facddb9d2471b7c5c172c4eeea89febebb34e2f9192116c361b3a3f7b8e4181e", + "tests/test_la_decode_mtp.py": "0af705c288e6c176a610c0da6344f414bfe79f8752571e2816683e69ca840319", + "tests/test_la_kvbuffer.py": "33aec4bd91f031b8fc55f7917e3fee916a9c987728e617ee18c36766999fdb8d", + "tests/test_large_cp_emulation.py": "37e5e0eebf108e9673442b15fa6f2893d010ac65242a0ac76fdd65cc475fd174", + "tests/test_large_cp_query_layout.py": "ba5504cc75c7bab381ba9918dd6179abcd38d9030df5558ca4f602322a3ca6d6", + "tests/test_lightning_attn_prefill_dispatch.py": "002ca5602944b82083fb5a8c0cce45ab7f7dc4661efde9a6beca6d232fb708cc", + "tests/test_lightning_attn_prefill_sm90.py": "7817d21b004690f54fc75ae45fb4bf4aaed2a0d85b445e27608a456cbbf4f7d7", + "tests/test_lightning_decode.py": "58d8d89dd162d2d9ed16256fddef92cddfc0b37954d79acb58bae321a45bd82b", + "tests/test_lightning_decode_pool.py": "faf013566520c5869eafbf576437c08e3e85921a075c6ff8b9698b5b37dcea04", + "tests/test_lightning_sm100_prefill.py": "93bd5e250dc9b5db4cf7bbdc6bae07c87c20157d5e6ac388afcc5e3be6c829c3", + "tests/test_ptx_umma_masked.py": "f88ca7d76598396d147a58e23d535bcc80bcdb3aa52ca15b5f1a8d1e3a1c9082", + "tests/test_ptx_umma_ws.py": "6043d672a5456cd039d4bdec5425dd434a15c21f13ab8e0003ab56d44291b568", + "tests/test_standalone_kernel_surface.py": "7d06007483aaa21bba94e9c49353cbc3ddf868a75bfa3a5b7b28fd51fd78b60e", + "tests/test_test_suite_profile.py": "cbabf13caa933a1e4259796a947874280774b28f0883738c63d0636c5fa1ecab", + "tests/test_thd_1m_cases.py": "8f49b87c0f75fdedb359ba382752b124e3c9c4f6c6aac14c69c9678b6d3f5a08", + "tests/test_thd_cost_fit.py": "80811592b6e22b4c0ace12b68b081345c23a5288fcadef4a6482b9ffa16a84b1", + "tests/test_thd_planner.py": "b78eaad978f14b420b29ac4805eaaba968771030e45fd949cc1f23b81f36b500", + "tests/test_thd_redistribute.py": "6967330ec4b8e86273de0ff2069f3e6d648832404745d528b0b9f990f14bdceb" + }, + "platforms": { + "h20_cp16": { + "full_csa_rows": 21, + "fp32_check_rows": 12, + "one_m_cases": { + "1m-count-mixed-1000-1long": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-mixed-1000-1long-contiguous.json", + "total_ms": 1734.75927734375, + "min_ms": 1730.8006591796875, + "max_ms": 1741.20263671875, + "allocated_gib": 40.67960977554321, + "reserved_gib": 45.763671875, + "prepare_ms": 18.482470943126827, + "layout_prepare_ms": 17.99330097856, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 1.9999728836119175, + "max_spans": 135 + }, + "planner": { + "source": "layout-1m-count-mixed-1000-1long-planner.json", + "total_ms": 1079.9918212890625, + "min_ms": 1078.9337158203125, + "max_ms": 1080.5509033203125, + "allocated_gib": 42.25836515426636, + "reserved_gib": 46.767578125, + "prepare_ms": 67.1296549262479, + "layout_prepare_ms": 66.67199596995488, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0013937532701230287, + "dq": 0.0007082964070750158, + "dkv": 0.0006137254351491774, + "d_index_q": 0.000786539812462648, + "d_weights": 0.0008088782633845682, + "d_pool_kv": 0.0009303882391162907, + "d_pool_score": 0.0009171219579661385, + "d_ape": 4.0186864686063254e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999987006187439, + "max_spans": 137 + }, + "zigzag": { + "source": "layout-1m-count-mixed-1000-1long-zigzag.json", + "total_ms": 1110.866943359375, + "min_ms": 1110.0362548828125, + "max_ms": 1112.42529296875, + "allocated_gib": 41.40707540512085, + "reserved_gib": 46.76953125, + "prepare_ms": 71.14337000530213, + "layout_prepare_ms": 70.65627403790131, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014044624923471708, + "dq": 0.000718481723676461, + "dkv": 0.0006745502854727569, + "d_index_q": 0.0008122306113631543, + "d_weights": 0.0008234815695709059, + "d_pool_kv": 0.0009008851498893785, + "d_pool_score": 0.000896099670380151, + "d_ape": 4.149431394764015e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 2.999989103525877, + "max_spans": 1200 + } + }, + "speedup_vs_contiguous": 1.6062707542295707, + "gap_to_best_baseline_pct": -2.779371755985738, + "prepare_plus_iteration_ms": { + "contiguous": 1753.2417482868768, + "planner": 1147.1214762153104, + "zigzag": 1182.0103133646771 + }, + "prepare_inclusive_gap_pct": -2.951652515624259, + "documents": 1000 + }, + "1m-count-tiny-mix-1000": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-tiny-mix-1000-contiguous.json", + "total_ms": 806.0984497070312, + "min_ms": 805.9192504882812, + "max_ms": 806.1605834960938, + "allocated_gib": 39.01279067993164, + "reserved_gib": 46.146484375, + "prepare_ms": 27.179943048395216, + "layout_prepare_ms": 26.643655030056834, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 143 + }, + "planner": { + "source": "layout-1m-count-tiny-mix-1000-planner.json", + "total_ms": 805.8177490234375, + "min_ms": 805.7346801757812, + "max_ms": 806.04150390625, + "allocated_gib": 39.0129189491272, + "reserved_gib": 44.91015625, + "prepare_ms": 13.915454037487507, + "layout_prepare_ms": 13.422106392681599, + "policy": "contiguous", + "relative_rms": { + "output": 0.0009711554023225683, + "dq": 0.0006152729484808843, + "dkv": 0.0003926991846874368, + "d_index_q": 0.0003016586499429551, + "d_weights": 0.0003208988010868512, + "d_pool_kv": 0.0003709002742156663, + "d_pool_score": 0.000379869535872596, + "d_ape": 3.3729356601707806e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 143 + }, + "zigzag": { + "source": "layout-1m-count-tiny-mix-1000-zigzag.json", + "total_ms": 821.2994995117188, + "min_ms": 821.0466918945312, + "max_ms": 822.2781372070312, + "allocated_gib": 39.00111246109009, + "reserved_gib": 46.18359375, + "prepare_ms": 76.13682397641242, + "layout_prepare_ms": 75.66057000076398, + "policy": "zigzag", + "relative_rms": { + "output": 0.0010180829441058961, + "dq": 0.0006456383045368285, + "dkv": 0.0004906538276004508, + "d_index_q": 0.00034542113019621524, + "d_weights": 0.00033085709677908745, + "d_pool_kv": 0.00037554118525190026, + "d_pool_score": 0.0003727287946029054, + "d_ape": 3.2555742636422667e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.49914050847291946, + "max_spans": 1200 + } + }, + "speedup_vs_contiguous": 1.000348342641911, + "gap_to_best_baseline_pct": -0.034822134156908824, + "prepare_plus_iteration_ms": { + "contiguous": 833.2783927554265, + "planner": 819.733203060925, + "zigzag": 897.4363234881312 + }, + "prepare_inclusive_gap_pct": -1.6255299324048456, + "documents": 1000 + }, + "1m-single": { + "world_size": 16, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-single-contiguous.json", + "total_ms": 2905.626953125, + "min_ms": 2882.656982421875, + "max_ms": 2919.6826171875, + "allocated_gib": 42.58390665054321, + "reserved_gib": 47.701171875, + "prepare_ms": 6.134580064099282, + "layout_prepare_ms": 5.915455054491758, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.999951247125864, + "max_spans": 1 + }, + "planner": { + "source": "layout-1m-single-planner.json", + "total_ms": 1930.8380126953125, + "min_ms": 1929.918701171875, + "max_ms": 1935.875732421875, + "allocated_gib": 43.17422676086426, + "reserved_gib": 48.498046875, + "prepare_ms": 64.26966283470392, + "layout_prepare_ms": 64.06422145664692, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.0014571483324829387, + "dq": 0.0007421278526508277, + "dkv": 0.0007551298904303639, + "d_index_q": 0.0007261477208478219, + "d_weights": 0.0007914909302235379, + "d_pool_kv": 0.0009610520429536651, + "d_pool_score": 0.0009602856176819985, + "d_ape": 1.3991300052915012e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 5.820766091346741e-11, + "workspace_gib": 3.999988541007042, + "max_spans": 2 + }, + "zigzag": { + "source": "layout-1m-single-zigzag.json", + "total_ms": 1944.2569580078125, + "min_ms": 1939.1024169921875, + "max_ms": 1955.9830322265625, + "allocated_gib": 43.17422676086426, + "reserved_gib": 48.5, + "prepare_ms": 7.630876265466213, + "layout_prepare_ms": 7.41471815854311, + "policy": "zigzag", + "relative_rms": { + "output": 0.0014060094451772223, + "dq": 0.0007140955156081367, + "dkv": 0.0006730968914330185, + "d_index_q": 0.000726804709767014, + "d_weights": 0.0007835076200786753, + "d_pool_kv": 0.00101384945694068, + "d_pool_score": 0.0010062370245195436, + "d_ape": 1.4595408260036373e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 8.731149137020111e-11, + "workspace_gib": 3.999988541007042, + "max_spans": 2 + } + }, + "speedup_vs_contiguous": 1.5048527810310464, + "gap_to_best_baseline_pct": -0.6901837361173557, + "prepare_plus_iteration_ms": { + "contiguous": 2911.7615331890993, + "planner": 1995.1076755300164, + "zigzag": 1951.8878342732787 + }, + "prepare_inclusive_gap_pct": 2.214258447531603, + "documents": 1 + } + }, + "max_relative_rms": { + "output": 0.0022360824205777032, + "dq": 0.0011669460245709953, + "dkv": 0.002013003506138166, + "d_index_q": 0.0026207664468348288, + "d_weights": 0.0019498450002418547, + "d_pool_kv": 0.0035533125482489977, + "d_pool_score": 0.0029468591104983063, + "d_ape": 4.616907922474155e-05, + "kda_output": 0, + "kda_dq": 0, + "kda_dk": 0, + "kda_dv": 0, + "kda_dg": 0, + "kda_dbeta": 0 + }, + "loss_abs_diff": 5.820766091346741e-10, + "independent_kda": [ + { + "case": "uniform-1024", + "label": "baseline", + "passed": true, + "total_ms": 30.399328231811523 + }, + { + "case": "uniform-1024", + "label": "candidate", + "passed": true, + "total_ms": 26.572128295898438 + }, + { + "case": "uniform-4096", + "label": "baseline", + "passed": true, + "total_ms": 35.26870346069336 + }, + { + "case": "uniform-4096", + "label": "candidate", + "passed": true, + "total_ms": 25.992992401123047 + }, + { + "case": "uniform-16384", + "label": "baseline", + "passed": true, + "total_ms": 58.847904205322266 + }, + { + "case": "uniform-16384", + "label": "candidate", + "passed": true, + "total_ms": 29.499359130859375 + }, + { + "case": "near-uniform", + "label": "baseline", + "passed": true, + "total_ms": 58.68083190917969 + }, + { + "case": "near-uniform", + "label": "candidate", + "passed": true, + "total_ms": 29.76799964904785 + }, + { + "case": "weak-decay-uniform", + "label": "baseline", + "passed": true, + "total_ms": 58.779518127441406 + }, + { + "case": "weak-decay-uniform", + "label": "candidate", + "passed": true, + "total_ms": 29.494943618774414 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "baseline", + "passed": true, + "total_ms": 111.36412811279297 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "candidate", + "passed": true, + "total_ms": 111.15647888183594 + } + ], + "raw_sha256": { + "layout-check-single-contiguous.json": "df4f9940c230c68ca647c70e636a8259f30b3630fa56f4ff8505e59eeb7fb713", + "layout-check-uniform-zigzag.json": "12951e3aa159381afd92fe6a34f398118ced9484f0d70b663dba6c8da869c165", + "layout-check-dense-zigzag.json": "2753074b3e030efce01c3cf5690f0ac79107321a5efcf26346c5b92eabc0242e", + "layout-check-mixed-planner.json": "b2b48f44dabbe1078a343704c104182875b4eabb67a46ac256baeb0ec13777a4", + "layout-1m-single-planner.json": "e547b82aa37d1dce0353e67acd524f02f238d7e7e20290426541875e0f40b540", + "layout-1m-count-mixed-1000-1long-zigzag.json": "811050d5906a45c3bd89e7a54a498116d97f7fa16faaa760079cab881e140c65", + "layout-check-mixed-zigzag.json": "a292ec0bee1f427930f1a8195491e5c333897f72f57cc98b35211b8a27af0fdd", + "layout-1m-count-mixed-1000-1long-contiguous.json": "583a4487a52242e40ba34476924a962d7ec56289821da15fdc0af2cba0380ce1", + "layout-check-single-zigzag.json": "7c72292b8584f667c3446d0725953f23b967b13903d5b5dafa21aa8f1dee4a4e", + "layout-1m-count-tiny-mix-1000-contiguous.json": "7c973181c8902c9891993cf2b8237872ce0c7f880adc7a724ed0ad1ed840d7b7", + "layout-check-single-planner.json": "8fabf01ebe04935b40a5a9455abf5f982f5565fdcb5c4b34c4cde9501ebb9f3a", + "layout-check-uniform-planner.json": "74fc7347994c5a6c98065a84c0fee011c5894727cf0ac9fc916f15847410068f", + "layout-check-mixed-contiguous.json": "6aa870d6ce4d270dd740fa5dd6f2fdfbe7dd0b0320d20e1f7a9c8be07f5a4a1a", + "layout-1m-single-contiguous.json": "21b0740f9da731c6c51354ef8d5eaf0e6716cf79293d073156444bba35c073da", + "layout-check-dense-planner.json": "8b9df01222bf46328824a7856df56889f1c281e8da5aceca66b0740450edd097", + "layout-1m-single-zigzag.json": "5e2adbed587139640c0236552a46902e790c8b4fea51fe1098e0575b9e44b6aa", + "layout-1m-count-tiny-mix-1000-planner.json": "4fc9ef2dfbdf4d73aade67b6227b2a2a37774556a1befc4ce78ee98a6a6a0ed3", + "layout-1m-count-tiny-mix-1000-zigzag.json": "33231fee890bc3d94bfd53dfd4b1f5068c10e19faebfed835382f503f53353d7", + "layout-check-uniform-contiguous.json": "6f21a533c62df42232b337f1fd4424386813480793c57c7bc7de6af109157957", + "layout-check-dense-contiguous.json": "910a2defc4a846c6bd43dfa3c6aef7ea6d0d6ef51745fdeea99b4be572cec5f4", + "layout-1m-count-mixed-1000-1long-planner.json": "3c64c6cc910cc4111ef9f4533bf235fcf5aac8a18b32e4cdd5d8f9f100c18cec" + }, + "independent_raw_sha256": "3829d509f0bf33a97df9065301cd294d52e2dbf0323df0124586080e5b7818d7" + }, + "b300_cp8": { + "full_csa_rows": 21, + "fp32_check_rows": 12, + "one_m_cases": { + "1m-count-mixed-1000-1long": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-mixed-1000-1long-contiguous.json", + "total_ms": 563.7484130859375, + "min_ms": 561.4152221679688, + "max_ms": 566.3046264648438, + "allocated_gib": 69.97186422348022, + "reserved_gib": 72.40625, + "prepare_ms": 11.447699856944382, + "layout_prepare_ms": 10.858646943233907, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 3.9999988861382008, + "max_spans": 262 + }, + "planner": { + "source": "layout-1m-count-mixed-1000-1long-planner.json", + "total_ms": 465.6362609863281, + "min_ms": 464.8326110839844, + "max_ms": 466.11572265625, + "allocated_gib": 72.94540643692017, + "reserved_gib": 85.4140625, + "prepare_ms": 37.901079980656505, + "layout_prepare_ms": 37.406648974865675, + "policy": "selection-paired-local", + "relative_rms": { + "output": 0.000981929418930544, + "dq": 0.0003586859933571614, + "dkv": 0.0005251233580841019, + "d_index_q": 0.000768516565343547, + "d_weights": 0.0005601425977036474, + "d_pool_kv": 0.00075289034085658, + "d_pool_score": 0.0007494019256045927, + "d_ape": 3.2458577600061467e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 5.999798700213432, + "max_spans": 264 + }, + "zigzag": { + "source": "layout-1m-count-mixed-1000-1long-zigzag.json", + "total_ms": 480.0238342285156, + "min_ms": 479.3189392089844, + "max_ms": 482.1390686035156, + "allocated_gib": 72.85924434661865, + "reserved_gib": 74.9140625, + "prepare_ms": 68.89537198003381, + "layout_prepare_ms": 68.42616200447083, + "policy": "zigzag", + "relative_rms": { + "output": 0.0009359406560165561, + "dq": 0.0003445985393205624, + "dkv": 0.0005055089868584756, + "d_index_q": 0.0007650331133222328, + "d_weights": 0.0005585781202017105, + "d_pool_kv": 0.0007500151256218873, + "d_pool_score": 0.0007479169339269727, + "d_ape": 3.378714550586395e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 5.999901540577412, + "max_spans": 2000 + } + }, + "speedup_vs_contiguous": 1.2107055663830488, + "gap_to_best_baseline_pct": -2.9972622641354696, + "prepare_plus_iteration_ms": { + "contiguous": 575.1961129428819, + "planner": 503.53734096698463, + "zigzag": 548.9192062085494 + }, + "prepare_inclusive_gap_pct": -8.267494510717299, + "documents": 1000 + }, + "1m-count-tiny-mix-1000": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-count-tiny-mix-1000-contiguous.json", + "total_ms": 276.4977722167969, + "min_ms": 276.32672119140625, + "max_ms": 277.500244140625, + "allocated_gib": 67.35119819641113, + "reserved_gib": 69.52734375, + "prepare_ms": 14.639637200161815, + "layout_prepare_ms": 14.148701098747551, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 0.5, + "max_spans": 174 + }, + "planner": { + "source": "layout-1m-count-tiny-mix-1000-planner.json", + "total_ms": 276.5791931152344, + "min_ms": 275.75604248046875, + "max_ms": 278.41815185546875, + "allocated_gib": 67.35119819641113, + "reserved_gib": 69.52734375, + "prepare_ms": 13.63393000792712, + "layout_prepare_ms": 13.120088959112763, + "policy": "contiguous", + "relative_rms": { + "output": 0.000816856606217396, + "dq": 0.0003701320592240906, + "dkv": 0.00028171521311393853, + "d_index_q": 0.0001906295405525901, + "d_weights": 0.00016998027710324372, + "d_pool_kv": 0.00024510550786153437, + "d_pool_score": 0.00023626867066475846, + "d_ape": 2.881648668361276e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.1641532182693481e-10, + "workspace_gib": 0.5, + "max_spans": 174 + }, + "zigzag": { + "source": "layout-1m-count-tiny-mix-1000-zigzag.json", + "total_ms": 288.1846923828125, + "min_ms": 286.8497314453125, + "max_ms": 289.92547607421875, + "allocated_gib": 67.31179094314575, + "reserved_gib": 78.9453125, + "prepare_ms": 76.26319001428783, + "layout_prepare_ms": 75.76619903557003, + "policy": "zigzag", + "relative_rms": { + "output": 0.0008172618750855098, + "dq": 0.0003660674276407079, + "dkv": 0.0002623909333939321, + "d_index_q": 0.00018360667662851773, + "d_weights": 0.00016406571075909486, + "d_pool_kv": 0.0002447536048006018, + "d_pool_score": 0.00023747323117413518, + "d_ape": 2.8442642578901785e-06, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.3283064365386963e-10, + "workspace_gib": 0.49526814743876457, + "max_spans": 2000 + } + }, + "speedup_vs_contiguous": 0.99970561452031, + "gap_to_best_baseline_pct": 0.02944721680204765, + "prepare_plus_iteration_ms": { + "contiguous": 291.1374094169587, + "planner": 290.2131231231615, + "zigzag": 364.44788239710033 + }, + "prepare_inclusive_gap_pct": -0.31747424545962577, + "documents": 1000 + }, + "1m-single": { + "world_size": 8, + "contract": { + "heads": 64, + "top_k": 512, + "with_kda": true, + "kda_batch_invariant": true, + "kda_gate_scale": 1.0, + "candidate_mib": 0, + "redistribution_scope": "selection", + "pool_backward_order": "per-loss", + "repeats": 5, + "warmups": 3 + }, + "layouts": { + "contiguous": { + "source": "layout-1m-single-contiguous.json", + "total_ms": 878.70654296875, + "min_ms": 874.7012329101562, + "max_ms": 881.0527954101562, + "allocated_gib": 73.0117564201355, + "reserved_gib": 75.380859375, + "prepare_ms": 7.158453925512731, + "layout_prepare_ms": 6.8812749814242125, + "policy": "contiguous", + "relative_rms": {}, + "loss_abs_diff": 0.0, + "workspace_gib": 7.9999911189079285, + "max_spans": 1 + }, + "planner": { + "source": "layout-1m-single-planner.json", + "total_ms": 664.1754760742188, + "min_ms": 663.5114135742188, + "max_ms": 668.2277221679688, + "allocated_gib": 73.01857900619507, + "reserved_gib": 75.88671875, + "prepare_ms": 15.7493349397555, + "layout_prepare_ms": 15.474541927687824, + "policy": "selection-local-2x", + "relative_rms": { + "output": 0.001168400329939734, + "dq": 0.00039265724673352774, + "dkv": 0.0005302362191601861, + "d_index_q": 0.0007076503704418045, + "d_weights": 0.0005489574816612602, + "d_pool_kv": 0.0007111586908343255, + "d_pool_score": 0.0006894097289940324, + "d_ape": 1.3500283722646205e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 2.9103830456733704e-11, + "workspace_gib": 7.9999911189079285, + "max_spans": 2 + }, + "zigzag": { + "source": "layout-1m-single-zigzag.json", + "total_ms": 668.0455932617188, + "min_ms": 664.5176391601562, + "max_ms": 668.7427978515625, + "allocated_gib": 73.01857900619507, + "reserved_gib": 75.884765625, + "prepare_ms": 12.595297070220113, + "layout_prepare_ms": 12.315480038523674, + "policy": "zigzag", + "relative_rms": { + "output": 0.0011720185988330905, + "dq": 0.00039552385403338535, + "dkv": 0.0005946938023163722, + "d_index_q": 0.0006038215373079134, + "d_weights": 0.0005245172798592142, + "d_pool_kv": 0.000809623655331361, + "d_pool_score": 0.0007646019067327281, + "d_ape": 1.3008000488275949e-05, + "kda_output": 0.0, + "kda_dq": 0.0, + "kda_dk": 0.0, + "kda_dv": 0.0, + "kda_dg": 0.0, + "kda_dbeta": 0.0 + }, + "loss_abs_diff": 1.7462298274040222e-10, + "workspace_gib": 7.9999911189079285, + "max_spans": 2 + } + }, + "speedup_vs_contiguous": 1.3230035956198996, + "gap_to_best_baseline_pct": -0.5793193198991409, + "prepare_plus_iteration_ms": { + "contiguous": 885.8649968942627, + "planner": 679.9248110139742, + "zigzag": 680.6408903319389 + }, + "prepare_inclusive_gap_pct": -0.10520662630413291, + "documents": 1 + } + }, + "max_relative_rms": { + "output": 0.0024348185275864243, + "dq": 0.0007232451638847554, + "dkv": 0.0005946938023163722, + "d_index_q": 0.000768516565343547, + "d_weights": 0.0005601425977036474, + "d_pool_kv": 0.000809623655331361, + "d_pool_score": 0.0007646019067327281, + "d_ape": 3.0188638995342922e-05, + "kda_output": 0, + "kda_dq": 0, + "kda_dk": 0, + "kda_dv": 0, + "kda_dg": 0, + "kda_dbeta": 0 + }, + "loss_abs_diff": 1.5133991837501526e-09, + "independent_kda": [ + { + "case": "uniform-1024", + "label": "baseline", + "passed": true, + "total_ms": 13.65231990814209 + }, + { + "case": "uniform-1024", + "label": "candidate", + "passed": true, + "total_ms": 12.879487991333008 + }, + { + "case": "uniform-4096", + "label": "baseline", + "passed": true, + "total_ms": 14.166272163391113 + }, + { + "case": "uniform-4096", + "label": "candidate", + "passed": true, + "total_ms": 12.644831657409668 + }, + { + "case": "uniform-16384", + "label": "baseline", + "passed": true, + "total_ms": 26.2923526763916 + }, + { + "case": "uniform-16384", + "label": "candidate", + "passed": true, + "total_ms": 23.619583129882812 + }, + { + "case": "near-uniform", + "label": "baseline", + "passed": true, + "total_ms": 25.801279067993164 + }, + { + "case": "near-uniform", + "label": "candidate", + "passed": true, + "total_ms": 23.4333438873291 + }, + { + "case": "weak-decay-uniform", + "label": "baseline", + "passed": true, + "total_ms": 26.233535766601562 + }, + { + "case": "weak-decay-uniform", + "label": "candidate", + "passed": true, + "total_ms": 23.613311767578125 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "baseline", + "passed": true, + "total_ms": 29.493568420410156 + }, + { + "case": "isolated-ranks-with-crossing-peer", + "label": "candidate", + "passed": true, + "total_ms": 29.489408493041992 + } + ], + "raw_sha256": { + "layout-check-single-contiguous.json": "1fe908c09dff05415909118c66af592465e8f91343458d52a983c1c7b13ce871", + "layout-check-uniform-zigzag.json": "3285988d7171e294fe04d1dcb5478bd7075ca1ecd3767243b171f11f63ef8eb7", + "layout-check-dense-zigzag.json": "d23fb5db1ec3e09cbd0188e0b551d71c5fb4c00c7c0fbd745bff637740f4bc8a", + "layout-check-mixed-planner.json": "d846a3cbaff139fb00a4925639ad41892c8309870a9db51528a4de303e5eab5a", + "layout-1m-single-planner.json": "1dd73977e2e22a6c2a0e43d9834e3cf1054faa5ee24fe0a0c6e30d89093c2a37", + "layout-1m-count-mixed-1000-1long-zigzag.json": "c49c0de6b7a58a18a0829a98f80b560c80a4d61dec734e70eb208d51caee5b8b", + "layout-check-mixed-zigzag.json": "571f0e879504feb4dba87b75352e2e147b75054ff654ec94829d844bce2bfca2", + "layout-1m-count-mixed-1000-1long-contiguous.json": "1746f1715f6c65066db2ba233c76d34eb5088e1c26c5fb129a83c6791ab6f893", + "layout-check-single-zigzag.json": "2448feea4f6eb7c30e71d9901d301149ca38a716a578cd78842da88e446210bb", + "layout-1m-count-tiny-mix-1000-contiguous.json": "676fe0e3c57bf5f367a318771a62766a3c26711e34075d652bc1c26e4a2320ef", + "layout-check-single-planner.json": "c405c4b8908c997d16a6663e8b01e7d07c4307ec52d0845fe529f892e3304469", + "layout-check-uniform-planner.json": "ca3388e5689d4dbeacc218952621ba9e12a128664f7e56853ae3e00f94befdd4", + "layout-check-mixed-contiguous.json": "86fda36489cd9e03c72137363ee8fe90eb2cca8a823c9031656ccaca5bff74ab", + "layout-1m-single-contiguous.json": "f72c50eb44db7e50c7e09635063c63c9b97363764b0c7cc76b9afc66d6297916", + "layout-check-dense-planner.json": "4c159ab030611c0c9a29b81a10d46e1cde31cd14a8ff324b2a6836f7519c8fa5", + "layout-1m-single-zigzag.json": "0e50108c69a225344769d79dacaba4e9f70db7e1c8d2bd0c05a8e660bffc8384", + "layout-1m-count-tiny-mix-1000-planner.json": "83b4f0d29314fbaeb5884ea3e8d564c5ecb844ccf99c156c9157d29c284842cb", + "layout-1m-count-tiny-mix-1000-zigzag.json": "2a258d9126aafcb5c4bd0a6982e60c3f5c2b7834f7643b459c35013af991fe88", + "layout-check-uniform-contiguous.json": "96a42331a4114a8a30f2cc8294f1987dfab63b4b477281081b0c013cc63ce4c4", + "layout-check-dense-contiguous.json": "1074540bc05ba6fde4c79146e380af1ef0375be014bae0bd094fd9843bfba259", + "layout-1m-count-mixed-1000-1long-planner.json": "09f3b10bad5248e5df5039485a16f5189aea02aa1ca3d18c2a621ff587129f2a" + }, + "independent_raw_sha256": "67bebb11c6aac209a0cdae42d493378fada7952f4df806fdd3007a735cf9cf10" + } + } +} diff --git a/tests/test_csa_cp_selection.py b/tests/test_csa_cp_selection.py new file mode 100644 index 00000000..17776068 --- /dev/null +++ b/tests/test_csa_cp_selection.py @@ -0,0 +1,263 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Host planning and execution-contract checks without a native dependency.""" + +from dataclasses import replace +from itertools import accumulate +from types import SimpleNamespace + +import pytest +import torch + +from benchmarks.cp_layouts import query_layouts +from cula.distributed import THDPlannerConfig, THDSelectionCostModel, plan_thd_selection +from cula.hybrid import CsaCPSelectionPlan, CsaCPSelectionWorkspace, csa_cp, cudnn_csa_select_cp +from cula.hybrid.dsa import DsaSelectionPlan + + +def costs(world): + return THDSelectionCostModel( + candidate_ns=0.1, + query_ns=8, + span_us=5, + bytes_per_query=2 * 4 * 2 + 2 * 2 + 8 * 8, + intra_node_gbps=40, + inter_node_gbps=20, + rank_node_ids=(0,) * world, + ) + + +@pytest.mark.parametrize("tokens", [128 << 10, 256 << 10, 1 << 20]) +@pytest.mark.parametrize("world", [8, 16]) +@pytest.mark.parametrize("kind", ["single", "uniform", "near-uniform", "mixed", "long-tail"]) +@pytest.mark.parametrize("policy", ["contiguous", "zigzag", "planner"]) +def test_plans_match_existing_execution_metadata(tokens, world, kind, policy): + if kind == "single": + lengths = [tokens] + elif kind == "uniform": + lengths = [tokens // 1024] * 1024 + elif kind == "near-uniform": + lengths = [tokens // 512 - 4, tokens // 512 + 4] * 256 + elif kind == "mixed": + lengths = [64] * 1000 + [(tokens - 64000) // 2] * 2 + else: + lengths = [64] * 500 + [256] * 12 + [tokens - 35072] + cu = (0, *accumulate(lengths)) + model = costs(world) + layouts, selected = query_layouts(cu, world, policy, cost_model=model, planner_strategy="selection") + # First, interior and last source ranks cover both complete and split docs. + for rank in (0, world // 2, world - 1): + p = CsaCPSelectionPlan.from_cu_seqlens(cu, rank=rank, world_size=world, top_k=8, policy=policy, cost_model=model) + assert p.ranks == layouts and p.policy == selected + local = layouts[rank] + assert p.exchange.source_rows == tokens // world + assert p.exchange.target_rows == local.local_tokens + ranges = tuple((s.sequence_start // 4, s.sequence_end // 4) for s in local.spans) + assert p.key_ranges == ranges + if not local.local_tokens: + assert p.selection is None + continue + prefix = (0, *accumulate(z - a for a, z in ranges)) + expected = DsaSelectionPlan.from_cu_seqlens( + local.local_cu_seqlens, + prefix, + top_k=8, + q_causal_offsets=local.causal_offsets, + ) + assert p.selection == expected + + +@pytest.mark.parametrize( + "options", + [ + {"world_size": 0}, + {"rank": True}, + {"rank": 8}, + {"top_k": 0}, + {"top_k": 2049}, + {"top_k": True}, + {"policy": "unknown"}, + {"max_tokens_per_rank": 15}, + {"candidate_budget_bytes": 0}, + {"candidate_budget_bytes": True}, + {"cost_model": costs(4)}, + ], +) +def test_invalid_plan_options(options): + kwargs = dict(rank=0, world_size=8, top_k=8, policy="contiguous") + kwargs.update(options) + with pytest.raises((ValueError, TypeError)): + CsaCPSelectionPlan.from_cu_seqlens((0, 128), **kwargs) + + +@pytest.mark.parametrize("cu", [(0, 5, 128), (0, 16, 12), (1, 128), (0, 2**31), (0, 4)]) +def test_invalid_global_metadata_fails_on_every_rank(cu): + for rank in range(8): + with pytest.raises(ValueError): + CsaCPSelectionPlan.from_cu_seqlens(cu, rank=rank, world_size=8, top_k=8, policy="contiguous") + + +def test_planner_needs_explicit_costs_and_never_reads_device_metadata(): + with pytest.raises(ValueError, match="cost_model"): + CsaCPSelectionPlan.from_cu_seqlens((0, 128), rank=0, world_size=8, top_k=8) + with pytest.raises(ValueError, match="CPU tensor"): + CsaCPSelectionPlan.from_cu_seqlens(torch.empty(2, device="meta"), rank=0, world_size=8, top_k=8, policy="contiguous") + + +def test_empty_zigzag_destination_and_block_balanced_sources(): + plans = [CsaCPSelectionPlan.from_cu_seqlens((0, 4, 36), rank=r, world_size=8, top_k=8, policy="zigzag") for r in range(8)] + assert all(p.exchange.source_rows % 4 == 0 for p in plans) + assert sum(p.exchange.target_rows for p in plans) == 36 + # Unequal source counts are intentional: nine blocks across eight ranks. + assert {p.exchange.source_rows for p in plans} == {4, 8} + tiny = [ + CsaCPSelectionPlan.from_cu_seqlens( + tuple(range(0, 36, 4)), rank=r, world_size=8, top_k=8, policy="zigzag", max_tokens_per_rank=32 + ) + for r in range(8) + ] + assert any(p.selection is None for p in tiny) + + +@pytest.mark.parametrize("identity", [True, False]) +def test_source_order_result_lifetime_and_validation(monkeypatch, identity): + plan = CsaCPSelectionPlan.from_cu_seqlens((0, 16), rank=0, world_size=1, top_k=8, policy="contiguous", cost_model=costs(1)) + plan = replace(plan, exchange=replace(plan.exchange, identity=identity)) + calls = [] + + def route(tensor): + calls.append("route") + return tensor if identity else tensor.flip(0) + + def restore(tensor): + calls.append("restore") + return tensor if identity else tensor.flip(0) + + exchange = SimpleNamespace(device=torch.device("cpu"), group=None, distribute=route, restore=restore) + native = SimpleNamespace(block_ids=torch.empty(16, 8, dtype=torch.int32), scores=torch.empty(16, 8)) + workspace = CsaCPSelectionWorkspace(plan, exchange, native, slice(0, 4), None) + monkeypatch.setattr(csa_cp.dist, "get_rank", lambda group: 0) + monkeypatch.setattr(csa_cp.dist, "get_world_size", lambda group: 1) + seen = [] + + def select(q, k, w, **kwargs): + seen.append(kwargs) + native.block_ids.copy_(q[:, 0, 0, None].expand(-1, 8).int()) + native.scores.copy_(q[:, 0, 0, None].expand(-1, 8).float()) + return native + + monkeypatch.setattr(csa_cp, "cudnn_dsa_select", select) + qi = torch.arange(16, dtype=torch.bfloat16)[:, None, None].expand(-1, 2, 4).clone().requires_grad_() + keys, weights = torch.ones(4, 1, 4, dtype=torch.bfloat16), torch.ones(16, 2, dtype=torch.bfloat16) + result = cudnn_csa_select_cp(qi, keys, weights, workspace=workspace, sm_scale=0.5) + assert torch.equal(result.block_ids[:, 0], torch.arange(16)) + assert torch.equal(result.scores[:, 0], torch.arange(16).float()) + native.block_ids.fill_(-1) + native.scores.zero_() + assert torch.equal(result.block_ids[:, 0], torch.arange(16)) + assert torch.equal(result.scores[:, 0], torch.arange(16).float()) + assert not result.scores.requires_grad + assert calls == ["route", "route", "restore", "restore"] + assert seen[0]["deterministic"] is False and seen[0]["sm_scale"] == 0.5 + for bad_q, bad_k, bad_w in [ + (qi.float(), keys, weights), + (qi, keys.float(), weights), + (qi[:, :, ::2], keys, weights), + (qi, keys, weights[:, :1]), + ]: + with pytest.raises((TypeError, ValueError)): + cudnn_csa_select_cp(bad_q, bad_k, bad_w, workspace=workspace) + with pytest.raises(ValueError, match="bytes_per_query"): + cudnn_csa_select_cp(qi, keys, weights.float(), workspace=workspace) + with pytest.raises(ValueError, match="finite"): + cudnn_csa_select_cp(qi, keys, weights, workspace=workspace, sm_scale=float("nan")) + with pytest.raises(TypeError, match="bool"): + cudnn_csa_select_cp(qi, keys, weights, workspace=workspace, deterministic=1) + assert len(calls) == 4 # Invalid calls fail before any collective. + + +@pytest.mark.parametrize("world", [8, 16, 64]) +@pytest.mark.parametrize("tokens", [128 << 10, 256 << 10]) +def test_effective_chunk_floor_limits_high_cp_fragmentation(world, tokens): + from cula.distributed.thd_planner import _adaptive_items + + config = THDPlannerConfig(min_chunk_tokens=1024) + pieces = _adaptive_items((0, tokens), world, config, chunks_per_rank=6, sequences={0}) + assert all(item.tokens >= 1024 for item in pieces) + assert len(pieces) <= tokens // 1024 + plan = plan_thd_selection( + (0, tokens), world_size=world, max_tokens_per_rank=2 * tokens // world, cost_model=costs(world), config=config + ) + assert all(span.tokens >= 1024 for rank in plan.ranks for span in rank.spans) + assert sum(rank.local_tokens for rank in plan.ranks) == tokens + + +def test_chunk_floor_preserves_short_source_fragments_and_fallback(): + cu = (0, 4, 128 << 10) + plan = plan_thd_selection( + cu, world_size=64, max_tokens_per_rank=4096, cost_model=costs(64), config=THDPlannerConfig(min_chunk_tokens=4096) + ) + assert sum(rank.local_tokens for rank in plan.ranks) == cu[-1] + assert any(s.tokens == 4 for r in plan.ranks for s in r.spans) + # An infeasible grain preference must not remove the contiguous fallback. + assert plan.policy == "contiguous" + + +@pytest.mark.parametrize("floor", [-4, 1, True, 1.5]) +def test_invalid_chunk_floor(floor): + with pytest.raises((ValueError, TypeError)): + plan_thd_selection( + (0, 128), + world_size=8, + max_tokens_per_rank=32, + cost_model=costs(8), + config=THDPlannerConfig(min_chunk_tokens=floor), + ) + + +@pytest.mark.parametrize("world", [1, 8, 16, 64]) +@pytest.mark.parametrize("lengths", [[4] * 1024, [4, 128, 512, 65536], [131072], [64] * 998 + [33600, 33600]]) +def test_identity_metrics_match_general_cost_model(world, lengths): + from cula.distributed.thd_planner import _contiguous_items, _metrics, _source_selection_candidate + + cu = (0, *accumulate(lengths)) + config, model = THDPlannerConfig(), costs(world) + ranks, metrics = _source_selection_candidate(_contiguous_items(cu, world, 4), world, model) + expected = _metrics([s for r in ranks for s in r.spans], world_size=world, block_size=4, config=config) + score = model.estimate_ms(ranks, block_size=4) + assert metrics == replace(expected, score=score) + + +@pytest.mark.parametrize("ranges", [[], [(0, 0)], [(3, 9)], [(7, 11), (0, 3), (7, 11)], [(5, 5), (12, 16)]]) +def test_range_indices_preserve_duplicates_gaps_and_order(ranges): + from cula._metadata import _range_indices + + actual = _range_indices(ranges, torch.device("cpu")) + expected = torch.tensor([i for a, z in ranges for i in range(a, z)], dtype=torch.int64) + assert torch.equal(actual, expected) + + +@pytest.mark.parametrize("tokens", [128 << 10, 256 << 10, 1 << 20]) +@pytest.mark.parametrize("case,documents", [("uniform-64", 64), ("near-uniform-64", 64), ("mixed-22", 22)]) +def test_dozens_of_document_cases_cover_each_context(tokens, case, documents): + from benchmarks.bench_csa_cp_windows import cases + + lengths = cases(tokens)[case] + assert len(lengths) == documents and sum(lengths) == tokens + assert all(n > 0 and n % 4 == 0 for n in lengths) + cu = (0, *accumulate(lengths)) + for policy in ("contiguous", "zigzag", "planner"): + plan = CsaCPSelectionPlan.from_cu_seqlens( + cu, + rank=0, + world_size=8, + top_k=8, + policy=policy, + cost_model=costs(8), + config=THDPlannerConfig(min_chunk_tokens=1024), + ) + assert sum(r.local_tokens for r in plan.ranks) == tokens + intervals = sorted((s.token_start, s.token_end) for r in plan.ranks for s in r.spans) + assert intervals[0][0] == 0 and intervals[-1][1] == tokens + assert all(a[1] == b[0] for a, b in zip(intervals, intervals[1:])) diff --git a/tests/test_csa_matrix_report.py b/tests/test_csa_matrix_report.py new file mode 100644 index 00000000..a4b853d4 --- /dev/null +++ b/tests/test_csa_matrix_report.py @@ -0,0 +1,130 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest + +from benchmarks.summarize_csa_layout_matrix import summarize + + +def _matrix(tmp_path): + tensors = [ + "output", + "dq", + "dkv", + "d_index_q", + "d_weights", + "d_pool_kv", + "d_pool_score", + "d_ape", + "kda_output", + "kda_dq", + "kda_dk", + "kda_dv", + "kda_dg", + "kda_dbeta", + ] + for layout, latency, setup in [("contiguous", 10, 1), ("zigzag", 8, 3), ("planner", 7, 5)]: + rows = [ + dict( + rank=rank, + finite=True, + input_match=True, + redistribution_bitwise=True, + samples=[dict(total_ms=latency + rank, peak_allocated_gib=1, peak_reserved_gib=2)], + layout_prepare_ms=setup, + common_prepare_ms=2, + kda_prepare_ms=3, + total_prepare_ms=setup + 5, + selected_policy=layout, + selection_workspace_bytes=0, + selection_spans=1, + layout_vs_contiguous=( + None if layout == "contiguous" else {k: dict(finite=True, relative_rms=0, max_abs=0) for k in tensors} + ), + loss_abs_diff=None if layout == "contiguous" else 0, + ) + for rank in range(2) + ] + data = dict(world_size=2, config=dict(layout=layout, repeats=1, sequence_lengths=[1 << 20], with_kda=True), ranks=rows) + (tmp_path / f"layout-1m-fixture-{layout}.json").write_text(json.dumps(data)) + + +def test_report_distinguishes_steady_and_preparation_inclusive_cost(tmp_path): + _matrix(tmp_path) + case = summarize(tmp_path)["1m-fixture"] + assert case["speedup_vs_contiguous"] == 11 / 8 + assert case["prepare_plus_iteration_ms"] == dict(contiguous=17, zigzag=17, planner=18) + assert case["prepare_inclusive_gap_pct"] == pytest.approx(100 * (18 / 17 - 1)) + + +@pytest.mark.parametrize( + "broken", + [ + "missing_all", + "missing_dg", + "bad_loss", + "nan_loss", + "missing_rank", + "nan_time", + "negative_memory", + "heads", + "kda_batch_invariant", + "kda_gate_scale", + "candidate_mib", + "nan_prepare", + "inconsistent_prepare", + ], +) +def test_report_rejects_incomplete_precision_evidence(tmp_path, broken): + _matrix(tmp_path) + path = tmp_path / "layout-1m-fixture-planner.json" + data = json.loads(path.read_text()) + row = data["ranks"][0] + if broken == "missing_all": + row["layout_vs_contiguous"] = None + elif broken == "missing_dg": + del row["layout_vs_contiguous"]["kda_dg"] + elif broken == "bad_loss": + row["loss_abs_diff"] = 1 + elif broken == "nan_loss": + row["loss_abs_diff"] = float("nan") + elif broken == "missing_rank": + data["ranks"].pop() + elif broken == "nan_time": + row["samples"][0]["total_ms"] = float("nan") + elif broken == "negative_memory": + row["samples"][0]["peak_allocated_gib"] = -1 + elif broken in ("heads", "kda_batch_invariant", "kda_gate_scale", "candidate_mib"): + data["config"][broken] = 999 + elif broken == "nan_prepare": + row["total_prepare_ms"] = float("nan") + elif broken == "inconsistent_prepare": + row["total_prepare_ms"] += 1 + path.write_text(json.dumps(data)) + with pytest.raises(ValueError): + summarize(tmp_path) + + +@pytest.mark.parametrize("layout", ["contiguous", "zigzag", "planner"]) +@pytest.mark.parametrize("rank", [0, 1]) +def test_report_rejects_failed_fp32_check(tmp_path, layout, rank): + _matrix(tmp_path) + path = tmp_path / f"layout-1m-fixture-{layout}.json" + data = json.loads(path.read_text()) + data["ranks"][rank]["check"] = {"pass_": False} + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="failed correctness check"): + summarize(tmp_path) + + +@pytest.mark.parametrize("check", [None, {"pass_": True}]) +def test_report_accepts_optional_or_successful_fp32_check(tmp_path, check): + _matrix(tmp_path) + for path in tmp_path.glob("*.json"): + data = json.loads(path.read_text()) + for rank in data["ranks"]: + rank["check"] = check + path.write_text(json.dumps(data)) + assert "1m-fixture" in summarize(tmp_path) diff --git a/tests/test_full_csa_cp_metadata.py b/tests/test_full_csa_cp_metadata.py new file mode 100644 index 00000000..3f18e02d --- /dev/null +++ b/tests/test_full_csa_cp_metadata.py @@ -0,0 +1,112 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks of the experimental full-CSA benchmark's addressing contract.""" + +import json +import math +import sys +from types import SimpleNamespace + +import pytest +import torch + +from benchmarks import bench_full_csa_cp as bench +from benchmarks.bench_full_csa_cp import expand_blocks, json_safe, metrics, query_metadata, validate_source_alignment +from cula.distributed import THDRankLayout, THDSpan + + +@pytest.mark.parametrize("visible", [1, 2, 3, 4, 5, 6, 7, 8, 17]) +@pytest.mark.parametrize("origin", [0, 128, 1024]) +def test_complete_blocks_and_causal_tail(visible, origin): + count = min(visible // 4, 4) + ids = torch.full((1, 8), -1, dtype=torch.int32) + # Valid entries intentionally have holes; ids/scores must stay paired. + ids[0, 1 : 2 * count : 2] = torch.arange(count) + scores = ids.float() + 0.25 + sorted_ids, sorted_scores, members, tokens, lengths = expand_blocks( + ids, scores, torch.tensor([origin]), torch.tensor([visible]) + ) + actual = tokens[0, : lengths.item()].tolist() + expected = list(range(origin, origin + 4 * count)) + expected += list(range(origin + visible // 4 * 4, origin + visible)) + assert actual == expected + assert len(actual) == len(set(actual)) + assert torch.all(tokens[0, lengths.item() :] == -1) + assert torch.all(members[0, 4 * count :] == -1) + assert torch.equal(sorted_scores, sorted_ids.float() + 0.25) + + +def test_all_selected_blocks_leave_room_for_three_tail_tokens(): + ids = torch.arange(128, dtype=torch.int32)[None] + _, _, _, tokens, lengths = expand_blocks(ids, ids.float(), torch.tensor([2048]), torch.tensor([515])) + assert lengths.item() == 515 + assert tokens.shape[1] % 128 == 0 + assert torch.equal(tokens[0, :515], torch.arange(2048, 2048 + 515)) + assert torch.all(tokens[0, 515:] == -1) + + +@pytest.mark.parametrize("spans", [(), ((128, 160, 64, 256),), ((512, 520, 256, 1024), (64, 128, 0, 256))]) +def test_bulk_metadata_preserves_addresses_including_reordered_and_empty_spans(spans): + layout = THDRankLayout(0, tuple(THDSpan(i, a, z, start, end, 0) for i, (a, z, start, end) in enumerate(spans))) + origin, visible, rows = query_metadata(layout, "cpu") + expected = [(start, t - start + 1, t) for a, z, start, end in spans for t in range(a, z)] + assert torch.stack((origin, visible, rows), dim=1).tolist() == [list(row) for row in expected] + assert origin.dtype == visible.dtype == rows.dtype == torch.int32 + assert all(x.is_contiguous() for x in (origin, visible, rows)) + + +def test_bulk_metadata_is_prepared_on_cpu_under_a_different_default_device(): + layout = THDRankLayout(0, (THDSpan(0, 4, 8, 0, 16, 0),)) + with torch.device("meta"): + origin, visible, rows = query_metadata(layout, "cpu") + assert origin.tolist() == [0] * 4 + assert visible.tolist() == [5, 6, 7, 8] + assert rows.tolist() == [4, 5, 6, 7] + + +@pytest.mark.parametrize("lengths,world", [([32], 8), ([4, 28], 8), ([8, 8], 2), ([4], 1)]) +def test_source_alignment_accepts_group_wide_aligned_shards(lengths, world): + validate_source_alignment(lengths, world) + + +@pytest.mark.parametrize("lengths,world", [([36], 8), ([12, 24], 8), ([6, 26], 8), ([6], 1)]) +def test_source_alignment_rejects_misaligned_shards_or_documents(lengths, world): + with pytest.raises(ValueError, match="source CP shards and documents must be block aligned"): + validate_source_alignment(lengths, world) + + +@pytest.mark.parametrize("rank", range(8)) +def test_invalid_alignment_stops_every_rank_before_plan_or_collective(monkeypatch, rank): + monkeypatch.setitem(sys.modules, "cudnn", SimpleNamespace(DSA=object())) + monkeypatch.setattr(bench, "_load_cudnn_index_pool_ops", lambda: (None, None)) + monkeypatch.setattr(bench.torch.cuda, "synchronize", lambda device: None) + monkeypatch.setattr(bench.dist, "get_rank", lambda: rank) + monkeypatch.setattr(bench.dist, "get_world_size", lambda: 8) + + def unexpected(*args, **kwargs): + pytest.fail("invalid global alignment reached plan construction or a collective") + + monkeypatch.setattr(bench, "build_kimi_cp_plan", unexpected) + monkeypatch.setattr(bench.dist, "broadcast", unexpected) + with pytest.raises(ValueError, match="block aligned"): + bench.FullCSA([36], heads=64, top_k=128, device="cpu", candidate_mib=0) + + +def test_failure_report_is_strict_json_without_changing_failure_evidence(): + errors = metrics(torch.tensor([float("nan")]), torch.ones(1)) + raw = { + "finite": False, + "check": {"pass_": False, "errors": errors}, + "samples": [{"time": float("inf")}], + "extra": (float("-inf"), 1.5, True, None), + } + encoded = json.dumps(json_safe(raw), allow_nan=False) + decoded = json.loads(encoded) + assert decoded["finite"] is False + assert decoded["check"]["pass_"] is False + assert decoded["check"]["errors"] == {"max_abs": None, "relative_rms": None, "finite": False} + assert decoded["samples"] == [{"time": None}] + assert decoded["extra"] == [None, 1.5, True, None] + assert math.isnan(raw["check"]["errors"]["max_abs"]) + assert math.isinf(raw["samples"][0]["time"]) diff --git a/tests/test_hybrid_dsa_plan.py b/tests/test_hybrid_dsa_plan.py index f4c1c1c6..234fccb9 100644 --- a/tests/test_hybrid_dsa_plan.py +++ b/tests/test_hybrid_dsa_plan.py @@ -30,6 +30,26 @@ def test_selection_plan_coalesces_short_packed_sequences(): assert window.candidate_offsets == (0, 28, 73) +def test_thousands_of_short_sequences_keep_exact_window_budget_boundaries(): + # Each four-token document contributes exactly one complete-block logit. + count, capacity = 4097, 1000 + plan = DsaSelectionPlan.from_cu_seqlens( + tuple(4 * i for i in range(count + 1)), + tuple(range(count + 1)), + top_k=1, + candidate_budget_bytes=capacity * 4, + ) + assert [w.candidate_floats for w in plan.windows] == [1000, 1000, 1000, 1000, 97] + assert [(w.query_start, w.query_end) for w in plan.windows] == [ + (0, 4000), + (4000, 8000), + (8000, 12000), + (12000, 16000), + (16000, 16388), + ] + assert all(w.candidate_offsets == tuple(range(w.candidate_floats + 1)) for w in plan.windows) + + def test_selection_plan_splits_one_sequence_to_bound_peak_scratch(): plan = DsaSelectionPlan.from_cu_seqlens( [0, 32], @@ -46,6 +66,36 @@ def test_selection_plan_splits_one_sequence_to_bound_peak_scratch(): assert plan.windows[1].q_causal_offsets == (plan.windows[1].query_start,) +@pytest.mark.parametrize("block", [1, 4, 16]) +@pytest.mark.parametrize("length", [1, 3, 16, 33]) +@pytest.mark.parametrize("offset", [0, 5, 64]) +def test_selection_windows_use_exact_maximal_prefix_at_budget_boundary(block, length, offset): + # Independent row enumeration covers unaligned offsets, zero-work rows, + # whole-document fits and the final short remainder of a split document. + rows = [(offset + i + 1) // block for i in range(length)] + whole = sum(rows) + for budget in {max(1, max(rows)), max(1, max(rows), whole - 1), max(1, whole), whole + 1}: + plan = DsaSelectionPlan.from_cu_seqlens( + [0, length], + [0, max(1, (offset + length) // block)], + top_k=1, + block_size=block, + q_causal_offsets=[offset], + candidate_budget_bytes=budget * 4, + ) + cursor = 0 + for window in plan.windows: + assert window.query_start == cursor + cursor = window.query_end + used = sum(rows[window.query_start : cursor]) + assert window.candidate_offsets == (0, used) + assert window.q_causal_offsets == (offset + window.query_start,) + assert used <= budget + if cursor < length: + assert used + rows[cursor] > budget + assert cursor == length + + def test_selection_plan_keeps_chunked_prefill_position_offset(): plan = DsaSelectionPlan.from_cu_seqlens( [0, 8], diff --git a/tests/test_kda_independent_benchmark.py b/tests/test_kda_independent_benchmark.py new file mode 100644 index 00000000..10838dad --- /dev/null +++ b/tests/test_kda_independent_benchmark.py @@ -0,0 +1,41 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from benchmarks.bench_kda_independent_shards import paired_samples + + +@pytest.mark.parametrize("warmups,repeats", [(0, 1), (1, 4), (3, 3)]) +def test_paired_samples_warm_both_routes_and_record_actual_order(warmups, repeats): + calls = [] + + def run(label): + calls.append(label) + return {"total_ms": len(calls), "fwd_ms": 1.0} + + samples = paired_samples(run, warmups=warmups, repeats=repeats) + assert len(calls) == 2 * (warmups + repeats) + assert calls[: 2 * warmups].count("baseline") == warmups + assert calls[: 2 * warmups].count("candidate") == warmups + for pair in range(repeats): + expected = ["baseline", "candidate"] if pair % 2 == 0 else ["candidate", "baseline"] + offset = 2 * (warmups + pair) + assert calls[offset : offset + 2] == expected + for position, label in enumerate(expected): + sample = samples[label][pair] + assert sample["pair_index"] == pair + assert sample["execution_order"] == expected + assert sample["position"] == position + assert sample["total_ms"] == offset + position + 1 + assert sample["fwd_ms"] == 1.0 + assert all(len(rows) == repeats for rows in samples.values()) + + +@pytest.mark.parametrize("warmups,repeats", [(-1, 1), (0, 0)]) +def test_paired_samples_reject_invalid_counts_before_running(warmups, repeats): + def unexpected(label): + pytest.fail("invalid counts executed a benchmark route") + + with pytest.raises(ValueError): + paired_samples(unexpected, warmups=warmups, repeats=repeats) diff --git a/tests/test_kimi_cp_helpers.py b/tests/test_kimi_cp_helpers.py index 932f48e4..2947b9b5 100644 --- a/tests/test_kimi_cp_helpers.py +++ b/tests/test_kimi_cp_helpers.py @@ -1,8 +1,10 @@ # Copyright 2025-2026 Ant Group Co., Ltd. # SPDX-License-Identifier: Apache-2.0 +from dataclasses import replace from types import SimpleNamespace +import pytest import torch from cula.distributed import kda as cp @@ -488,15 +490,18 @@ def test_boundary_transport_summary_contract_rejects_precision_or_shape_changes( raise AssertionError("invalid transport summaries must be rejected") -def test_distributed_autograd_delegates_summary_composition_to_cudnn(monkeypatch): +@pytest.mark.parametrize("multiple_documents", [False, True]) +def test_distributed_autograd_delegates_summary_composition_to_cudnn(monkeypatch, multiple_documents): plan = _transport_plan(rank=2, pre=2, post=1) + if multiple_documents: + plan = replace(plan, last_sequence=1, local_cu_seqlens=(0, 2, 4), first_boundary_end=2, last_boundary_start=2) heads, key_dim, value_dim = 2, 4, 3 q = torch.zeros(4, heads, key_dim) k = torch.zeros_like(q) v = torch.zeros(4, heads, value_dim) g = torch.zeros_like(q) beta = torch.zeros(4, heads) - cu = torch.tensor([0, 4], dtype=torch.int32) + cu = torch.tensor(plan.local_cu_seqlens, dtype=torch.int32) options = cp._KdaOptions( scale=None, use_qk_l2norm_in_kernel=False, @@ -538,19 +543,20 @@ def attention(q, k, v, g, beta, cu, **kwargs): ) def summary_bwd(q, k, g, beta, d_o, cu, **kwargs): - assert kwargs["output_context"] is True + assert q.shape[0] == plan.first_boundary_end + assert kwargs["output_context"] is (not multiple_documents) assert kwargs["checkpoint_every_n_tokens"] == 16 - return ( + result = ( torch.full((1, heads, value_dim, key_dim), 5.0), torch.full((1, heads, key_dim, key_dim), 6.0), - torch.tensor([7], dtype=torch.uint8), ) + return result if multiple_documents else (*result, torch.tensor([7], dtype=torch.uint8)) def backward(d_o, q, k, v, g, beta, cu, scale, **kwargs): observed["following_h"] = kwargs["following_state_grad"].clone() observed["following_transition"] = kwargs["following_transition"].clone() observed["composed"] = kwargs["composed_initial_state"].clone() - observed["context"] = kwargs["context"].clone() + observed["context"] = kwargs["context"] observed["checkpoints"] = kwargs["state_checkpoints"].clone() return ( torch.ones_like(q), @@ -587,14 +593,19 @@ def backward(d_o, q, k, v, g, beta, cu, scale, **kwargs): assert observed["following_h"].shape[0] == 1 assert torch.equal(observed["following_transition"], torch.full((1, heads, key_dim, key_dim), 6.0)) assert torch.equal(observed["composed"], torch.full((1, heads, value_dim, key_dim), 4.0)) - assert torch.equal(observed["context"], torch.tensor([7], dtype=torch.uint8)) + if multiple_documents: + assert observed["context"] is None + else: + assert torch.equal(observed["context"], torch.tensor([7], dtype=torch.uint8)) assert torch.equal(observed["checkpoints"], torch.full((1, heads, value_dim, key_dim), 3.0)) assert len(gradients) == 12 assert gradients[6:8] == (None, None) -def test_single_rank_fast_path_calls_attention_export(monkeypatch): - plan = KimiCPPlan.from_global_cu_seqlens([0, 2], rank=0, world_size=1) +@pytest.mark.parametrize("world", [1, 4]) +@pytest.mark.parametrize("major", [9, 10]) +def test_independent_shards_call_native_attention_export(monkeypatch, world, major): + plan = KimiCPPlan.from_global_cu_seqlens(list(range(0, 2 * world + 1, 2)), rank=0, world_size=world) q = torch.zeros(2, 1, 4) v = torch.zeros(2, 1, 3) beta = torch.zeros(2, 1) @@ -606,6 +617,7 @@ def attention(*args, **kwargs): observed["kwargs"] = kwargs return torch.ones_like(v), torch.empty(0) + monkeypatch.setattr(cp.torch.cuda, "get_device_capability", lambda device: (major, 0)) monkeypatch.setattr(cp, "_validate_inputs", lambda *args, **kwargs: None) monkeypatch.setattr(cp, "_load_cudnn_kda_ops", lambda: (attention, object())) @@ -613,3 +625,40 @@ def attention(*args, **kwargs): assert torch.equal(result, torch.ones_like(v)) assert all(actual is expected for actual, expected in zip(observed["args"][:5], (q, q, v, q, beta), strict=True)) + + +@pytest.mark.parametrize("option", ["use_qk_l2norm_in_kernel", "use_beta_sigmoid_in_kernel", "safe_gate"]) +@pytest.mark.parametrize( + "world,major,backend,reject", + [ + (4, 9, "auto", True), + (4, 10, "auto", False), + (1, 9, "auto", False), + (4, 10, "hopper-state-probe", True), + (4, 9, "frost-summary", False), + ], +) +def test_independent_shards_validate_only_multirank_hopper_options(monkeypatch, option, world, major, backend, reject): + plan = KimiCPPlan.from_global_cu_seqlens(list(range(0, 2 * world + 1, 2)), rank=0, world_size=world) + q = torch.zeros(2, 1, 4) + beta = torch.zeros(2, 1) + cu = torch.tensor([0, 2], dtype=torch.int32) + observed = {} + + def attention(*args, **kwargs): + observed.update(kwargs) + return q, None + + monkeypatch.setattr(cp, "_validate_inputs", lambda *args, **kwargs: None) + monkeypatch.setattr(cp.torch.cuda, "get_device_capability", lambda device: (major, 0)) + monkeypatch.setattr(cp, "_load_cudnn_kda_ops", lambda: (attention, object())) + options = {option: True} + if option == "safe_gate": + options.update(a_log=torch.zeros(1), dt_bias=torch.zeros(1, 4)) + if reject: + with pytest.raises(ValueError, match=option): + cp.cudnn_kda_cp(q, q, q, q, beta, cu, plan=plan, backend=backend, **options) + assert not observed + else: + assert cp.cudnn_kda_cp(q, q, q, q, beta, cu, plan=plan, backend=backend, **options) is q + assert observed[option] is True diff --git a/tests/test_kimi_cp_plan.py b/tests/test_kimi_cp_plan.py index f6c6c791..f2f8eae1 100644 --- a/tests/test_kimi_cp_plan.py +++ b/tests/test_kimi_cp_plan.py @@ -103,3 +103,26 @@ def test_cpu_local_boundaries_must_match_plan(): torch.tensor([0, 3], dtype=torch.int32), device=torch.device("cpu"), ) + + +@pytest.mark.parametrize("world", [2, 4, 8, 16, 64]) +def test_document_local_shards_skip_boundary_exchange_globally(world): + cu = _global_cu([1024] * (world * 3)) + plans = [KimiCPPlan.from_global_cu_seqlens(cu, rank=rank, world_size=world) for rank in range(world)] + assert all(not p.requires_boundary_exchange for p in plans) + assert all(p.pre_num_ranks == p.post_num_ranks == 0 for p in plans) + + +def test_isolated_rank_cannot_skip_peers_boundary_collective(): + # Rank zero is independent, but the second document crosses ranks 1/2. + plans = [KimiCPPlan.from_global_cu_seqlens([0, 4, 12], rank=rank, world_size=3) for rank in range(3)] + assert plans[0].pre_num_ranks == plans[0].post_num_ranks == 0 + assert all(p.requires_boundary_exchange for p in plans) + + +def test_independent_detection_uses_supplied_uneven_shards(): + offsets = (0, 3, 11, 16) + plans = [ + KimiCPPlan.from_global_cu_seqlens([0, 1, 3, 11, 13, 16], rank=r, world_size=3, shard_offsets=offsets) for r in range(3) + ] + assert all(not p.requires_boundary_exchange for p in plans) diff --git a/tests/test_thd_1m_cases.py b/tests/test_thd_1m_cases.py new file mode 100644 index 00000000..7b78a1e1 --- /dev/null +++ b/tests/test_thd_1m_cases.py @@ -0,0 +1,91 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +from collections import Counter + +import pytest + +from benchmarks.bench_csa_layout_matrix import performance_cases +from benchmarks.thd_1m_cases import TOTAL_TOKENS, counted_cases, heldout_cases, manifest, statistics_cases, summarize + + +@pytest.mark.parametrize("case", statistics_cases(), ids=lambda case: case.name) +def test_exact_packing_and_provenance(case): + assert sum(case.lengths) == TOTAL_TOKENS + assert all(isinstance(n, int) and n > 0 and n % 4 == 0 for n in case.lengths) + for length, count in Counter(case.long_documents).items(): + assert case.lengths.count(length) == count + assert 0 <= case.final_document_truncation < 16384 + stats = summarize(case) + assert sum(b["documents"] for b in stats["token_length_histogram"]) == len(case.lengths) + assert sum(b["tokens"] for b in stats["token_length_histogram"]) == TOTAL_TOKENS + + +def test_reproducible_cases_and_order_only_controls(): + cases = statistics_cases() + assert cases == statistics_cases() + assert len({case.name for case in cases}) == len(cases) == 18 + lookup = {case.name: case for case in cases} + for base in ("1m-stat-mix80", "1m-stat-mix80-2x256k"): + for order in ("ascending", "descending", "alternating"): + candidate = lookup[f"{base}-{order}"].lengths + assert Counter(candidate) == Counter(lookup[base].lengths) + assert candidate != lookup[base].lengths + assert lookup[base + "-ascending"].lengths == tuple(sorted(lookup[base].lengths)) + assert lookup[base + "-descending"].lengths == tuple(sorted(lookup[base].lengths, reverse=True)) + assert lookup["1m-stat-mix80"].lengths != statistics_cases(seed=123)[2].lengths + + +def test_matrix_keeps_baseline_and_manifest_matches_execution(): + baseline = performance_cases() + assert len(baseline) == 5 + assert [c[0] for c in baseline] == ["1m-single", "1m-uniform", "1m-near-uniform", "1m-20s2l", "1m-power-law"] + cases = performance_cases("statistics", seed=123) + saved = manifest(seed=123)["cases"] + assert [(name, lengths) for name, lengths, _ in cases] == [(c["name"], list(c["lengths"])) for c in saved] + assert len(performance_cases("all")) == 39 + with pytest.raises(ValueError, match="unknown suite"): + performance_cases("typo") + + +@pytest.mark.parametrize("case", counted_cases(), ids=lambda case: case.name) +def test_fixed_document_counts(case): + assert sum(case.lengths) == TOTAL_TOKENS + assert 256 <= len(case.lengths) <= 1024 + assert all(n > 0 and n % 4 == 0 for n in case.lengths) + assert len(case.lengths) == int(next(s for s in case.name.split("-") if s in {"256", "512", "1000", "1024"})) + + +def test_counted_manifest_matches_the_actual_matrix(): + saved = manifest(seed=123, suite="counted")["cases"] + cases = performance_cases("counted", seed=123) + assert [(name, lengths) for name, lengths, _ in cases] == [(c["name"], list(c["lengths"])) for c in saved] + + +def test_heldout_lengths_vary_without_changing_budget_or_document_counts(): + first, second = heldout_cases(20261114), heldout_cases(20261115) + assert first == heldout_cases(20261114) + assert len(first) == 5 + assert {len(c.lengths) for c in first} == {257, 1000, 768, 513} + for a, b in zip(first, second): + assert a.name == b.name + assert len(a.lengths) == len(b.lengths) + assert sorted(a.lengths) != sorted(b.lengths) + for case in (a, b): + assert sum(case.lengths) == TOTAL_TOKENS + assert all(n > 0 and n % 4 == 0 for n in case.lengths) + saved = manifest(20261114, "heldout")["cases"] + actual = performance_cases("heldout", 20261114) + assert [(name, lengths) for name, lengths, _ in actual] == [(c["name"], list(c["lengths"])) for c in saved] + + +def test_validation_suite_has_31_distinct_exact_1m_shapes(): + from benchmarks.bench_csa_layout_matrix import performance_cases + + cases = performance_cases("validation") + assert len(cases) == len({name for name, _, _ in cases}) == 31 + assert all(sum(lengths) == 1 << 20 and all(n > 0 and n % 4 == 0 for n in lengths) for _, lengths, _ in cases) + for seed in (20261114, 20261115, 20261116): + observed = {name: lengths for name, lengths, _ in cases} + for case in heldout_cases(seed): + assert observed[f"{case.name}-seed{seed}"] == list(case.lengths) diff --git a/tests/test_thd_cost_fit.py b/tests/test_thd_cost_fit.py new file mode 100644 index 00000000..10573e44 --- /dev/null +++ b/tests/test_thd_cost_fit.py @@ -0,0 +1,199 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from benchmarks.bench_thd_transport import transport_layout +from benchmarks.calibrate_thd_selection import nonnegative_fit, routing_critical_ms +from cula.distributed import THDRedistributionPlan + + +def test_nonnegative_fit_recovers_independent_costs(): + x = [[1, 0, 0], [0, 1, 0], [0, 0, 1], [2, 3, 4]] + weights, rmse = nonnegative_fit(x, [2, 5, 7, 47]) + assert weights == pytest.approx([2, 5, 7]) + assert rmse == pytest.approx(0, abs=1e-10) + + +def test_fit_drops_negative_cost_instead_of_cancelling_compute(): + weights, _ = nonnegative_fit([[1, 0, 0], [0, 1, 0], [0, 0, 1]], [2, -5, 7]) + assert weights == pytest.approx([2, 0, 7]) + + +@pytest.mark.parametrize("features,targets", [([], []), ([[1, 2, 3]], []), ([[1, 2]], [3]), ([[float("nan"), 1, 2]], [3])]) +def test_fit_rejects_incomplete_or_nonfinite_measurements(features, targets): + with pytest.raises(ValueError): + nonnegative_fit(features, targets) + + +def test_routing_fit_excludes_kda_and_selection_arrival_waits(): + def rank(kda, query, selection, restore): + return dict( + kda_fwd_ms=kda, + query_redistribute_ms=query, + pool_fwd_ms=1, + pool_gather_ms=2, + selection_key_layout_ms=0.5, + selection_ms=selection, + selection_restore_ms=restore, + ) + + # Rank 0 waits 80 ms before query exchange and 60 ms before restore. + # Both exchanges take 5/3 ms after the last rank arrives, respectively. + a, b = rank(20, 85, 10, 63), rank(100, 5, 70, 3) + assert routing_critical_ms([a, b]) == pytest.approx(8.5) + # Shifting the slow phase earlier on the other rank leaves transfer cost + # unchanged. Neither the original waiting times nor rank identity matter. + assert routing_critical_ms([b, a]) == pytest.approx(8.5) + assert routing_critical_ms([rank(100, 5, 70, 3)] * 2) == pytest.approx(8.5) + + +def test_routing_fit_rejects_invalid_phase_measurements(): + with pytest.raises(ValueError): + routing_critical_ms([]) + + +@pytest.mark.parametrize("kind", ["intra", "intra_dense", "inter", "inter_dense", "striped"]) +@pytest.mark.parametrize("world,per_node", [(4, 2), (8, 4), (16, 8)]) +def test_transport_calibration_exercises_the_declared_links(kind, world, per_node): + rows = 128 + ranks = transport_layout(world, rows, per_node, kind) + for rank in range(world): + p = THDRedistributionPlan.build(ranks, tuple(i * rows for i in range(world + 1)), rank) + assert p.target_rows == p.source_rows == rows + assert sum(p.send_counts) == sum(p.receive_counts) == rows + assert not p.identity + peers = [i for i, count in enumerate(p.send_counts) if count] + if kind.startswith("intra"): + assert all(i // per_node == rank // per_node for i in peers) + assert (rank in peers) == (kind == "intra_dense") + elif kind.startswith("inter"): + assert all(i // per_node != rank // per_node for i in peers) + else: + assert peers == list(range(world)) + assert len(set(p.send_counts)) == 1 + + +@pytest.mark.parametrize("world,per_node", [(4, 2), (8, 8), (16, 8)]) +@pytest.mark.parametrize("kind", ["intra_pair", "intra", "intra_dense", "striped"]) +@pytest.mark.parametrize("overlap", [False, True]) +def test_transport_fit_prediction_uses_the_same_cost_model_as_planning(world, per_node, kind, overlap): + from benchmarks.fit_thd_transport import predict + from cula.distributed import THDSelectionCostModel + + rows = 128 + layouts = transport_layout(world, rows, per_node, kind) + plans = [THDRedistributionPlan.build(layouts, tuple(i * rows for i in range(world + 1)), r) for r in range(world)] + intra_out = [0] * world + inter_out = [0] * world + intra_in = [0] * world + inter_in = [0] * world + node_out = [0] * (world // per_node) + node_in = node_out.copy() + degrees_out = [0] * world + degrees_in = [0] * world + edge = [] + for s, p in enumerate(plans): + for t, n in enumerate(p.send_counts): + if s == t or not n: + continue + edge.append(n) + degrees_out[s] += 1 + degrees_in[t] += 1 + if s // per_node == t // per_node: + intra_out[s] += n + intra_in[t] += n + else: + inter_out[s] += n + inter_in[t] += n + node_out[s // per_node] += n + node_in[t // per_node] += n + f = dict( + rank_intra_rows=intra_out + intra_in, + rank_inter_rows=inter_out + inter_in, + peer_rows=max(edge), + node_rows=max(node_out + node_in), + peer_degree=max(degrees_out + degrees_in), + copies=0, + ) + coeff = [1, 3, 4, 2, 0.5, 0.007, 0.01] + factor = 65536 * 20608 / 1e6 + m = THDSelectionCostModel( + candidate_ns=0, + query_ns=0, + span_us=0, + bytes_per_query=20608, + rank_node_ids=tuple(r // per_node for r in range(world)), + intra_node_gbps=factor / coeff[0], + inter_node_gbps=factor / coeff[1], + peer_gbps=factor / coeff[2], + inter_node_aggregate_gbps=factor / coeff[3], + packing_ns_per_query=coeff[4] * 1e6 / 65536, + peer_startup_us=coeff[5] * 1000, + exchange_us=coeff[6] * 1000, + overlap_intra_inter=overlap, + ) + assert predict(f, coeff, overlap) == pytest.approx(m.estimate_ms(layouts, block_size=4)) + + +@pytest.fixture +def transport_fit_inputs(monkeypatch): + from types import SimpleNamespace + + import numpy as np + + optimize = pytest.importorskip("scipy.optimize") + # Keep export/metadata regressions independent of optimizer convergence. + coefficients = np.array([1.0, 3.0, 4.0, 2.0, 0.5, 0.007, 0.01]) + monkeypatch.setattr(optimize, "differential_evolution", lambda *a, **kw: SimpleNamespace(x=coefficients.copy())) + monkeypatch.setattr(optimize, "least_squares", lambda *a, **kw: SimpleNamespace(x=coefficients.copy())) + profile = dict(model=dict(bytes_per_query=20608, rank_node_ids=[0] * 8, packing_ns_per_query=8.5)) + feature = dict(rank_intra_rows=[128] * 16, rank_inter_rows=[0] * 16, peer_rows=128, node_rows=0, copies=0, peer_degree=1) + record = dict(heldout=False, marker_exact=True, critical_median_ms=1, features=feature, kind="intra", rows=128) + return dict(world=8, ranks_per_node=8, records=[record.copy() for _ in range(6)]), profile + + +@pytest.mark.parametrize("world,per_node", [(8, 4), (16, 8), (8, 8), (4, 1)]) +@pytest.mark.parametrize("old_per_node", [4, 8]) +def test_transport_fit_exports_calibration_topology(transport_fit_inputs, world, per_node, old_per_node): + from copy import deepcopy + + from benchmarks.fit_thd_transport import fit + + raw, profile = transport_fit_inputs + profile["model"]["rank_node_ids"] = [r // old_per_node for r in range(8)] + original = deepcopy(profile) + raw.update(world=world, ranks_per_node=per_node) + result = fit(raw, profile) + assert result["model"]["rank_node_ids"] == [r // per_node for r in range(world)] + assert result["transport_calibration"]["world_size"] == world + assert result["transport_calibration"]["ranks_per_node"] == per_node + assert profile == original + + +@pytest.mark.parametrize("world,per_node", [(0, 4), (1, 1), (8, 0), (8, 3), (8, 16), (8.0, 4), (8, 4.0), (True, 1), (8, True)]) +def test_transport_fit_rejects_invalid_topology(transport_fit_inputs, world, per_node): + from benchmarks.fit_thd_transport import fit + + raw, profile = transport_fit_inputs + raw.update(world=world, ranks_per_node=per_node) + with pytest.raises(ValueError, match="topology"): + fit(raw, profile) + + +@pytest.mark.parametrize("heldout", [False, True]) +@pytest.mark.parametrize("timing", [float("nan"), float("inf"), float("-inf"), 0, -1]) +def test_transport_fit_rejects_invalid_timings_before_optimization(transport_fit_inputs, monkeypatch, heldout, timing): + import scipy.optimize + + from benchmarks.fit_thd_transport import fit + + raw, profile = transport_fit_inputs + raw["records"].append(dict(raw["records"][0], heldout=heldout, critical_median_ms=timing)) + + def unexpected(*args, **kwargs): + pytest.fail("invalid timing reached the optimizer") + + monkeypatch.setattr(scipy.optimize, "differential_evolution", unexpected) + with pytest.raises(ValueError, match="finite positive"): + fit(raw, profile) diff --git a/tests/test_thd_planner.py b/tests/test_thd_planner.py index 963cd543..1b6b52d7 100644 --- a/tests/test_thd_planner.py +++ b/tests/test_thd_planner.py @@ -3,10 +3,22 @@ import random from collections import defaultdict +from itertools import permutations, product import pytest -from cula.distributed.thd_planner import THDPlannerConfig, plan_thd_layout +from cula.distributed.thd_planner import ( + THDPlannerConfig, + THDRankLayout, + THDSelectionCostModel, + THDSpan, + _contiguous_items, + _localize_spans, + _minimum_assignment, + _transfer_lower_bound, + plan_thd_layout, + plan_thd_selection, +) def _boundaries(lengths): @@ -114,3 +126,470 @@ def test_random_thd_plans_preserve_coverage_and_never_lose_to_contiguous(): _assert_exact_cover(plan, boundaries) assert plan.metrics.score <= dict(plan.evaluated_scores)["contiguous"] assert max(plan.metrics.token_loads) - min(plan.metrics.token_loads) <= plan.block_size + + +def _cost_model(**overrides): + values = dict( + candidate_ns=0.1, + query_ns=1.0, + span_us=1.0, + bytes_per_query=20608, + intra_node_gbps=200.0, + inter_node_gbps=20.0, + packing_ns_per_query=20.0, + rank_node_ids=(0, 0, 1, 1), + exchange_us=10.0, + ) + return THDSelectionCostModel(**(values | overrides)) + + +def test_calibrated_model_rejects_migration_when_network_cost_dominates(): + lengths = [1 << 20] + cheap = plan_thd_layout(_boundaries(lengths), world_size=4, cost_model=_cost_model()) + expensive = plan_thd_layout( + _boundaries(lengths), + world_size=4, + cost_model=_cost_model(intra_node_gbps=0.001, inter_node_gbps=0.001), + ) + assert cheap.policy != "contiguous" + assert expensive.policy == "contiguous" + assert cheap.score_unit == expensive.score_unit == "ms" + _assert_exact_cover(cheap, _boundaries(lengths)) + assert cheap.metrics.score < dict(cheap.evaluated_scores)["contiguous"] + + +def test_identity_cost_contains_no_transport_and_margin_can_keep_it(): + boundaries = _boundaries([1024] * 1024) + plan = plan_thd_layout(boundaries, world_size=4, cost_model=_cost_model()) + assert plan.policy == "contiguous" + # One document's exact complete-block candidate count: sum(floor(i/4)). + candidates = sum(i // 4 for i in range(1, 1025)) * 256 + expected = candidates * 0.1 / 1e6 + 262144 / 1e6 + 256 / 1e3 + assert plan.metrics.score == pytest.approx(expected) + assert len(plan.evaluated_scores) == 1 # Proven lower bound prunes all migrations. + conservative = plan_thd_layout([0, 1 << 20], world_size=4, cost_model=_cost_model(min_gain_fraction=0.99)) + assert conservative.policy == "contiguous" + + +@pytest.mark.parametrize( + "overrides", + [ + {"candidate_ns": float("nan")}, + {"inter_node_gbps": 0}, + {"packing_ns_per_query": -1}, + {"min_gain_fraction": 1}, + {"bytes_per_query": True}, + {"rank_node_ids": (0, False)}, + {"peer_gbps": 0}, + {"peer_gbps": float("inf")}, + {"peer_gbps": True}, + {"min_gain_ms": -1}, + {"min_gain_ms": float("nan")}, + {"min_gain_ms": True}, + ], +) +def test_cost_model_rejects_invalid_profiles(overrides): + with pytest.raises((ValueError, TypeError)): + _cost_model(**overrides) + + +def test_cost_model_requires_matching_group_topology(): + with pytest.raises(ValueError, match="node map"): + plan_thd_layout([0, 65536], world_size=8, cost_model=_cost_model()) + + +@pytest.mark.parametrize("selection", [False, True]) +def test_absolute_saving_budget_prunes_search_without_changing_relative_margin(selection): + kwargs = {"max_tokens_per_rank": 524288} if selection else {} + planner = plan_thd_selection if selection else plan_thd_layout + reusable = planner([0, 1 << 20], world_size=4, cost_model=_cost_model(), **kwargs) + assert reusable.policy != "contiguous" + # Above the entire baseline cost, no possible redistribution can repay + # the caller's preparation budget. No candidate search is necessary. + one_use = planner([0, 1 << 20], world_size=4, cost_model=_cost_model(min_gain_ms=10000), **kwargs) + assert one_use.policy == "contiguous" + assert len(one_use.evaluated_scores) == 1 + savings = dict(reusable.evaluated_scores)["contiguous"] - reusable.metrics.score + rejected = planner([0, 1 << 20], world_size=4, cost_model=_cost_model(min_gain_ms=savings + 1), **kwargs) + assert rejected.policy == "contiguous" + + +def test_rank_matching_equals_exhaustive_optimum(): + generator = random.Random(20261113) + for size in range(1, 7): + for _ in range(8): + costs = [[generator.random() * 100 for _ in range(size)] for _ in range(size)] + assignment = _minimum_assignment(costs) + actual = sum(costs[i][j] for i, j in enumerate(assignment)) + expected = min(sum(costs[i][j] for i, j in enumerate(p)) for p in permutations(range(size))) + assert sorted(assignment) == list(range(size)) + assert actual == pytest.approx(expected) + assert _minimum_assignment([[0.0] * 4 for _ in range(4)]) == (0, 1, 2, 3) + + +@pytest.mark.parametrize("total", [65536, 65540]) +def test_localized_bins_preserve_capacity_and_do_not_worsen_chosen_score(total): + plan = plan_thd_layout([0, total], world_size=4, cost_model=_cost_model(min_gain_fraction=0.0)) + _assert_exact_cover(plan, [0, total]) + assert max(plan.metrics.token_loads) - min(plan.metrics.token_loads) <= 4 + scores = dict(plan.evaluated_scores) + assert plan.metrics.score <= min(value for name, value in scores.items() if not name.endswith("-local")) + original = plan_thd_layout([0, total], world_size=4) + localized = _localize_spans([s for r in original.ranks for s in r.spans], 4, 4, _cost_model()) + loads = [sum(s.tokens for s in localized if s.owner == r) for r in range(4)] + assert tuple(loads) == original.metrics.token_loads + + +@pytest.mark.parametrize("capacity_factor", [1, 1.25, 2]) +def test_selection_planner_preserves_cover_and_explicit_capacity(capacity_factor): + rng = random.Random(62813) + for _ in range(25): + lengths = [4 * rng.randint(1, 12000) for _ in range(rng.randint(1, 40))] + cu = _boundaries(lengths) + capacity = 4 * int(((cu[-1] // 4 + 3) // 4) * capacity_factor) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=capacity, cost_model=_cost_model()) + # Empty target ranks are legal. Exact coverage is checked independently + # of scheduling order and must survive inverse redistribution. + spans = sorted((s.token_start, s.token_end) for r in plan.ranks for s in r.spans) + assert spans[0][0] == 0 and spans[-1][1] == cu[-1] + assert all(a[1] == b[0] for a, b in zip(spans, spans[1:])) + for r in plan.ranks: + assert r.local_tokens <= capacity + for s in r.spans: + assert s.owner == r.rank + assert (s.sequence_start, s.sequence_end) == tuple(cu[s.sequence : s.sequence + 2]) + assert s.sequence_start <= s.token_start < s.token_end <= s.sequence_end + assert plan.metrics.score <= dict(plan.evaluated_scores)["contiguous"] + assert plan == plan_thd_selection(cu, world_size=4, max_tokens_per_rank=capacity, cost_model=_cost_model()) + + +def test_selection_keeps_short_documents_in_source_ownership(): + cu = _boundaries([64] * 100 + [524288, 517888]) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=524288, cost_model=_cost_model()) + assert plan.policy != "contiguous" + for layout in plan.ranks: + for span in layout.spans: + if span.sequence < 100: + assert span.owner == 0 + assert span.tokens == 64 + + +@pytest.mark.parametrize("capacity,error", [(True, TypeError), (0, ValueError), (15, ValueError), (12, ValueError)]) +def test_selection_rejects_invalid_row_capacity(capacity, error): + with pytest.raises(error, match="max_tokens_per_rank"): + plan_thd_selection([0, 64], world_size=4, max_tokens_per_rank=capacity, cost_model=_cost_model()) + + +@pytest.mark.parametrize("lengths", [[32], [4, 12, 16], [4] * 8]) +@pytest.mark.parametrize("bandwidth", [0.01, 1, 10000]) +def test_transfer_lower_bound_never_exceeds_exhaustive_partition_optimum(lengths, bandwidth): + # Enumerate every two-rank assignment of eight blocks, including empty + # ranks and unequal counts. Coalesce before scoring to avoid counting + # artificial fragment costs as necessary work. + cu = tuple(_boundaries(lengths)) + model = _cost_model(rank_node_ids=(0, 1), intra_node_gbps=bandwidth, inter_node_gbps=bandwidth) + source = _contiguous_items(cu, 2, 4) + ranks = tuple( + THDRankLayout( + r, + tuple( + THDSpan(i.sequence, i.token_start, i.token_end, i.sequence_start, i.sequence_end, r) + for i, owner in source + if owner == r + ), + ) + for r in range(2) + ) + total_work = sum(sum(i // 4 for i in range(1, length + 1)) for length in lengths) + mean = (model.candidate_ns * total_work / 1e6 + model.query_ns * 32 / 1e6 + model.span_us * len(lengths) / 1e3) / 2 + bound = _transfer_lower_bound(source, 2, 4, model, mean, model.estimate_ms(ranks, block_size=4)) + optimum = float("inf") + for owners in product(range(2), repeat=8): + spans = [] + for seq, (start, end) in enumerate(zip(cu, cu[1:])): + a = start + for z in range(start + 4, end + 1, 4): + if z == end or owners[z // 4] != owners[a // 4]: + spans.append(THDSpan(seq, a, z, start, end, owners[a // 4])) + a = z + candidate = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(2)) + optimum = min(optimum, model.estimate_ms(candidate, block_size=4)) + assert bound <= optimum + 1e-12 + + +def test_peer_limit_prices_aggregate_edge_rows_not_individual_fragments(): + from benchmarks.bench_thd_transport import transport_layout + + model = _cost_model( + candidate_ns=0, + query_ns=0, + span_us=0, + packing_ns_per_query=0, + exchange_us=0, + peer_gbps=10, + intra_node_gbps=1000, + inter_node_gbps=1000, + ) + permutation = transport_layout(4, 128, 2, "inter") + fragmented = tuple( + THDRankLayout( + r.rank, + tuple( + THDSpan(s.sequence, a, a + 64, s.sequence_start, s.sequence_end, s.owner) + for s in r.spans + for a in range(s.token_start, s.token_end, 64) + ), + ) + for r in permutation + ) + expected = 128 * model.bytes_per_query / (10 * 1e6) + assert model.estimate_ms(permutation, block_size=4) == pytest.approx(expected) + assert model.estimate_ms(fragmented, block_size=4) == pytest.approx(expected) + striped = transport_layout(4, 128, 2, "striped") + assert model.estimate_ms(striped, block_size=4) == pytest.approx(expected / 4) + + +@pytest.mark.parametrize("world", [2, 4, 8, 16, 32, 64]) +@pytest.mark.parametrize("peer_rate", [0.01, 12.0, 500.0]) +def test_selection_peer_model_remains_bounded_on_unseen_topologies(world, peer_rate): + rng = random.Random(73381 + world) + lengths = [4 * rng.randint(1, 256) for _ in range(127)] + [131072, 262144] + rng.shuffle(lengths) + cu = _boundaries(lengths) + capacity = 8 * ((cu[-1] // 4 + world - 1) // world) + model = _cost_model(rank_node_ids=tuple(r // 8 for r in range(world)), peer_gbps=peer_rate) + plan = plan_thd_selection(cu, world_size=world, max_tokens_per_rank=capacity, cost_model=model) + spans = sorted((s.token_start, s.token_end) for r in plan.ranks for s in r.spans) + assert spans[0][0] == 0 and spans[-1][1] == cu[-1] + assert all(a[1] == b[0] for a, b in zip(spans, spans[1:])) + assert max(plan.metrics.token_loads) <= capacity + assert plan.metrics.score <= dict(plan.evaluated_scores)["contiguous"] + assert plan.metrics.score == model.estimate_ms(plan.ranks, block_size=4) + + +def test_full_zigzag_can_win_even_when_all_documents_are_below_split_threshold(): + from benchmarks.cp_layouts import query_layouts + + cu = _boundaries([16384, 4096, 1024]) + model = _cost_model( + candidate_ns=10, + query_ns=0, + span_us=0, + packing_ns_per_query=0, + exchange_us=0, + intra_node_gbps=1e9, + inter_node_gbps=1e9, + min_gain_fraction=0, + ) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=16384, cost_model=model) + zigzag, _ = query_layouts(cu, 4, "zigzag") + assert plan.policy == "zigzag" + assert plan.ranks == zigzag + assert dict(plan.evaluated_scores)["zigzag"] == model.estimate_ms(zigzag, block_size=4) + assert len(plan.evaluated_scores) == 2 # No long-document candidates. + _assert_exact_cover(plan, cu) + + +def test_full_zigzag_obeys_capacity_even_when_short_chunks_concentrate_on_rank_zero(): + cu = _boundaries([4] * 8 + [64]) + model = _cost_model(candidate_ns=10000, min_gain_fraction=0, min_gain_ms=0) + plan = plan_thd_selection(cu, world_size=4, max_tokens_per_rank=24, cost_model=model) + assert "zigzag" not in dict(plan.evaluated_scores) + assert max(plan.metrics.token_loads) <= 24 + + +@pytest.mark.parametrize("world", [2, 4, 8, 16, 64]) +def test_shared_full_zigzag_matches_explicit_early_late_block_definition(world): + from cula.distributed import zigzag_thd_layout + + lengths = [4, 12, 64, 260, 16384] + cu = _boundaries(lengths) + ranks = zigzag_thd_layout(cu, world_size=world) + for r in range(world): + expected = [] + for d, length in enumerate(lengths): + q, rem = divmod(length // 4, 2 * world) + for piece in (r, 2 * world - 1 - r): + begin = cu[d] + 4 * (piece * q + min(piece, rem)) + end = cu[d] + 4 * ((piece + 1) * q + min(piece + 1, rem)) + if end > begin: + expected.append((d, begin, end)) + assert [(s.sequence, s.token_start, s.token_end) for s in ranks[r].spans] == expected + + +def test_shared_node_bandwidth_and_peer_startup_are_not_per_fragment_costs(): + from benchmarks.bench_thd_transport import transport_layout + + model = _cost_model( + candidate_ns=0, + query_ns=0, + span_us=0, + packing_ns_per_query=0, + exchange_us=0, + intra_node_gbps=1e9, + inter_node_gbps=1e9, + inter_node_aggregate_gbps=10, + peer_startup_us=7, + ) + pair = transport_layout(4, 128, 2, "inter_pair") + ring = transport_layout(4, 128, 2, "inter") + dense = transport_layout(4, 128, 2, "inter_dense") + row_ms = 128 * model.bytes_per_query / (10 * 1e6) + assert model.estimate_ms(pair, block_size=4) == pytest.approx(row_ms + 0.007) + assert model.estimate_ms(ring, block_size=4) == pytest.approx(2 * row_ms + 0.007) + assert model.estimate_ms(dense, block_size=4) == pytest.approx(2 * row_ms + 0.014) + + +@pytest.mark.parametrize( + "overrides", + [ + dict(peer_startup_us=-1), + dict(peer_startup_us=True), + dict(peer_startup_us=float("nan")), + dict(inter_node_aggregate_gbps=0), + dict(inter_node_aggregate_gbps=True), + dict(inter_node_aggregate_gbps=float("inf")), + ], +) +def test_topology_model_rejects_invalid_limits(overrides): + with pytest.raises(ValueError): + _cost_model(**overrides) + + +@pytest.mark.parametrize("world", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("overlap", [False, True]) +@pytest.mark.parametrize("topology", ["contiguous", "interleaved", "irregular"]) +def test_zigzag_bound_never_overprices_concrete_layout(world, overlap, topology): + from cula.distributed.thd_planner import _zigzag_lower_bound, zigzag_thd_layout + + rng = random.Random(27113 + world) + for _ in range(50): + cu = _boundaries([4 * rng.randint(1, 8192) for _ in range(rng.randint(1, 50))]) + model = _cost_model( + rank_node_ids=tuple( + r // 3 if topology == "contiguous" else r % 3 if topology == "interleaved" else rng.randrange(world) + for r in range(world) + ), + peer_startup_us=7, + inter_node_aggregate_gbps=12, + peer_gbps=8, + overlap_intra_inter=overlap, + ) + source = _contiguous_items(cu, world, 4) + bound = _zigzag_lower_bound(cu, source, world, 4, model) + actual = model.estimate_ms(zigzag_thd_layout(cu, world_size=world), block_size=4) + assert bound <= actual + 1e-10 + + +def test_transport_prices_only_executed_permutations(): + from dataclasses import replace + + from benchmarks.bench_thd_transport import transport_layout + + layouts = transport_layout(4, 128, 2, "striped") + model = _cost_model( + candidate_ns=0, + query_ns=0, + span_us=0, + packing_ns_per_query=1000, + exchange_us=0, + intra_node_gbps=1000, + inter_node_gbps=1000, + ) + unpriced = replace(model, packing_ns_per_query=0) + assert model.estimate_ms(layouts, block_size=4) == unpriced.estimate_ms(layouts, block_size=4) + reversed_layouts = tuple(THDRankLayout(r.rank, tuple(reversed(r.spans))) for r in layouts) + assert model.estimate_ms(reversed_layouts, block_size=4) - unpriced.estimate_ms( + reversed_layouts, block_size=4 + ) == pytest.approx(0.128) + + +def test_source_local_spans_still_price_local_reordering(): + from benchmarks.cp_layouts import query_layouts + + layouts, _ = query_layouts(_boundaries([16] * 8), 4, "contiguous") + model = _cost_model(candidate_ns=0, query_ns=0, span_us=0, packing_ns_per_query=1000, exchange_us=10) + assert model.estimate_ms(layouts, block_size=4) == 0 + reordered = tuple(THDRankLayout(r.rank, r.spans[::-1]) for r in layouts) + # No rows cross ranks, but source-side packing and its inverse still + # execute, as does the group-wide eager exchange for a nonidentity map. + assert model.estimate_ms(reordered, block_size=4) == pytest.approx(0.032 + 0.010) + + +@pytest.mark.parametrize("margin", [0, 0.05]) +def test_pinned_critical_path_avoids_useless_long_document_search(monkeypatch, margin): + cu = _boundaries([64] * 998 + [33600] * 2) + model = _cost_model(candidate_ns=0.001, span_us=10, min_gain_fraction=margin) + + def forbidden(*args, **kwargs): + pytest.fail("moving long documents cannot improve the pinned critical path") + + monkeypatch.setattr("cula.distributed.thd_planner._selection_assign", forbidden) + monkeypatch.setattr("cula.distributed.thd_planner._paired_selection_items", forbidden) + plan = plan_thd_selection( + cu, world_size=4, max_tokens_per_rank=65536, cost_model=model, config=THDPlannerConfig(min_chunk_tokens=1024) + ) + assert plan.policy == "contiguous" + assert plan.metrics.score == model.estimate_ms(plan.ranks, block_size=4) + _assert_exact_cover(plan, cu) + + +@pytest.mark.parametrize("world", [1, 2, 8, 64]) +@pytest.mark.parametrize("floor", [0, 4, 16, 1024]) +def test_zigzag_grain_check_equals_materialized_candidate(world, floor): + from cula.distributed.thd_planner import _zigzag_meets_floor, zigzag_thd_layout + + rng = random.Random(812 + world) + for lengths in ([4] * world, [4 * world], [4, 1024, 65536], [4 * rng.randint(1, 1024) for _ in range(17)]): + cu = _boundaries(lengths) + source = _contiguous_items(cu, world, 4) + fragments = {(s.sequence, s.token_start, s.token_end) for s, _ in source} + zigzag = zigzag_thd_layout(cu, world_size=world) + expected = all( + s.tokens >= floor or (s.sequence, s.token_start, s.token_end) in fragments for r in zigzag for s in r.spans + ) + assert _zigzag_meets_floor(cu, world, 4, floor, fragments) == expected + + +def test_independent_link_model_and_its_outgoing_bound(): + from dataclasses import replace + + from benchmarks.bench_thd_transport import transport_layout + + ranks = transport_layout(4, 128, 2, "striped") + serial = _cost_model( + candidate_ns=0, query_ns=0, span_us=0, packing_ns_per_query=0, exchange_us=0, intra_node_gbps=10, inter_node_gbps=10 + ) + parallel = replace(serial, overlap_intra_inter=True) + assert parallel.estimate_ms(ranks, block_size=4) == pytest.approx(2 / 3 * serial.estimate_ms(ranks, block_size=4)) + assert parallel._outgoing_capacity_gbps(0) == 20 + assert serial._outgoing_capacity_gbps(0) == 10 + with pytest.raises(TypeError, match="boolean"): + replace(serial, overlap_intra_inter=1) + + +def test_parallel_link_lower_bound_against_all_small_assignments(): + from cula.distributed.thd_planner import _coalesce, _make_item + + cu = (0, 4, 12, 24) + world = 3 + model = _cost_model( + rank_node_ids=(0, 0, 1), + candidate_ns=10000, + query_ns=10, + overlap_intra_inter=True, + peer_startup_us=2, + inter_node_aggregate_gbps=25, + ) + base = _contiguous_items(cu, world, 4) + spans = _coalesce(base) + source = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world)) + mean = ( + model.candidate_ns * sum(i.causal_work for i, _ in base) + model.query_ns * 24 + ) / 1e6 / world + model.span_us * 3 / 1e3 / world + bound = _transfer_lower_bound(base, world, 4, model, mean, model.estimate_ms(source, block_size=4)) + blocks = [_make_item(d, a, a + 4, cu[d], cu[d + 1], 4) for d in range(3) for a in range(cu[d], cu[d + 1], 4)] + for owners in product(range(world), repeat=6): + spans = _coalesce(zip(blocks, owners)) + ranks = tuple(THDRankLayout(r, tuple(s for s in spans if s.owner == r)) for r in range(world)) + assert bound <= model.estimate_ms(ranks, block_size=4) + 1e-10 diff --git a/tests/test_thd_redistribute.py b/tests/test_thd_redistribute.py new file mode 100644 index 00000000..bbb3a593 --- /dev/null +++ b/tests/test_thd_redistribute.py @@ -0,0 +1,130 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU protocol tests, including unequal counts and a nontrivial receive order.""" + +from dataclasses import replace + +import pytest +import torch + +from benchmarks.cp_layouts import query_layouts +from cula._metadata import balanced_shards +from cula.distributed import THDRedistributionPlan, THDRedistributor + + +@pytest.mark.parametrize("policy", ["contiguous", "zigzag", "planner"]) +@pytest.mark.parametrize("boundaries", [(0, 64), (0, 4, 12, 32, 64), (0, 32768), (0, 4, 8, 12, 16)]) +def test_exchange_and_its_transpose(monkeypatch, policy, boundaries): + world = 4 + layouts, _ = query_layouts(boundaries, world, policy) + # Reverse local span order to require a genuine receive permutation too. + if policy != "contiguous": + layouts = tuple(replace(x, spans=x.spans[::-1]) for x in layouts) + offsets = balanced_shards(boundaries[-1], world) + plans = [THDRedistributionPlan.build(layouts, offsets, rank) for rank in range(world)] + source = [torch.arange(offsets[r], offsets[r + 1], dtype=torch.float64)[:, None].repeat(1, 3) for r in range(world)] + target = [ + ( + torch.cat([torch.arange(s.token_start, s.token_end, dtype=torch.float64) for s in x.spans]) + if x.spans + else torch.empty(0, dtype=torch.float64) + )[:, None].repeat(1, 3) + for x in layouts + ] + for rank, plan in enumerate(plans): + monkeypatch.setattr("torch.distributed.get_world_size", lambda group: world) + monkeypatch.setattr("torch.distributed.get_rank", lambda group: rank) + exchange = THDRedistributor(plan, "cpu") + calls = [] + reverse, factor = False, 1 + + def collective(out, packed, output_split_sizes, input_split_sizes, group): + calls.append(reverse) + tensors = [x * factor for x in (target if reverse else source)] + parts = [] + for sender, peer_plan in enumerate(plans): + from cula.distributed.redistribute import _permutations + + ranges = peer_plan.receive_ranges if reverse else peer_plan.send_ranges + rows = peer_plan.target_rows if reverse else peer_plan.source_rows + order, _ = _permutations(ranges, rows, "cpu") + data = tensors[sender] if order is None else tensors[sender].index_select(0, order) + counts = peer_plan.receive_counts if reverse else peer_plan.send_counts + begin = sum(counts[:rank]) + parts.append(data[begin : begin + counts[rank]]) + out.copy_(torch.cat(parts)) + expected_counts = plan.receive_counts if reverse else plan.send_counts + assert tuple(input_split_sizes) == expected_counts + assert sum(output_split_sizes) == out.shape[0] + # Verify the sender's local packing as well, independently of the + # mocked receive construction. + ranges = plan.receive_ranges if reverse else plan.send_ranges + idx = torch.tensor([i for a, z in ranges for i in range(a, z)], dtype=torch.int64) + assert torch.equal(packed, tensors[rank].index_select(0, idx)) + + monkeypatch.setattr("torch.distributed.all_to_all_single", collective) + leaf = source[rank].clone().requires_grad_() + routed = exchange.distribute(leaf) + assert torch.equal(routed, target[rank]) + reverse, factor = True, 2 + # Nonuniform gradients expose a wrong inverse even when a sum-of-rows + # test would pass. Empty destinations still participate in backward. + routed.backward(target[rank] * factor) + assert torch.equal(leaf.grad, source[rank] * factor) + reverse, factor = True, 1 + leaf = target[rank].clone().requires_grad_() + restored = exchange.restore(leaf) + assert torch.equal(restored, source[rank]) + reverse, factor = False, 3 + restored.backward(source[rank] * factor) + assert torch.equal(leaf.grad, target[rank] * factor) + assert calls == ([] if plan.identity else [False, True, True, False]) + + +def test_reject_overlap(): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + broken = (layouts[0], replace(layouts[1], spans=tuple(replace(span, owner=1) for span in layouts[0].spans)), *layouts[2:]) + with pytest.raises(ValueError, match="without gaps or overlap"): + THDRedistributionPlan.build(broken, (0, 16, 32, 48, 64), 0) + + +@pytest.mark.parametrize("field,value", [("owner", False), ("token_start", 0.0), ("token_end", 16.0), ("sequence_end", 64.0)]) +def test_reject_noninteger_span_metadata_before_collectives(field, value): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + span = replace(layouts[0].spans[0], **{field: value}) + layouts = (replace(layouts[0], spans=(span,)), *layouts[1:]) + with pytest.raises(TypeError, match="must be integers"): + THDRedistributionPlan.build(layouts, (0, 16, 32, 48, 64), 0) + + +def test_locally_unchanged_rank_cannot_skip_a_peer_exchange(): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + layouts = tuple( + replace(layouts[source], rank=target, spans=tuple(replace(s, owner=target) for s in layouts[source].spans)) + for target, source in enumerate((0, 2, 1, 3)) + ) + plans = [THDRedistributionPlan.build(layouts, (0, 16, 32, 48, 64), r) for r in range(4)] + assert plans[0].remote_rows == plans[3].remote_rows == 0 + assert all(not plan.identity for plan in plans) + + +def test_prepare_indices_do_not_follow_the_framework_default_device(): + from cula.distributed.redistribute import _permutations + + with torch.device("meta"): + order, inverse = _permutations(((4, 8), (0, 4)), 8, "cpu") + assert order.device.type == inverse.device.type == "cpu" + assert order.tolist() == [4, 5, 6, 7, 0, 1, 2, 3] + assert inverse.tolist() == [4, 5, 6, 7, 0, 1, 2, 3] + + +def test_unindexed_cuda_device_is_resolved_at_prepare(monkeypatch): + layouts, _ = query_layouts((0, 64), 4, "contiguous") + plan = THDRedistributionPlan.build(layouts, (0, 16, 32, 48, 64), 2) + monkeypatch.setattr("torch.distributed.get_world_size", lambda group: 4) + monkeypatch.setattr("torch.distributed.get_rank", lambda group: 2) + monkeypatch.setattr("torch.cuda.current_device", lambda: 3) + # Identity needs no device allocation, so this also runs without CUDA. + exchange = THDRedistributor(plan, "cuda") + assert exchange.device == torch.device("cuda:3")