diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 8c340d6d..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "third_party/flash-linear-attention"] - path = third_party/flash-linear-attention - url = https://github.com/fla-org/flash-linear-attention.git diff --git a/BENCHMARK_GB200.md b/BENCHMARK_GB200.md index 96057b25..ad1ecf7e 100644 --- a/BENCHMARK_GB200.md +++ b/BENCHMARK_GB200.md @@ -10,6 +10,10 @@ ## KDA (Kimi Delta Attention) +> The tables below are historical full-KDA measurements retained for reference. +> The current standalone intra-kernel microbenchmark has a different scope and +> configuration and does not reproduce these results. + ### Fixed-Length (H=64, D=128, bf16) | B | T | FLA Triton (ms) | cuLA (ms) | Speedup | @@ -54,10 +58,10 @@ Summary (10 configs): **avg=1.34x**, min=1.21x, max=1.44x. Summary (18 configs): **avg=1.39x**, min=1.37x, max=1.40x. -To reproduce: +Current intra-kernel microbenchmark (different scope): ```bash -python benchmarks/bench_kda.py --mode both +python benchmarks/bench_kda_chunk_intra.py ``` ## Lightning Attention diff --git a/BENCHMARK_GB200_CUDA_130.md b/BENCHMARK_GB200_CUDA_130.md index 2d349461..4c63487d 100644 --- a/BENCHMARK_GB200_CUDA_130.md +++ b/BENCHMARK_GB200_CUDA_130.md @@ -10,6 +10,10 @@ ## KDA (Kimi Delta Attention) +> The tables below are historical full-KDA measurements retained for reference. +> The current standalone intra-kernel microbenchmark has a different scope and +> configuration and does not reproduce these results. + ### Fixed-Length (H=64, D=128, bf16) | B | T | FLA Triton (ms) | cuLA (ms) | Speedup | @@ -54,10 +58,10 @@ Summary (10 configs): **avg=1.33x**, min=1.20x, max=1.39x. Summary (18 configs): **avg=1.35x**, min=1.33x, max=1.37x. -To reproduce: +Current intra-kernel microbenchmark (different scope): ```bash -python benchmarks/bench_kda.py --mode both +python benchmarks/bench_kda_chunk_intra.py ``` ## Lightning Attention diff --git a/BENCHMARK_GB300.md b/BENCHMARK_GB300.md index 87b9d695..e961f6a1 100644 --- a/BENCHMARK_GB300.md +++ b/BENCHMARK_GB300.md @@ -10,6 +10,10 @@ ## KDA (Kimi Delta Attention) +> The tables below are historical full-KDA measurements retained for reference. +> The current standalone intra-kernel microbenchmark has a different scope and +> configuration and does not reproduce these results. + ### Fixed-Length (H=64, D=128, bf16) | B | T | FLA Triton (ms) | cuLA (ms) | Speedup | @@ -54,10 +58,10 @@ Summary (10 configs): **avg=1.45x**, min=1.36x, max=1.57x. Summary (18 configs): **avg=1.32x**, min=1.29x, max=1.35x. -To reproduce: +Current intra-kernel microbenchmark (different scope): ```bash -python benchmarks/bench_kda.py --mode both +python benchmarks/bench_kda_chunk_intra.py ``` ## Lightning Attention diff --git a/README.md b/README.md index 0e611af0..332bdd2a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Linear attention mechanisms reformulate standard attention to use linear-time state updates instead of quadratic pairwise interactions, making them well suited for long-context LLM workloads. Recent variants such as [GLA](https://arxiv.org/abs/2312.06635), [KDA](http://arxiv.org/abs/2510.26692), [GDN](https://arxiv.org/abs/2412.06464), and [Lightning Attention](https://arxiv.org/abs/2405.17381) further improve expressiveness with gating, delta-style updates, and chunkwise decomposition. -**cuLA** provides hand-tuned CUDA implementations of these linear attention variants, targeting NVIDIA Blackwell (SM10X) and Hopper (SM90) GPUs. It is designed as a submodule of [flash-linear-attention (FLA)](https://github.com/fla-org/flash-linear-attention), sharing the same interface — adopting cuLA requires only a one-line import change. For ease of maintenance, cuLA is currently developed as a standalone library; the end goal is for users to seamlessly access these kernels through FLA. Since FLA already has a kernel dispatch mechanism in place, integration will be ready soon. +**cuLA** provides hand-tuned CuTeDSL implementations of these linear attention variants, targeting NVIDIA Blackwell (SM10X) and Hopper (SM90) GPUs. The runtime is a standalone kernel library: it does not import or vendor FLA, Triton, or a C++/CUDA extension. Complete operators and deliberately exposed atomic kernels are available through `cula.kernels`. > **⚠️ Early Stage:** cuLA is in its early development phase. Many kernels still have significant room for optimization, and the API may evolve. We warmly welcome contributions from the community — whether it's performance tuning, new algorithm support, bug fixes, or architectural improvements. Every contribution helps push the boundaries of linear attention on modern GPUs! @@ -32,17 +32,12 @@ Platform-independent wheels containing the CuTeDSL sources are available on [Git Replace `` with the release tag (e.g., `v0.2.0`) and `` with the version (e.g., `0.2.0`). Or download the `.whl` file directly from the [Releases page](https://github.com/inclusionAI/cuLA/releases) and install it with `pip install .whl`. -The cuLA wheel does not install FLA. Install `flash-linear-attention` before importing `chunk_kda`: - - pip install flash-linear-attention - ### Build from Source **Clone cuLA & dependencies:** ```bash git clone https://github.com/inclusionAI/cuLA.git -git submodule update --init --recursive ``` **Install PyTorch and CuTeDSL:** @@ -64,9 +59,6 @@ pip install 'nvidia-cutlass-dsl[cu13]>=4.6.2,<4.9' **Install cuLA & dependencies:** ```bash -# Install flash-linear-attention for benchmark repro -pip install -e third_party/flash-linear-attention - # Install cuLA pip install -e . --no-build-isolation ``` @@ -108,28 +100,26 @@ See [the SM90 Lightning prefill pipeline](docs/lightning_attn_sm90.md) for the warp-group roles, WGMMA dataflow, pipeline stages, recurrent-state placement, and fixed/packed scheduling modes. -### KDA (Kimi Delta Attention) — Blackwell (SM10X) - -Just change the import: +### KDA (Kimi Delta Attention) prefill ```python import torch -from cula.kda import chunk_kda # <-- one-line change from fla.ops.kda +from cula.kernels.kda import kda_prefill B, T, H, K, V = 2, 2048, 32, 128, 128 device = 'cuda' -q = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16, requires_grad=True) -k = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16, requires_grad=True) -v = torch.randn(B, T, H, V, device=device, dtype=torch.bfloat16, requires_grad=True) +q = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16) +k = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16) +v = torch.randn(B, T, H, V, device=device, dtype=torch.bfloat16) g = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16) * 0.1 # gate (log space) beta = torch.randn(B, T, H, device=device, dtype=torch.bfloat16).sigmoid() A_log = torch.randn(H, device=device, dtype=torch.float32) * 0.01 dt_bias = torch.zeros(H * K, device=device, dtype=torch.float32) init_state = torch.zeros(B, H, K, V, device=device, dtype=torch.float32) -# Forward -o, final_state = chunk_kda( +# Forward-only standalone operator +o, final_state = kda_prefill( q=q, k=k, v=v, g=g, beta=beta, A_log=A_log, dt_bias=dt_bias, @@ -141,28 +131,27 @@ o, final_state = chunk_kda( lower_bound=-5.0, ) -# Backward -do = torch.randn_like(o) -o.backward(do) - print(f'Output shape: {o.shape}') # [2, 2048, 32, 128] print(f'Final state shape: {final_state.shape}') # [2, 32, 128, 128] ``` **Notes:** -- `safe_gate=True` is required to leverage TensorCore acceleration. +- `kda_prefill` is forward-only. The modular SM100 training primitives are exposed separately rather than hidden behind an FLA autograd wrapper. +- `safe_gate=True` is required by this path. - `beta` supports both `float32` and `bfloat16`; `initial_state` must be `float32`. - `cu_seqlens` (for variable-length sequences) must be `int32`. ## Usage See [USAGE.md](USAGE.md) for detailed usage examples and notes. +See [the standalone kernel inventory](docs/KERNEL_VALUE.md) for the public +kernel boundary and the overlap assessment against FlashInfer and cuDNN. ## Benchmarks Benchmarks run on a single **NVIDIA GB200/H200** GPU with **PyTorch 2.9.1**, **Triton 3.5.1**. -FLA baseline: [flash-linear-attention v0.5.0](https://github.com/fla-org/flash-linear-attention/releases/tag/v0.5.0). +Historical result files use FLA as an external baseline. FLA is not a runtime or source-tree dependency. **Blackwell (SM10X)** @@ -179,11 +168,17 @@ See [BENCHMARK_H200.md](BENCHMARK_H200.md) for CuTe DSL FlashKDA results on an H - **FlashKDA Prefill (Hopper):** **avg 2.72x** speedup over FLA across 28 fixed-length and variable-length configs, up to **7.56x**. - **FlashKDA Intracard CP (Hopper):** **4.29x geo-mean** speedup over serial FlashKDA on 28 CP-engaged long-sequence configs, up to **7.83x**. -To reproduce the benchmark suites directly: +The commands below run the currently maintained benchmark entry points. The +Blackwell KDA command is an intra-kernel microbenchmark; it does not reproduce +the historical full-KDA headline numbers above, which used different shapes +and included the complete operator pipeline. + +The comparison commands below require FLA to be installed separately; FLA is +not imported by the cuLA runtime package. ```bash # Blackwell (SM10X) -python benchmarks/bench_kda.py --mode both +python benchmarks/bench_kda_chunk_intra.py python benchmarks/bench_lightning_attn_prefill.py --modes no_state varlen python benchmarks/bench_la_decode_vs_fla.py --heads 64 --head-dim 128 @@ -205,11 +200,8 @@ python -m pytest -m cula_slow -q # Default tests plus every extended CUDA case python -m pytest -m cula_full -q -# Tests for modular KDA forward against FLA Triton implementation -python -m pytest tests/test_kda_sm100_chunk_vs_fla.py -v -# Tests for modular KDA forward against naive KDA reference -python -m pytest tests/test_kda_sm100_chunk_vs_naive.py -v -# Tests for the SM90 CuTeDSL two-kernel prefill + intracard CP (vs FLA) +# Optional reference tests (require a separately installed FLA) +# SM90 CuTeDSL two-kernel prefill + intracard CP python -m pytest tests/test_kda_sm90_prefill_vs_fla.py tests/test_kda_sm90_intracard_cp.py -v # Tests for Lightning Attention prefill on SM100 python tests/test_lightning_sm100_prefill.py @@ -218,13 +210,6 @@ python -m pytest tests/test_lightning_attn_prefill_dispatch.py tests/test_lightn # Tests for Lightning Attention decode python -m pytest tests/test_lightning_decode.py -v -# The two chunk-vs-reference files also retain their narrower KDA fast/slow split. -# Fast (default) — representative correctness paths for default CI and local iteration -python -m pytest tests/test_kda_sm100_chunk_vs_naive.py tests/test_kda_sm100_chunk_vs_fla.py -v -# Slow — broader stress coverage for nightly or manual runs -python -m pytest -m kda_slow tests/test_kda_sm100_chunk_vs_naive.py tests/test_kda_sm100_chunk_vs_fla.py -v -# Full sweep (fast + slow) — run before submitting a PR -python -m pytest -m kda_full tests/test_kda_sm100_chunk_vs_naive.py tests/test_kda_sm100_chunk_vs_fla.py -v ```
@@ -252,7 +237,7 @@ See [REPO_LAYOUT.md](REPO_LAYOUT.md) for the full directory structure and a summ ## Roadmap -* [ ] Integrate into [flash-linear-attention](https://github.com/fla-org/flash-linear-attention) via FLA's kernel dispatch mechanism +* [ ] Stabilize the standalone atomic-kernel contracts and publish per-kernel benchmarks. * [ ] Polynomial approximation to mitigate the exponential bottleneck, as in [Flash-Attentiton-4](https://arxiv.org/abs/2603.05451). * [ ] Larger chunk size and 2-CTA on SM10X for improved throughput. * [ ] Continuous optimization via agentic methods such as [AVO](https://arxiv.org/abs/2603.24517). @@ -262,7 +247,7 @@ See [REPO_LAYOUT.md](REPO_LAYOUT.md) for the full directory structure and a summ **Train** -* [x] Modular KDA Forward (SM10X, compatible with [Kimi CP](https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/cp/README.md)) +* [x] Modular KDA Forward (SM10X) * [x] kda chunk intra * [x] chunk gated delta h * [x] recompute wu @@ -284,7 +269,7 @@ See [REPO_LAYOUT.md](REPO_LAYOUT.md) for the full directory structure and a summ * [ ] Fused KDA prefill kernel (SM10X) -* [ ] MTP support +* [x] KDA and Lightning Attention MTP decode/verify support * [ ] More aggressive fusion of small neighboring kernels like cumsum for inference scenarios. diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md index ad48d1f7..e45517a3 100644 --- a/REPO_LAYOUT.md +++ b/REPO_LAYOUT.md @@ -8,15 +8,16 @@ cuLA/ │ ├── __init__.py │ ├── utils.py # Architecture, stream-buffer, and cu_seqlens helpers │ │ -│ ├── kda/ # KDA public API, wrappers, autograd, and support kernels -│ │ ├── __init__.py # Lazy exports for chunk, prefill, and decode APIs +│ ├── kernels/ # Stable standalone lazy exports + static registry +│ │ ├── kda.py # KDA operators and atomic primitives +│ │ ├── lightning.py # Lightning operators +│ │ └── registry.py # Architecture/level/status metadata +│ │ +│ ├── kda/ # KDA public wrappers +│ │ ├── __init__.py # Lazy exports for prefill and decode APIs │ │ ├── prefill.py # Public FlashKDA prefill validation and routing │ │ ├── flashkda.py # SM90 CuTeDSL K1+K2 wrapper, autograd shell, and CP selection -│ │ ├── chunk.py # SM100 modular chunk API + autograd -│ │ ├── chunk_fwd.py # SM100 forward orchestration -│ │ ├── chunk_intra.py # CuTeDSL forward intra + Triton backward intra -│ │ ├── chunk_bwd.py # Triton + FLA + CuTeDSL backward orchestration -│ │ └── wy_intra.py / wy_recompute.py # Triton WY preparation kernels +│ │ └── chunk_intra.py # Pure-CuTeDSL SM100 intra/WU composition │ │ │ ├── lightning/ # [non-KDA] Lightning Attention public wrappers │ │ @@ -47,7 +48,6 @@ cuLA/ ├── tests/ # CPU registry tests and architecture-marked GPU tests ├── docs/ # Pipeline and backend-dispatch design notes ├── scripts/build_wheel.sh -├── third_party/flash-linear-attention/ # FLA baseline and reused operations ├── README.md USAGE.md REPO_LAYOUT.md RECOMMENDED_CODING_STYLE.md └── setup.py pyproject.toml LICENSE ``` @@ -56,7 +56,8 @@ cuLA/ | Directory | Language | Description | |-----------|----------|-------------| -| `cula/kda/` | Python / Triton | Public KDA APIs, wrappers, autograd, routing, and supporting Triton kernels. `kda_prefill` calls FlashKDA directly without a native fallback. See [`cula/kda/README.md`](cula/kda/README.md). | +| `cula/kernels/` | Python | Framework-independent public surface and static kernel inventory. Imports are lazy and do not compile kernels. | +| `cula/kda/` | Python / CuTeDSL | KDA prefill/decode wrappers and the pure-CuTeDSL SM100 modular composition. `kda_prefill` calls FlashKDA directly without a native fallback. See [`cula/kda/README.md`](cula/kda/README.md). | | `cula/ops/kda/` | Python (CuTeDSL) | CuTeDSL KDA kernels organized into SM100 modular kernels, SM90 FlashKDA K1+K2 and intracard CP, decode, and experimental code. | | `cula/ops/lightning/` · `cula/ops/experimental/` | Python (CuTeDSL) | `[non-KDA]` Lightning/linear-attention kernels and prototypes. Lightning prefill dispatches to the SM90 or SM100 backend from Q's device capability. | | `cula/ops/{inv,ptx}.py` · `cula/ops/sm100/ptx.py` | Python | Shared low-level helpers used across operators. | diff --git a/USAGE.md b/USAGE.md index ff6e0a1a..e3d40dbd 100644 --- a/USAGE.md +++ b/USAGE.md @@ -10,11 +10,11 @@ cuLA exposes several KDA entry points targeting different GPU architectures: | Kernel | GPU | Import | |---|---|---| -| Modular Forward | Blackwell (SM100) | `from cula.kda import chunk_kda` | +| Modular Intra/WU | Blackwell (SM100) | `from cula.kernels.kda import chunk_kda_fwd_intra` | | Two-Kernel Prefill | Hopper (SM90) | `from cula.kda import flashkda_prefill` | | Public Prefill | Hopper (SM90) / Blackwell (SM100) | `from cula.kda import kda_prefill` | -`chunk_kda` and `flashkda_prefill` each call a fixed implementation. +`chunk_kda_fwd_intra` and `flashkda_prefill` each call owned CuTeDSL implementations. `kda_prefill` is the public FlashKDA wrapper and has no native-extension fallback. The removed `kda_prefill_hopper`, `kda_prefill_hopper_opt`, and `kda_prefill_hopper_auto` names should be migrated to one of these FlashKDA @@ -61,50 +61,39 @@ with torch.inference_mode(): ) ``` -### Modular Forward (SM100 — Blackwell) +### Modular Intra/WU primitives (SM100 — Blackwell) -The modular forward kernel replaces sub-kernels of KDA in FLA (chunk_intra, chunk_delta_h, fwd_o, etc.) for easy integration with [Kimi CP](https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/cp/README.md). +This boundary accepts an already activated, chunk-cumulative FP32 gate and +returns the intra-chunk matrices plus W/U preparation tensors. Gate activation, +cumsum, recurrent-state propagation, and output projection remain explicit +caller-owned stages. #### Example ```python import torch -from cula.kda import chunk_kda +from cula.kernels.kda import chunk_kda_fwd_intra B, T, H, K, V = 2, 2048, 32, 128, 128 device = 'cuda' -q = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16, requires_grad=True) -k = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16, requires_grad=True) -v = torch.randn(B, T, H, V, device=device, dtype=torch.bfloat16, requires_grad=True) -g = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16) * 0.1 +q = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16) +k = torch.randn(B, T, H, K, device=device, dtype=torch.bfloat16) +v = torch.randn(B, T, H, V, device=device, dtype=torch.bfloat16) +gk = torch.randn(B, T, H, K, device=device, dtype=torch.float32) * 0.1 beta = torch.randn(B, T, H, device=device, dtype=torch.bfloat16).sigmoid() -A_log = torch.randn(H, device=device, dtype=torch.float32) * 0.01 -dt_bias = torch.zeros(H * K, device=device, dtype=torch.float32) -init_state = torch.zeros(B, H, K, V, device=device, dtype=torch.float32) -o, final_state = chunk_kda( - q=q, k=k, v=v, g=g, beta=beta, - A_log=A_log, dt_bias=dt_bias, - initial_state=init_state, - output_final_state=True, - use_qk_l2norm_in_kernel=True, - use_gate_in_kernel=True, +w, u, qg, kg, Aqk, Akk = chunk_kda_fwd_intra( + q=q, k=k, v=v, gk=gk, beta=beta, safe_gate=True, - lower_bound=-5.0, ) - -# Backward is supported -o.backward(torch.randn_like(o)) - -print(f'Output shape: {o.shape}') # [2, 2048, 32, 128] -print(f'Final state shape: {final_state.shape}') # [2, 32, 128, 128] +print(w.shape, u.shape, Aqk.shape, Akk.shape) ``` **Notes** -- The backward pass is currently supported via FLA's implementation; further optimizations are on the roadmap. -- Compatible with [Kimi CP](https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/cp/README.md) via the `cp_context` parameter, same as in FLA. +- This is not a full KDA operator and has no autograd wrapper. +- Import individual primitives from `cula.kernels.kda` when integrating a custom pipeline. --- diff --git a/benchmarks/bench_intracard_cp.py b/benchmarks/bench_intracard_cp.py deleted file mode 100644 index 41e9de3a..00000000 --- a/benchmarks/bench_intracard_cp.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -bench_intracard_cp.py — Benchmark: Intracard Context-Parallel speedup - for chunk_kda (KDA forward) - -Measures the speedup of cuLA's intracard context-parallel path against the -non-CP baseline across a range of varlen configurations. Also verifies that -the heuristic does not regress throughput when CP is correctly bypassed. - -Usage: - python bench_intracard_cp.py [--ncu] [--sanitizer] - -With --ncu, warmup=1 and iters=1 for ncu profiling: - ncu --set full -o report python bench_intracard_cp.py --ncu -""" - -import argparse -import contextlib -import os -import pathlib -import sys - -os.environ.setdefault("CULA_INTRACARD_CP", "1") - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) - -import torch - -from benchmarks.utils import ( - SEED, - exclusive_cumsum, - prepare_safe_gate_inputs, - set_seed, -) -from cula.kda.chunk_fwd import chunk_kda_fwd -from cula.ops.kda.sm100.cp.chunk_delta_h import ( - compute_subseq_len, - prepare_subseq_cu_seqlens, - should_use_intracard_cp, -) -from cula.utils import get_device_sm_count - -# ============================================================ -# Constants -# ============================================================ -BT, D = 64, 128 -H_VALUES = [4, 8] -WARMUP = 25 -N_ITERS = 100 -NCU_MODE = False -SANITIZER_MODE = False - -# (tag, seq_lens) — each entry is tested at every H in H_VALUES -CONFIGS = [ - # --- single seq (ascending length) --- - ("T=4K", [4096]), - ("T=8K", [8192]), - ("T=32K", [32768]), - ("T=64K", [65536]), - ("T=128K", [131072]), - # --- equal-length batches (~32K total) --- - ("8x4K", [4096] * 8), - ("4x8K", [8192] * 4), - ("2x16K", [16384] * 2), - # --- asymmetric multi-seq --- - ("16K+16K", [16384, 16384]), - ("24K+8K", [24576, 8192]), - ("28K+4K", [28672, 4096]), - ("32K+256+256", [32768, 256, 256]), - ("40K+1K+8K", [40960, 1024, 8192]), - ("64K+512+256+128", [65536, 512, 256, 128]), - ("128K+1K", [131072, 1024]), - # --- 128K + several short seqs --- - ("128K+2x1K", [131072, 1024, 1024]), - ("128K+5x1K", [131072] + [1024] * 5), - ("128K+10x1K", [131072] + [1024] * 10), -] - - -# ============================================================ -# CP toggle -# ============================================================ -@contextlib.contextmanager -def cp_on(enable: bool): - old = os.environ.get("CULA_INTRACARD_CP") - os.environ["CULA_INTRACARD_CP"] = "1" if enable else "0" - try: - if enable: - with torch.inference_mode(): - yield - else: - yield - finally: - if old is None: - os.environ.pop("CULA_INTRACARD_CP", None) - else: - os.environ["CULA_INTRACARD_CP"] = old - - -# ============================================================ -# Helpers -# ============================================================ -def time_kernel(fn, warmup=None, n_iters=None): - if warmup is None: - warmup = 1 if (NCU_MODE or SANITIZER_MODE) else WARMUP - if n_iters is None: - n_iters = 1 if (NCU_MODE or SANITIZER_MODE) else N_ITERS - for _ in range(warmup): - fn() - torch.cuda.synchronize() - start_evt = torch.cuda.Event(enable_timing=True) - end_evt = torch.cuda.Event(enable_timing=True) - start_evt.record() - for _ in range(n_iters): - fn() - end_evt.record() - torch.cuda.synchronize() - return start_evt.elapsed_time(end_evt) / n_iters - - -def run_cp(q, k, v, g, beta, scale, A_log, dt_bias, cu_seqlens, lower_bound, *, enable_cp): - with cp_on(enable_cp): - chunk_kda_fwd( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - initial_state=None, - output_final_state=False, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens.cpu(), - safe_gate=True, - lower_bound=lower_bound, - use_gate_in_kernel=True, - A_log=A_log, - dt_bias=dt_bias, - ) - - -def predict_cp(seq_lens, H, num_sms): - cu = torch.tensor( - exclusive_cumsum(seq_lens), - dtype=torch.int32, - ) - if not should_use_intracard_cp(cu, num_sms, H, BT): - return False, 0 - max_len = int(torch.diff(cu).max().item()) - subseq_len = compute_subseq_len(max_len, num_sms, H, BT, num_seqs=len(seq_lens)) - _, split_info, total_subseqs = prepare_subseq_cu_seqlens(cu, subseq_len, BT) - return bool(split_info), total_subseqs - - -# ============================================================ -# Benchmark -# ============================================================ -def bench_cp(h_values, configs): - print("\n" + "=" * 100) - print(" Intracard CP Benchmark: CP-on vs CP-off") - print("=" * 100) - - device = torch.device("cuda") - num_sms = get_device_sm_count(device) - results = [] - - for H in h_values: - for tag, seq_lens in configs: - set_seed(SEED) - torch.cuda.empty_cache() - - total_T = sum(seq_lens) - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - inputs = prepare_safe_gate_inputs(1, total_T, H, D, device, cu_seqlens=cu_seqlens, seed=SEED) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, lower_bound = inputs["scale"], inputs["lower_bound"] - - pred, n_sub = predict_cp(seq_lens, H, num_sms) - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - ) - - ms_off = time_kernel(lambda: run_cp(**common, enable_cp=False)) - ms_on = time_kernel(lambda: run_cp(**common, enable_cp=True)) - - r = { - "tag": tag, - "H": H, - "total_T": total_T, - "pred": pred, - "n_sub": n_sub, - "ms_off": ms_off, - "ms_on": ms_on, - "speedup": ms_off / ms_on if ms_on > 0 else float("inf"), - } - results.append(r) - - del q, k, v, g, beta, A_log, dt_bias, inputs - torch.cuda.empty_cache() - - return results - - -# ============================================================ -# Report -# ============================================================ -def print_report(results, h_values): - sep = "=" * 110 - print(f"\n\n{sep}") - print(" BENCHMARK REPORT: Intracard CP") - print(" CP-on vs CP-off (same kernel, different code paths)") - print(f" D={D} dtype=bf16 safe_gate=True") - wu = 1 if (NCU_MODE or SANITIZER_MODE) else WARMUP - ni = 1 if (NCU_MODE or SANITIZER_MODE) else N_ITERS - mode_tag = " [NCU mode]" if NCU_MODE else (" [Sanitizer mode]" if SANITIZER_MODE else "") - print(f" Warmup={wu} Iters={ni}{mode_tag}") - print(sep) - - for H_val in h_values: - h_results = [r for r in results if r["H"] == H_val] - if not h_results: - continue - - print(f"\n [H={H_val}]") - print(f" {'─' * 95}") - print( - f" {'config':<24s} {'T':>7s} {'pred':>4s} {'sub':>4s}" - f" │ {'CP_off(ms)':>10s} {'CP_on(ms)':>10s} {'Speedup':>8s}" - ) - print(f" {'─' * 95}") - for r in h_results: - pred_s = "Y" if r["pred"] else "N" - print( - f" {r['tag']:<24s} {r['total_T']:>7d} {pred_s} {r['n_sub']:>4d}" - f" │ {r['ms_off']:>10.4f} {r['ms_on']:>10.4f} {r['speedup']:>7.2f}x" - ) - print(f" {'─' * 95}") - - # Summary - triggered = [r for r in results if r["pred"]] - bypassed = [r for r in results if not r["pred"]] - - if triggered: - speedups = [r["speedup"] for r in triggered] - geo = 1.0 - for s in speedups: - geo *= s - geo = geo ** (1 / len(speedups)) - print( - f"\n CP triggered ({len(triggered)} configs): " - f"geo-mean={geo:.2f}x best={max(speedups):.2f}x worst={min(speedups):.2f}x" - ) - - if bypassed: - ratios = [r["ms_on"] / r["ms_off"] for r in bypassed] - print( - f" CP bypassed ({len(bypassed)} configs): " - f"mean overhead={sum(ratios) / len(ratios):.3f}x max={max(ratios):.3f}x " - f"(1.00 = no regression)" - ) - - print(f"\n{sep}\n") - - -# ============================================================ -# Main -# ============================================================ -def main(): - parser = argparse.ArgumentParser(description="bench_intracard_cp: CP-on vs CP-off benchmark") - parser.add_argument( - "--ncu", - action="store_true", - help="NCU profiling mode: warmup=1, iters=1", - ) - parser.add_argument( - "--sanitizer", - action="store_true", - help="Sanitizer mode: warmup=1, iters=1", - ) - args = parser.parse_args() - - global NCU_MODE, SANITIZER_MODE - if args.ncu: - NCU_MODE = True - print("[NCU mode] warmup=1, iters=1") - if args.sanitizer: - SANITIZER_MODE = True - print("[Sanitizer mode] warmup=1, iters=1") - - results = bench_cp(H_VALUES, CONFIGS) - print_report(results, H_VALUES) - return results - - -if __name__ == "__main__": - main() diff --git a/benchmarks/bench_kda.py b/benchmarks/bench_kda.py deleted file mode 100644 index ea79102f..00000000 --- a/benchmarks/bench_kda.py +++ /dev/null @@ -1,460 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -bench_kda.py — Benchmark: cuLA CuTe DSL vs FLA Triton baseline - for chunk_kda (KDA forward) - -Compares: - - Accuracy: relative_rms_error, relative max diff between cuLA and FLA outputs - - Performance: kernel execution time (ms) with CUDA events - -Modes: - - Fixed-length: B=1, B=2 with various T - - Varlen: ~20 seqs with 2-3x length variation - -Usage: - python bench_kda.py [--mode fixed|varlen|both] [--ncu] - -With --ncu, warmup=1 and iters=1 for ncu profiling: - ncu --set full -o report python bench_kda.py --mode varlen --ncu -""" - -import argparse -import os -import pathlib -import sys - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) -os.environ.setdefault("FLA_USE_FAST_OPS", os.getenv("CULA_USE_FAST_MATH", "1")) # Enable fast ops in FLA for fair comparison - -import torch -from fla.ops.kda import chunk_kda as fla_chunk_kda - -from benchmarks.utils import ( - SEED, - benchmark_cuda_mode_fn, - build_varlen_configs, - exclusive_cumsum, - prepare_safe_gate_inputs, - relative_rms_error_rel_max_mean_abs, - set_seed, -) -from cula.kda import chunk_kda as cula_chunk_kda - -# ============================================================ -# Constants -# ============================================================ -H, D = 64, 128 -HV = H -WARMUP = 10 -N_ITERS = 30 -NCU_MODE = False -SANITIZER_MODE = False -DISABLE_RECOMPUTE = False # Whether to disable recompute (compute QG in forward) - - -# ============================================================ -# Helpers -# ============================================================ -def generate_balanced_seqlens(total_tokens, num_seqs): - base = total_tokens // num_seqs - remainder = total_tokens % num_seqs - return [base] * (num_seqs - 1) + [base + remainder] - - -def run_kda(q, k, v, g, beta, scale, A_log, dt_bias, init_state, cu_seqlens, lower_bound, fn): - return fn( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - initial_state=init_state, - output_final_state=True, - use_qk_l2norm_in_kernel=True, - cu_seqlens=cu_seqlens, - use_gate_in_kernel=True, - safe_gate=True, - lower_bound=lower_bound, - disable_recompute=DISABLE_RECOMPUTE, - ) - - -def check_determinism(H=4, total_T=8192, num_seqs=10, iters=10000): - """Run the kernel multiple times and check that outputs are identical.""" - device = torch.device("cuda") - D = 128 - seq_lens = generate_balanced_seqlens(total_T, num_seqs) - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - - inputs = prepare_safe_gate_inputs(1, total_T, H, D, device, cu_seqlens=cu_seqlens, num_v_heads=HV) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, init_state, lower_bound = inputs["scale"], inputs["init_state"], inputs["lower_bound"] - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=init_state, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - ) - - ref_out, ref_state = run_kda(**common, fn=cula_chunk_kda) - - for i in range(iters): - out, state = run_kda(**common, fn=cula_chunk_kda) - assert torch.isnan(out).sum() == 0, f"Output contains NaNs at iter {i}" - assert torch.isnan(state).sum() == 0, f"State contains NaNs at iter {i}" - assert torch.isfinite(out).all(), f"Output contains infs at iter {i}" - assert torch.isfinite(state).all(), f"State contains infs at iter {i}" - assert torch.equal(out, ref_out), f"Output mismatch at iter {i}" - assert torch.equal(state, ref_state), f"State mismatch at iter {i}" - - -# ============================================================ -# Fixed-length benchmark -# ============================================================ -def bench_fixed(configs): - print("\n" + "=" * 100) - print(f" Fixed-Length Benchmark: cuLA CuTe DSL vs FLA Triton disable_recompute={DISABLE_RECOMPUTE}") - print("=" * 100) - results = [] - - for B, T in configs: - set_seed(SEED) - device = torch.device("cuda") - torch.cuda.empty_cache() - - seq_lens = [T] * B - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - - inputs = prepare_safe_gate_inputs(B, T, H, D, device, cu_seqlens=cu_seqlens, num_v_heads=HV) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, init_state, lower_bound = inputs["scale"], inputs["init_state"], inputs["lower_bound"] - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=init_state, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - ) - - # Accuracy: compare outputs - o_fla, _ = run_kda(**common, fn=fla_chunk_kda) - o_cula, _ = run_kda(**common, fn=cula_chunk_kda) - torch.cuda.synchronize() - - relative_rms_error, rel_max, mean_diff = relative_rms_error_rel_max_mean_abs(o_fla, o_cula) - - # Performance - def fn_fla(**common_kw): - return lambda: run_kda(**common_kw, fn=fla_chunk_kda) - - def fn_cula(**common_kw): - return lambda: run_kda(**common_kw, fn=cula_chunk_kda) - - ms_fla = benchmark_cuda_mode_fn( - fn_fla(**common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - ms_cula = benchmark_cuda_mode_fn( - fn_cula(**common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - speedup = ms_fla / ms_cula if ms_cula > 0 else float("inf") - - r = { - "B": B, - "T": T, - "relative_rms_error": relative_rms_error, - "rel_max": rel_max, - "mean_diff": mean_diff, - "ms_fla": ms_fla, - "ms_cula": ms_cula, - "speedup": speedup, - } - results.append(r) - # print(f" B={B:2d} T={T:5d} done ({speedup:.2f}x)") - - del o_fla, o_cula, q, k, v, g, beta, A_log, dt_bias, inputs - torch.cuda.empty_cache() - - return results - - -# ============================================================ -# Varlen benchmark -# ============================================================ -def bench_varlen(configs): - print("\n" + "=" * 100) - print(f" Varlen Benchmark: cuLA CuTe DSL vs FLA Triton disable_recompute={DISABLE_RECOMPUTE}") - print("=" * 100) - results = [] - - for seq_lens, total_len, dist in configs: - set_seed(SEED) - device = torch.device("cuda") - torch.cuda.empty_cache() - - T = total_len - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - - inputs = prepare_safe_gate_inputs(1, T, H, D, device, cu_seqlens=cu_seqlens, num_v_heads=HV) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, init_state, lower_bound = inputs["scale"], inputs["init_state"], inputs["lower_bound"] - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=init_state, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - ) - - # Accuracy - o_fla, _ = run_kda(**common, fn=fla_chunk_kda) - o_cula, _ = run_kda(**common, fn=cula_chunk_kda) - torch.cuda.synchronize() - - relative_rms_error, rel_max, mean_diff = relative_rms_error_rel_max_mean_abs(o_fla, o_cula) - - # Performance - def fn_fla(**common_kw): - return lambda: run_kda(**common_kw, fn=fla_chunk_kda) - - def fn_cula(**common_kw): - return lambda: run_kda(**common_kw, fn=cula_chunk_kda) - - ms_fla = benchmark_cuda_mode_fn( - fn_fla(**common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - ms_cula = benchmark_cuda_mode_fn( - fn_cula(**common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - speedup = ms_fla / ms_cula if ms_cula > 0 else float("inf") - - n_seqs = len(seq_lens) - min_l, max_l = min(seq_lens), max(seq_lens) - avg_l = T // n_seqs - tag = f"{dist:>7s} {n_seqs:>2d}seqs T={T} [{min_l}..{max_l}] avg={avg_l}" - - r = { - "tag": tag, - "dist": dist, - "T_total": T, - "n_seqs": n_seqs, - "relative_rms_error": relative_rms_error, - "rel_max": rel_max, - "mean_diff": mean_diff, - "ms_fla": ms_fla, - "ms_cula": ms_cula, - "speedup": speedup, - } - results.append(r) - # print(f" {tag:45s} done ({speedup:.2f}x)") - - del o_fla, o_cula, q, k, v, g, beta, A_log, dt_bias, inputs - torch.cuda.empty_cache() - - return results - - -# ============================================================ -# Report -# ============================================================ -def print_report(fixed_results, varlen_results): - sep = "=" * 110 - print(f"\n\n{sep}") - print(" BENCHMARK REPORT: chunk_kda") - print(" cuLA CuTe DSL vs FLA Triton") - print(f" H={H} D={D} dtype=bf16 safe_gate=True disable_recompute={DISABLE_RECOMPUTE}") - wu = 1 if (NCU_MODE or SANITIZER_MODE) else WARMUP - ni = 1 if (NCU_MODE or SANITIZER_MODE) else N_ITERS - mode_tag = " [NCU mode]" if NCU_MODE else (" [Sanitizer mode]" if SANITIZER_MODE else "") - print(f" Warmup={wu} Iters={ni}{mode_tag}") - print(sep) - - if fixed_results: - print("\n [Fixed-Length]") - print(f" {'─' * 85}") - print( - f" {'B':>3s} {'T':>5s} │ {'rel_rmse':>10s} {'rel_max':>10s}" - f" │ {'FLA(ms)':>9s} {'cuLA(ms)':>11s} {'Speedup':>8s}" - ) - print(f" {'─' * 85}") - for r in fixed_results: - print( - f" {r['B']:3d} {r['T']:5d} │ " - f"{r['relative_rms_error']:10.6f} {r['rel_max']:10.6f} │ " - f"{r['ms_fla']:9.4f} {r['ms_cula']:11.4f} {r['speedup']:7.2f}x" - ) - print(f" {'─' * 85}") - - if varlen_results: - print("\n [Varlen]") - print(f" {'─' * 100}") - print( - f" {'Config':>45s} │ {'rel_rmse':>10s} {'rel_max':>10s} │ {'FLA(ms)':>9s} {'cuLA(ms)':>11s} {'Speedup':>8s}" - ) - print(f" {'─' * 100}") - for r in varlen_results: - print( - f" {r['tag']:>45s} │ " - f"{r['relative_rms_error']:10.6f} {r['rel_max']:10.6f} │ " - f"{r['ms_fla']:9.4f} {r['ms_cula']:11.4f} {r['speedup']:7.2f}x" - ) - print(f" {'─' * 100}") - - print(f"\n{sep}\n") - - -# ============================================================ -# Main -# ============================================================ -def main(): - parser = argparse.ArgumentParser(description="bench_kda: cuLA CuTe DSL vs FLA Triton baseline") - parser.add_argument( - "--mode", - type=str, - default="both", - choices=["fixed", "varlen", "both"], - help="Which benchmark mode to run (default: both)", - ) - parser.add_argument( - "--ncu", - action="store_true", - help="NCU profiling mode: warmup=1, iters=1", - ) - parser.add_argument( - "--sanitizer", - action="store_true", - help="Sanitizer mode: warmup=1, iters=1 (avoid Triton memory leak under compute-sanitizer)", - ) - parser.add_argument( - "--disable_recompute", - action="store_true", - help="Disable recompute in both FLA and cuLA (pre-compute QG)", - ) - global H - parser.add_argument( - "--heads", - type=int, - default=H, - help=f"Number of Q/K heads (H). Default: {H}", - ) - parser.add_argument( - "--hv", - type=int, - default=None, - help="Number of V heads (HV). Default: same as H (no GVA). Set HV > H to run in GVA mode.", - ) - args = parser.parse_args() - - global NCU_MODE, SANITIZER_MODE, DISABLE_RECOMPUTE - H = args.heads - if args.ncu: - NCU_MODE = True - print("[NCU mode] warmup=1, iters=1") - if args.sanitizer: - SANITIZER_MODE = True - print("[Sanitizer mode] warmup=1, iters=1") - if args.disable_recompute: - DISABLE_RECOMPUTE = True - print("[Disable recompute] pre-compute QG in forward") - HV = H - if args.hv is not None: - if args.hv < H or args.hv % H != 0: - raise ValueError(f"--hv must be a positive multiple of H ({H}), got {args.hv}") - HV = args.hv - if HV > H: - print(f"[GVA] HV={HV} (H={H}, ratio={HV // H}x)") - - fixed_configs = [ - # (B, T) - (1, 512), - (1, 1024), - (1, 4096), - (1, 8192), - (1, 16384), - (2, 512), - (2, 1024), - (2, 4096), - (2, 8192), - (2, 16384), - ] - - varlen_configs = build_varlen_configs( - num_seqs_list=(10, 20), - total_lens=(4096, 8192, 16384), - dists=("uniform", "random", "skewed"), - ) - - fixed_res, varlen_res = [], [] - - if not (args.ncu or args.sanitizer): - check_determinism(H=H, iters=10000) - - if args.mode in ("fixed", "both"): - fixed_res = bench_fixed(fixed_configs) - - if args.mode in ("varlen", "both"): - varlen_res = bench_varlen(varlen_configs) - - print_report(fixed_res, varlen_res) - return fixed_res, varlen_res - - -if __name__ == "__main__": - main() diff --git a/benchmarks/bench_kda_fwd_bwd_e2e.py b/benchmarks/bench_kda_fwd_bwd_e2e.py deleted file mode 100644 index 8a63c5eb..00000000 --- a/benchmarks/bench_kda_fwd_bwd_e2e.py +++ /dev/null @@ -1,640 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -bench_kda_fwd_bwd_e2e.py — Benchmark: cuLA CuTe DSL vs FLA Triton baseline - for chunk_kda forward + backward (end-to-end) - -Compares: - - Accuracy: relative_rms_error, relative max diff between cuLA and FLA outputs & gradients - - Performance: kernel execution time (ms) with CUDA events - -Modes: - - Fixed-length: B=1, B=2 with various T - - Varlen: ~20 seqs with 2-3x length variation - -Phases: - - forward: forward pass only - - e2e: forward + backward (end-to-end) - -Usage: - python bench_kda_fwd_bwd_e2e.py [--mode fixed|varlen|both] [--phase forward|e2e] [--ncu] - -With --ncu, warmup=1 and iters=1 for ncu profiling: - ncu --set full -o report python bench_kda_fwd_bwd_e2e.py --mode varlen --ncu -""" - -import argparse -import os -import pathlib -import sys - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) -os.environ.setdefault("FLA_USE_FAST_OPS", os.getenv("CULA_USE_FAST_MATH", "1")) - -import torch -from fla.ops.kda import chunk_kda as fla_chunk_kda - -from benchmarks.utils import ( - SEED, - benchmark_cuda_mode_fn, - build_varlen_configs, - exclusive_cumsum, - generate_random_seq_lens, - prepare_safe_gate_inputs, - relative_rms_error_rel_max_mean_abs, - set_seed, -) -from cula.kda import chunk_kda as cula_chunk_kda - -# ============================================================ -# Constants -# ============================================================ -H, D = 64, 128 -HV = H # Number of V heads (GVA: HV > H, HV % H == 0) -WARMUP = 25 -N_ITERS = 100 -NCU_MODE = False -SANITIZER_MODE = False -DISABLE_RECOMPUTE = False -PHASE = "e2e" # "forward" or "e2e" - - -# ============================================================ -# Helpers -# ============================================================ - - -def run_kda_e2e(q, k, v, g, beta, scale, A_log, dt_bias, init_state, cu_seqlens, lower_bound, do, dht, fn): - """Run KDA forward (+ backward if PHASE == 'e2e'). - - Clears gradients, runs forward, optionally backward. - """ - q.grad = None - k.grad = None - v.grad = None - g.grad = None - beta.grad = None - init_state.grad = None - - out, ht = fn( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - initial_state=init_state, - output_final_state=True, - use_qk_l2norm_in_kernel=True, - cu_seqlens=cu_seqlens, - use_gate_in_kernel=True, - safe_gate=True, - lower_bound=lower_bound, - disable_recompute=DISABLE_RECOMPUTE, - ) - if PHASE == "e2e": - out.backward(do) - return out, ht - - -def run_kda_e2e_with_grads(q, k, v, g, beta, scale, A_log, dt_bias, init_state, cu_seqlens, lower_bound, do, dht, fn): - """Run KDA forward + backward and return outputs + gradients for accuracy check.""" - q_c = q.detach().clone().requires_grad_(True) - k_c = k.detach().clone().requires_grad_(True) - v_c = v.detach().clone().requires_grad_(True) - g_c = g.detach().clone().requires_grad_(True) - b_c = beta.detach().clone().requires_grad_(True) - h_c = init_state.detach().clone().requires_grad_(True) - - out, ht = fn( - q=q_c, - k=k_c, - v=v_c, - g=g_c, - beta=b_c, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - initial_state=h_c, - output_final_state=True, - use_qk_l2norm_in_kernel=True, - cu_seqlens=cu_seqlens, - use_gate_in_kernel=True, - safe_gate=True, - lower_bound=lower_bound, - disable_recompute=DISABLE_RECOMPUTE, - ) - loss = (out * do).sum() + (ht * dht).sum() - loss.backward() - - return dict( - o=out, - ht=ht, - dq=q_c.grad, - dk=k_c.grad, - dv=v_c.grad, - dg=g_c.grad, - dbeta=b_c.grad, - dh0=h_c.grad, - ) - - -# ============================================================ -# Determinism check -# ============================================================ -def check_determinism(num_seqs=5, T=512, iters=20): - """Verify that cuLA chunk_kda produces identical outputs across repeated runs.""" - device = torch.device("cuda") - set_seed(SEED) - - seq_lens = generate_random_seq_lens(num_seqs, T, 63, seed=SEED) - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - - inputs = prepare_safe_gate_inputs(1, T, H, D, device, cu_seqlens=cu_seqlens, has_init_state=True, num_v_heads=HV) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, init_state, lower_bound = inputs["scale"], inputs["init_state"], inputs["lower_bound"] - - set_seed(SEED + 1) - do = torch.randn_like(v) - dht = torch.randn_like(init_state) - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=init_state, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - do=do, - dht=dht, - ) - - ref = run_kda_e2e_with_grads(**common, fn=cula_chunk_kda) - for i in range(iters): - out = run_kda_e2e_with_grads(**common, fn=cula_chunk_kda) - for name in ("o", "ht", "dq", "dk", "dv", "dg", "dbeta", "dh0"): - assert torch.isnan(out[name]).sum() == 0, f"[determinism] cuLA {name} has NaNs at iter {i}" - assert torch.isfinite(out[name]).all(), f"[determinism] cuLA {name} has infs at iter {i}" - assert torch.equal(out[name], ref[name]), f"[determinism] cuLA {name} mismatch at iter {i}" - return True - - -# ============================================================ -# Fixed-length benchmark -# ============================================================ -def bench_fixed(configs): - print("\n" + "=" * 120) - print(f" Fixed-Length E2E Benchmark: cuLA vs FLA phase={PHASE} disable_recompute={DISABLE_RECOMPUTE}") - print("=" * 120) - results = [] - - for B, T in configs: - set_seed(SEED) - device = torch.device("cuda") - torch.cuda.empty_cache() - - seq_lens = [T] * B - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - - inputs = prepare_safe_gate_inputs(B, T, H, D, device, cu_seqlens=cu_seqlens, has_init_state=True, num_v_heads=HV) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, init_state, lower_bound = inputs["scale"], inputs["init_state"], inputs["lower_bound"] - - # Generate do, dht for backward - set_seed(SEED + 1) - do = torch.randn_like(v) - dht = torch.randn_like(init_state) - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=init_state, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - do=do, - dht=dht, - ) - - # Accuracy: compare outputs and gradients - acc = {} - if PHASE == "e2e": - fla_results = run_kda_e2e_with_grads(**common, fn=fla_chunk_kda) - cula_results = run_kda_e2e_with_grads(**common, fn=cula_chunk_kda) - torch.cuda.synchronize() - - for name in ("o", "ht", "dq", "dk", "dv", "dg", "dbeta", "dh0"): - relative_rms_error, rel_max, mean_diff = relative_rms_error_rel_max_mean_abs( - fla_results[name], cula_results[name] - ) - acc[name] = {"relative_rms_error": relative_rms_error, "rel_max": rel_max, "mean_diff": mean_diff} - else: - # forward-only accuracy - o_fla, ht_fla = run_kda_e2e(**common, fn=fla_chunk_kda) - o_cula, ht_cula = run_kda_e2e(**common, fn=cula_chunk_kda) - torch.cuda.synchronize() - for name, ref, out in [("o", o_fla, o_cula), ("ht", ht_fla, ht_cula)]: - relative_rms_error, rel_max, mean_diff = relative_rms_error_rel_max_mean_abs(ref, out) - acc[name] = {"relative_rms_error": relative_rms_error, "rel_max": rel_max, "mean_diff": mean_diff} - - # For timing, use leaf tensors with requires_grad - q_t = q.detach().clone().requires_grad_(True) - k_t = k.detach().clone().requires_grad_(True) - v_t = v.detach().clone().requires_grad_(True) - g_t = g.detach().clone().requires_grad_(True) - beta_t = beta.detach().clone().requires_grad_(True) - h0_t = init_state.detach().clone().requires_grad_(True) - - timing_common = dict( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=h0_t, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - do=do, - dht=dht, - ) - - def fn_fla(**kw): - return lambda: run_kda_e2e(**kw, fn=fla_chunk_kda) - - def fn_cula(**kw): - return lambda: run_kda_e2e(**kw, fn=cula_chunk_kda) - - ms_fla = benchmark_cuda_mode_fn( - fn_fla(**timing_common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - ms_cula = benchmark_cuda_mode_fn( - fn_cula(**timing_common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - speedup = ms_fla / ms_cula if ms_cula > 0 else float("inf") - - r = { - "B": B, - "T": T, - "accuracy": acc, - "ms_fla": ms_fla, - "ms_cula": ms_cula, - "speedup": speedup, - } - results.append(r) - - del q, k, v, g, beta, A_log, dt_bias, inputs, do, dht - torch.cuda.empty_cache() - - return results - - -# ============================================================ -# Varlen benchmark -# ============================================================ -def bench_varlen(configs): - print("\n" + "=" * 120) - print(f" Varlen E2E Benchmark: cuLA vs FLA phase={PHASE} disable_recompute={DISABLE_RECOMPUTE}") - print("=" * 120) - results = [] - - for seq_lens, total_len, dist in configs: - set_seed(SEED) - device = torch.device("cuda") - torch.cuda.empty_cache() - - T = total_len - cu_seqlens = torch.tensor(exclusive_cumsum(seq_lens), dtype=torch.int32, device=device) - - inputs = prepare_safe_gate_inputs(1, T, H, D, device, cu_seqlens=cu_seqlens, has_init_state=True, num_v_heads=HV) - q, k, v, g, beta = inputs["q"], inputs["k"], inputs["v"], inputs["g"], inputs["beta"] - A_log, dt_bias = inputs["A_log"], inputs["dt_bias"] - scale, init_state, lower_bound = inputs["scale"], inputs["init_state"], inputs["lower_bound"] - - # Generate do, dht for backward - set_seed(SEED + 1) - do = torch.randn_like(v) - dht = torch.randn_like(init_state) - - common = dict( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=init_state, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - do=do, - dht=dht, - ) - - # Accuracy: compare outputs and gradients - acc = {} - if PHASE == "e2e": - fla_results = run_kda_e2e_with_grads(**common, fn=fla_chunk_kda) - cula_results = run_kda_e2e_with_grads(**common, fn=cula_chunk_kda) - torch.cuda.synchronize() - - for name in ("o", "ht", "dq", "dk", "dv", "dg", "dbeta", "dh0"): - relative_rms_error, rel_max, mean_diff = relative_rms_error_rel_max_mean_abs( - fla_results[name], cula_results[name] - ) - acc[name] = {"relative_rms_error": relative_rms_error, "rel_max": rel_max, "mean_diff": mean_diff} - else: - o_fla, ht_fla = run_kda_e2e(**common, fn=fla_chunk_kda) - o_cula, ht_cula = run_kda_e2e(**common, fn=cula_chunk_kda) - torch.cuda.synchronize() - for name, ref, out in [("o", o_fla, o_cula), ("ht", ht_fla, ht_cula)]: - relative_rms_error, rel_max, mean_diff = relative_rms_error_rel_max_mean_abs(ref, out) - acc[name] = {"relative_rms_error": relative_rms_error, "rel_max": rel_max, "mean_diff": mean_diff} - - # For timing, use leaf tensors with requires_grad - q_t = q.detach().clone().requires_grad_(True) - k_t = k.detach().clone().requires_grad_(True) - v_t = v.detach().clone().requires_grad_(True) - g_t = g.detach().clone().requires_grad_(True) - beta_t = beta.detach().clone().requires_grad_(True) - h0_t = init_state.detach().clone().requires_grad_(True) - - timing_common = dict( - q=q_t, - k=k_t, - v=v_t, - g=g_t, - beta=beta_t, - scale=scale, - A_log=A_log, - dt_bias=dt_bias, - init_state=h0_t, - cu_seqlens=cu_seqlens, - lower_bound=lower_bound, - do=do, - dht=dht, - ) - - def fn_fla(**kw): - return lambda: run_kda_e2e(**kw, fn=fla_chunk_kda) - - def fn_cula(**kw): - return lambda: run_kda_e2e(**kw, fn=cula_chunk_kda) - - ms_fla = benchmark_cuda_mode_fn( - fn_fla(**timing_common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - ms_cula = benchmark_cuda_mode_fn( - fn_cula(**timing_common), - default_warmup=WARMUP, - default_rep=N_ITERS, - ncu_mode=NCU_MODE, - sanitizer_mode=SANITIZER_MODE, - ) - speedup = ms_fla / ms_cula if ms_cula > 0 else float("inf") - - n_seqs = len(seq_lens) - min_l, max_l = min(seq_lens), max(seq_lens) - avg_l = T // n_seqs - tag = f"{dist:>7s} {n_seqs:>2d}seqs T={T} [{min_l}..{max_l}] avg={avg_l}" - - r = { - "tag": tag, - "dist": dist, - "T_total": T, - "n_seqs": n_seqs, - "accuracy": acc, - "ms_fla": ms_fla, - "ms_cula": ms_cula, - "speedup": speedup, - } - results.append(r) - - del q, k, v, g, beta, A_log, dt_bias, inputs, do, dht - torch.cuda.empty_cache() - - return results - - -# ============================================================ -# Report -# ============================================================ -def print_report(fixed_results, varlen_results): - sep = "=" * 130 - print(f"\n\n{sep}") - print(" BENCHMARK REPORT: chunk_kda forward+backward (E2E)") - print(" cuLA CuTe DSL vs FLA Triton") - print( - f" H={H} HV={HV} D={D} dtype=bf16 safe_gate=True phase={PHASE} disable_recompute={DISABLE_RECOMPUTE}" - ) - wu = 1 if (NCU_MODE or SANITIZER_MODE) else WARMUP - ni = 1 if (NCU_MODE or SANITIZER_MODE) else N_ITERS - mode_tag = " [NCU mode]" if NCU_MODE else (" [Sanitizer mode]" if SANITIZER_MODE else "") - print(f" Warmup={wu} Iters={ni}{mode_tag}") - print(sep) - - # Determine which accuracy keys to show - if PHASE == "e2e": - acc_keys = ["o", "ht", "dq", "dk", "dv", "dg", "dbeta", "dh0"] - else: - acc_keys = ["o", "ht"] - - acc_header = " ".join(f"{k:>10s}" for k in acc_keys) - - if fixed_results: - print("\n [Fixed-Length]") - print(f" {'─' * 125}") - - # Header - print(f" {'B':>3s} {'T':>5s} │ {'FLA(ms)':>9s} {'cuLA(ms)':>11s} {'Speedup':>8s} │ {'':>10s}{acc_header}") - print(f" {'─' * 125}") - - for r in fixed_results: - rel_max_vals = " ".join(f"{r['accuracy'].get(k, {}).get('rel_max', 0.0):10.6f}" for k in acc_keys) - relative_rms_error_vals = " ".join( - f"{r['accuracy'].get(k, {}).get('relative_rms_error', 0.0):10.6f}" for k in acc_keys - ) - # Line 1: timing + rel_max - print( - f" {r['B']:3d} {r['T']:5d} │ " - f"{r['ms_fla']:9.4f} {r['ms_cula']:11.4f} {r['speedup']:7.2f}x │ " - f"{'rel_max:':>10s}{rel_max_vals}" - ) - # Line 2: rel_rmse (no timing columns) - print(f" {'':3s} {'':5s} │ {'':9s} {'':11s} {'':8s} │ {'rel_rmse:':>10s}{relative_rms_error_vals}") - print(f" {'─' * 125}") - - if varlen_results: - print("\n [Varlen]") - print(f" {'─' * 140}") - - print(f" {'Config':>45s} │ {'FLA(ms)':>9s} {'cuLA(ms)':>11s} {'Speedup':>8s} │ {'':>10s}{acc_header}") - print(f" {'─' * 140}") - - for r in varlen_results: - rel_max_vals = " ".join(f"{r['accuracy'].get(k, {}).get('rel_max', 0.0):10.6f}" for k in acc_keys) - relative_rms_error_vals = " ".join( - f"{r['accuracy'].get(k, {}).get('relative_rms_error', 0.0):10.6f}" for k in acc_keys - ) - # Line 1: timing + rel_max - print( - f" {r['tag']:>45s} │ " - f"{r['ms_fla']:9.4f} {r['ms_cula']:11.4f} {r['speedup']:7.2f}x │ " - f"{'rel_max:':>10s}{rel_max_vals}" - ) - # Line 2: rel_rmse (no config/timing columns) - print(f" {'':>45s} │ {'':9s} {'':11s} {'':8s} │ {'rel_rmse:':>10s}{relative_rms_error_vals}") - print(f" {'─' * 140}") - - print(f"\n{sep}\n") - - -# ============================================================ -# Main -# ============================================================ -def main(): - parser = argparse.ArgumentParser(description="bench_kda_fwd_bwd_e2e: cuLA vs FLA (forward + backward)") - parser.add_argument( - "--mode", - type=str, - default="both", - choices=["fixed", "varlen", "both"], - help="Which benchmark mode to run (default: both)", - ) - parser.add_argument( - "--phase", - type=str, - default="e2e", - choices=["forward", "e2e"], - help="Benchmark phase: forward only or end-to-end (default: e2e)", - ) - parser.add_argument( - "--ncu", - action="store_true", - help="NCU profiling mode: warmup=1, iters=1", - ) - parser.add_argument( - "--sanitizer", - action="store_true", - help="Sanitizer mode: warmup=1, iters=1 (avoid Triton memory leak under compute-sanitizer)", - ) - parser.add_argument( - "--disable_recompute", - action="store_true", - help="Disable recompute in both FLA and cuLA (pre-compute QG)", - ) - global H - parser.add_argument( - "--heads", - type=int, - default=H, - help=f"Number of Q/K heads (H). Default: {H}", - ) - parser.add_argument( - "--hv", - type=int, - default=None, - help="Number of V heads (HV). Default: same as --heads. For GVA, set HV > H with HV %% H == 0", - ) - args = parser.parse_args() - - global NCU_MODE, SANITIZER_MODE, DISABLE_RECOMPUTE, PHASE, HV - H = args.heads - HV = args.hv if args.hv is not None else H - if HV < H or HV % H != 0: - raise ValueError(f"--hv ({HV}) must be >= --heads ({H}) and divisible by --heads") - if args.ncu: - NCU_MODE = True - print("[NCU mode] warmup=1, iters=1") - if args.sanitizer: - SANITIZER_MODE = True - print("[Sanitizer mode] warmup=1, iters=1") - if args.disable_recompute: - DISABLE_RECOMPUTE = True - print("[Disable recompute] pre-compute QG in forward") - PHASE = args.phase - - if not (args.ncu or args.sanitizer): - det_configs = [(5, 1024), (10, 4096), (10, 8192), (10, 16384)] - print("\n[Determinism Check] cuLA chunk_kda E2E ...") - for num_seqs, T in det_configs: - result = check_determinism(num_seqs=num_seqs, T=T, iters=1000) - print(f" num_seqs={num_seqs} T={T:5d} {'PASS' if result else 'FAIL'}") - print("[Determinism Check] All passed.\n") - - fixed_configs = [ - # (B, T) - (1, 512), - (1, 1024), - (1, 4096), - (1, 8192), - (1, 16384), - (2, 512), - (2, 1024), - (2, 4096), - (2, 8192), - (2, 16384), - ] - - varlen_configs = build_varlen_configs( - num_seqs_list=(10, 20), - total_lens=(4096, 8192, 16384), - dists=("uniform", "random", "skewed"), - ) - - fixed_res, varlen_res = [], [] - - if args.mode in ("fixed", "both"): - fixed_res = bench_fixed(fixed_configs) - - if args.mode in ("varlen", "both"): - varlen_res = bench_varlen(varlen_configs) - - print_report(fixed_res, varlen_res) - return fixed_res, varlen_res - - -if __name__ == "__main__": - main() diff --git a/cula/kda/README.md b/cula/kda/README.md index f326e96b..2f908cf9 100644 --- a/cula/kda/README.md +++ b/cula/kda/README.md @@ -6,15 +6,17 @@ Public entry points for KDA. For full examples (varlen, intracard CP, per-backen | Symbol | Description | Arch | Direction | |--------|-------------|------|-----------| -| `chunk_kda` | Chunked KDA prefill | SM100 (Blackwell) | Forward + backward | +| `chunk_kda_fwd_intra` | Atomic intra + W/U composition from cumulative gates | SM100 (Blackwell) | Forward primitives | | `flashkda_prefill` | KDA prefill (CuTeDSL K1+K2, intracard-CP capable) | SM90 (Hopper) | Forward only | | `kda_prefill` | Public FlashKDA prefill entry point | SM90/SM100 | Forward only | | `kda_decode` | Single-token decode | SM100 | Forward | | `fused_sigmoid_gating_delta_rule_update` | Decode state update | SM100 | Forward | -`chunk_kda` and `flashkda_prefill` each call a fixed implementation. +`chunk_kda_fwd_intra` and `flashkda_prefill` each call owned CuTeDSL implementations. `kda_prefill` calls FlashKDA directly; there is no C++/CUDA extension fallback. FlashKDA currently requires matching Q/K/V head counts (MHA rather than GVA). +The standalone package does not provide the former FLA-backed `chunk_kda` +autograd composition. ## Minimal example diff --git a/cula/kda/__init__.py b/cula/kda/__init__.py index 0cc27eec..91cc4417 100644 --- a/cula/kda/__init__.py +++ b/cula/kda/__init__.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Public KDA API exports for chunk, prefill, and decode""" +"""Public standalone KDA APIs.""" __all__ = [ - "chunk_kda", + "chunk_kda_fwd_intra", "kda_decode", "kda_decode_mtp", "kda_decode_mtp_recurrent", @@ -31,7 +31,7 @@ ] _LAZY = { - "chunk_kda": ("cula.kda.chunk", "chunk_kda"), + "chunk_kda_fwd_intra": ("cula.kda.chunk_intra", "chunk_kda_fwd_intra"), "flashkda_prefill": ("cula.kda.flashkda", "cula_kda_prefill"), "kda_prefill": ("cula.kda.prefill", "kda_prefill"), "kda_decode": ("cula.ops.kda.decode.cute", "kda_decode"), diff --git a/cula/kda/chunk.py b/cula/kda/chunk.py deleted file mode 100644 index af2ee9da..00000000 --- a/cula/kda/chunk.py +++ /dev/null @@ -1,438 +0,0 @@ -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang -# Related files are modified and supported by the Moonshot AI Team - -"""SM100 modular chunk KDA public API and autograd wrapper""" - -from typing import Literal - -import torch -from fla.modules.l2norm import l2norm_bwd, l2norm_fwd -from fla.ops.cp import FLACPContext -from fla.ops.utils.index import prepare_chunk_indices -from fla.utils import autocast_custom_bwd, autocast_custom_fwd, input_guard - -from cula.kda.chunk_bwd import chunk_kda_bwd -from cula.kda.chunk_fwd import chunk_kda_fwd -from cula.ops.kda.cp_mode import CPMode - - -class ChunkKDAFunction(torch.autograd.Function): - @staticmethod - @input_guard - @autocast_custom_fwd - def forward( - ctx, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - A_log: torch.Tensor, - dt_bias: torch.Tensor, - scale: float, - initial_state: torch.Tensor, - output_final_state: bool = False, - use_qk_l2norm_in_kernel: bool = False, - use_gate_in_kernel: bool = False, - cu_seqlens: torch.IntTensor | None = None, - cu_seqlens_cpu: torch.IntTensor | None = None, - safe_gate: bool = False, - lower_bound: float | None = None, - disable_recompute: bool = False, - return_intermediate_states: bool = False, - cp_context: FLACPContext | None = None, - use_intracard_cp: CPMode | None = None, - ): - chunk_size = 64 - - # Apply l2norm - q_rstd, k_rstd = None, None - if use_qk_l2norm_in_kernel: - q, q_rstd = l2norm_fwd(q) - k, k_rstd = l2norm_fwd(k) - - chunk_indices = ( - prepare_chunk_indices(cu_seqlens, chunk_size, cu_seqlens_cpu=cu_seqlens_cpu) if cu_seqlens is not None else None - ) - - g_org = g if use_gate_in_kernel else None - - (o, final_state, g, Aqk, Akk, w, u, qg, kg, v_new, h, initial_state) = chunk_kda_fwd( - q=q, - k=k, - v=v, - g=g, - beta=beta, - scale=scale, - initial_state=initial_state, - output_final_state=output_final_state, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens_cpu, - chunk_indices=chunk_indices, - safe_gate=safe_gate, - lower_bound=lower_bound, - use_gate_in_kernel=use_gate_in_kernel, - A_log=A_log, - dt_bias=dt_bias, - disable_recompute=disable_recompute, - return_intermediate_states=return_intermediate_states, - cp_context=cp_context, - use_intracard_cp=use_intracard_cp, - ) - - if return_intermediate_states: - assert torch.is_inference_mode_enabled(), "return_intermediate_states is only allowed in inference mode" - assert disable_recompute is False, "return_intermediate_states must be used with disable_recompute=False" - return o.type_as(q), final_state, h - - ctx.save_for_backward( - q, - q_rstd, - k, - k_rstd, - v, - g, - g_org, - beta, - A_log, - dt_bias, - Aqk, - Akk, - w, - u, - qg, - kg, - v_new, - h, - initial_state, - cu_seqlens, - chunk_indices, - ) - ctx.chunk_size = chunk_size - ctx.safe_gate = safe_gate - ctx.scale = scale - ctx.lower_bound = lower_bound - ctx.use_qk_l2norm_in_kernel = use_qk_l2norm_in_kernel - ctx.use_gate_in_kernel = use_gate_in_kernel - ctx.disable_recompute = disable_recompute - ctx.cp_context = cp_context - return o.type_as(q), final_state - - @staticmethod - @input_guard - @autocast_custom_bwd - def backward( - ctx, - do: torch.Tensor, - dht: torch.Tensor, - ): - ( - q, - q_rstd, - k, - k_rstd, - v, - g, - g_org, - beta, - A_log, - dt_bias, - Aqk, - Akk, - w, - u, - qg, - kg, - v_new, - h, - initial_state, - cu_seqlens, - chunk_indices, - ) = ctx.saved_tensors - - dq, dk, dv, db, dg, dh0, dA, dbias = chunk_kda_bwd( - q=q, - k=k, - v=v, - g=g, - beta=beta, - Aqk=Aqk, - Akk=Akk, - scale=ctx.scale, - initial_state=initial_state, - do=do, - dht=dht, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - chunk_size=ctx.chunk_size, - safe_gate=ctx.safe_gate, - g_org=g_org, - lower_bound=ctx.lower_bound, - use_gate_in_kernel=ctx.use_gate_in_kernel, - A_log=A_log, - dt_bias=dt_bias, - disable_recompute=ctx.disable_recompute, - w=w, - u=u, - qg=qg, - kg=kg, - v_new=v_new, - h=h, - cp_context=ctx.cp_context, - ) - if ctx.use_qk_l2norm_in_kernel: - dq = l2norm_bwd(q, q_rstd, dq) - dk = l2norm_bwd(k, k_rstd, dk) - - return ( - dq.to(q), - dk.to(k), - dv.to(v), - dg.to(g), - db.to(beta), - dA, - dbias, - None, - dh0, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ) - - -@torch.compiler.disable -def chunk_kda( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - scale: float | None = None, - initial_state: torch.Tensor | None = None, - output_final_state: bool = False, - use_qk_l2norm_in_kernel: bool = False, - use_gate_in_kernel: bool = False, - cu_seqlens: torch.IntTensor | None = None, - cu_seqlens_cpu: torch.IntTensor | None = None, - safe_gate: bool = False, - lower_bound: float | None = None, - disable_recompute: bool = False, - return_intermediate_states: bool = False, - cp_context: FLACPContext = None, - use_intracard_cp: Literal["auto"] | bool | None = None, - **kwargs, -): - r""" - Args: - q (torch.Tensor): - queries of shape `[B, T, H, K]`. - k (torch.Tensor): - keys of shape `[B, T, H, K]`. - v (torch.Tensor): - values of shape `[B, T, H, V]`. - g (torch.Tensor): - (forget) gating tensor (in log space!) of shape `[B, T, H, K]`. - beta (torch.Tensor): - betas of shape `[B, T, H]`. - scale (Optional[float]): - Scale factor for the KDA attention scores. - If not provided, it will default to `1 / sqrt(K)`. Default: `None`. - initial_state (Optional[torch.Tensor]): - Initial state of shape `[N, H, K, V]` for `N` input sequences. - For equal-length input sequences, `N` equals the batch size `B`. - Default: `None`. - output_final_state (Optional[bool]): - Whether to output the final state of shape `[N, H, K, V]`. Default: `False`. - use_qk_l2norm_in_kernel (bool): - Whether to apply L2norm to the q,k tensor internally. Default: `False`. - use_gate_in_kernel (bool): - Whether to compute the log-space KDA decay internally. - - If `True`: - The passed `g` acts as the raw input for `-exp(A_log).view(H, -1) * softplus(g + dt_bias.view(H, K))`. - Note that as part of the input arguments, - `A_log` (shape `[H]`) and the optional `dt_bias` (shape `[H * K]`) should be provided. - - If `False`, `g` is expected to be the pre-computed decay value. - Default: `False`. - cu_seqlens (torch.IntTensor): - Cumulative sequence lengths of shape `[N+1]` used for variable-length training, - consistent with the FlashAttention API. Must be int32. - cu_seqlens_cpu (torch.IntTensor): - Cumulative sequence lengths of shape `[N+1]` used for variable-length training, - consistent with the FlashAttention API. Must be int32. - safe_gate (bool): - Whether the kernel can assume the input gate values `g` are in a safe range. - When `True`, the kernel can use M=16 TensorCore acceleration. - The safe range is approximately [-5, 0). Default: `False`. - lower_bound (Optional[float]): - Lower bound for the forget gate activation function when `use_gate_in_kernel=True`. - This parameter modifies the internal forget gate activation and is recommended - to be set to `-5` when `safe_gate` is enabled. Default: `None`. - disable_recompute (bool): - Whether to disable gradient recomputation in the kernel. When `True`, the kernel - will save all intermediate activations for backward pass, which is beneficial - for training small models at the cost of increased memory usage. Default: `False`. - return_intermediate_states (bool): - If True, returns intermediate state `h` for inference scenarios (e.g., vLLM). - Must be used within `torch.inference_mode()` and will return a 3-tuple instead of 2-tuple. - This is not intended for training as it bypasses autograd. Default: `False`. - - Returns: - - Normal mode (return_intermediate_states=False): A tuple (o, final_state) - o (torch.Tensor): - Outputs of shape `[B, T, H, V]`. - final_state (torch.Tensor): - Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`. - - Inference mode (return_intermediate_states=True): A tuple (o, final_state, h) - o (torch.Tensor): - Outputs of shape `[B, T, H, V]`. - final_state (torch.Tensor): - Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`. - h (torch.Tensor): - Intermediate states of shape `[B, NT, H, K, V]` and dtype `bfloat16` for caching or further processing. - - For equal-length sequences: `NT = #chunks_per_sequence` (typically `ceil(T / chunk_size)`) - - For variable-length sequences (cu_seqlens): B is always 1 (flattened), NT is the total number of chunks across all sequences, determined by `prepare_chunk_indices(cu_seqlens, chunk_size)` - - Examples:: - >>> import torch - >>> import torch.nn.functional as F - >>> from einops import rearrange - >>> from fla.ops.kda import chunk_kda - # inputs with equal lengths - >>> B, T, H, K, V = 4, 2048, 4, 512, 512 - >>> q = torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda') - >>> k = torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda') - >>> v = torch.randn(B, T, H, V, dtype=torch.bfloat16, device='cuda') - >>> beta = torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda') - >>> g = torch.rand(B, T, H, K, dtype=torch.bfloat16, device='cuda') - >>> h0 = torch.randn(B, H, K, V, dtype=torch.bfloat16, device='cuda') - >>> A_log = torch.randn(H, dtype=torch.float32, device='cuda') - >>> dt_bias = torch.randn(H * K, dtype=torch.float32, device='cuda') - >>> o, ht = chunk_kda( - q, k, v, g, beta, - A_log=A_log, - dt_bias=dt_bias, - use_qk_l2norm_in_kernel=True, - use_gate_in_kernel=True, - initial_state=h0, - output_final_state=True - ) - # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required - >>> q, k, v, beta, g = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, beta, g)) - # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected - >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.int32) - >>> o, ht = chunk_kda( - q, k, v, g, beta, - A_log=A_log, - dt_bias=dt_bias, - use_qk_l2norm_in_kernel=True, - use_gate_in_kernel=True, - initial_state=h0, - output_final_state=True, - cu_seqlens=cu_seqlens - ) - """ - - use_intracard_cp = CPMode.parse(use_intracard_cp) - - if cp_context is not None: - if use_intracard_cp is CPMode.FORCE: - raise ValueError("use_intracard_cp=True cannot be combined with FLA cp_context.") - use_intracard_cp = CPMode.OFF - assert initial_state is None, "Initial state is not supported for CP" - assert output_final_state is False, "Output final state is not supported for CP" - assert cp_context.cu_seqlens is not None, "cu_seqlens is required for CP" - # Override cu_seqlens and cu_seqlens_cpu with the ones from the context - cu_seqlens = cp_context.cu_seqlens - if cp_context.cu_seqlens_cpu is not None: - cu_seqlens_cpu = cp_context.cu_seqlens_cpu - - if return_intermediate_states: - if use_intracard_cp is CPMode.FORCE: - raise ValueError("use_intracard_cp=True is not supported with return_intermediate_states=True.") - use_intracard_cp = CPMode.OFF - - if cu_seqlens is not None: - if q.shape[0] != 1: - raise ValueError( - f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." - f"Please flatten variable-length inputs before processing.", - ) - if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1: - raise ValueError( - f"The number of initial states is expected to be equal to the number of input sequences, " - f"i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}.", - ) - assert cu_seqlens.dtype == torch.int32, "cu_seqlens must be in int32" - if initial_state is not None: - assert initial_state.dtype == torch.float32, "initial_state must be in float32." - - A_log, dt_bias = kwargs.pop("A_log", None), kwargs.pop("dt_bias", None) - if kwargs: - raise TypeError(f"chunk_kda() got unexpected keyword arguments: {sorted(kwargs)}") - if use_gate_in_kernel: - assert A_log is not None, "A_log must be provided when use_gate_in_kernel=True." - - if safe_gate and use_gate_in_kernel: - if lower_bound is None: - raise ValueError("`lower_bound` must be specified when `safe_gate=True` and `use_gate_in_kernel=True`.") - if not (-5 <= lower_bound < 0): - raise ValueError(f"`lower_bound` must be in the safe range [-5, 0), got {lower_bound}.") - - # Validate head dimensions for GVA - B, T, H, K, HV = *q.shape, v.shape[2] - assert q.shape == k.shape, f"q and k must have the same shape, got q={q.shape} vs k={k.shape}" - assert q.dtype == k.dtype == v.dtype == torch.bfloat16, "q, k, v must be in bfloat16." - assert beta.dtype == torch.bfloat16 or beta.dtype == torch.float32, "beta must be in bfloat16 or float32." - assert q.shape[-1] == k.shape[-1] == v.shape[-1] == 128, "Currently we only support head dim of 128 for KDA" - assert HV % H == 0, ( - f"For GVA, num_v_heads (HV={HV}) must be evenly divisible by num_qk_heads (H={H}), but got HV % H = {HV % H}" - ) - assert g.shape == (B, T, HV, K), f"g must have shape [B, T, HV, K]={[B, T, HV, K]}, got {list(g.shape)}" - assert beta.shape == (B, T, HV), f"beta must have shape [B, T, HV]={[B, T, HV]}, got {list(beta.shape)}" - - if scale is None: - scale = k.shape[-1] ** -0.5 - return ChunkKDAFunction.apply( - q, - k, - v, - g, - beta, - A_log, - dt_bias, - scale, - initial_state, - output_final_state, - use_qk_l2norm_in_kernel, - use_gate_in_kernel, - cu_seqlens, - cu_seqlens_cpu, - safe_gate, - lower_bound, - disable_recompute, - return_intermediate_states, - cp_context, - use_intracard_cp, - ) diff --git a/cula/kda/chunk_bwd.py b/cula/kda/chunk_bwd.py deleted file mode 100644 index e07fc959..00000000 --- a/cula/kda/chunk_bwd.py +++ /dev/null @@ -1,720 +0,0 @@ -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang - -"""SM90/SM100 modular chunk KDA backward orchestration.""" - -import functools -import importlib - -import torch -import triton -import triton.language as tl -from fla.ops.common.chunk_delta_h import chunk_gated_delta_rule_bwd_dhu -from fla.ops.common.chunk_delta_h import chunk_gated_delta_rule_fwd_h as _triton_chunk_gated_delta_rule_fwd_h -from fla.ops.cp import FLACPContext -from fla.ops.cp.chunk_delta_h import ( - chunk_gated_delta_rule_bwd_dhu_pre_process, - expand_h0, -) -from fla.ops.kda.chunk_bwd import recompute_w_u_fwd as recompute_w_u_fwd_triton -from fla.ops.kda.gate import kda_gate_bwd, kda_gate_chunk_cumsum -from fla.ops.utils import chunk_local_cumsum, prepare_chunk_indices -from fla.ops.utils.constant import RCP_LN2 -from fla.ops.utils.op import exp2 -from fla.utils import ( - IS_NVIDIA_HOPPER, - autotune_cache_kwargs, - check_shared_mem, -) - -from cula.kda.chunk_intra import chunk_kda_bwd_intra -from cula.ops.kda.sm100.bwd_wy_dqkg import chunk_kda_bwd_wy_dqkg_fused as chunk_kda_bwd_wy_dqkg_fused_cutedsl -from cula.utils import get_device_sm_version - -_delta_h_mod = importlib.import_module("cula.ops.kda.sm100.delta_h") -chunk_gated_delta_rule_fwd_h = _delta_h_mod.chunk_gated_delta_rule_fwd_h - - -@functools.cache -def _get_sm100_cutedsl_recompute_wu(): - return importlib.import_module("cula.ops.kda.sm100.recompute_wu").recompute_w_u_fwd - - -def _sm90_triton_chunk_gated_delta_rule_fwd_h( - k: torch.Tensor, - w: torch.Tensor, - u: torch.Tensor, - g: torch.Tensor | None = None, - gk: torch.Tensor | None = None, - initial_state: torch.Tensor | None = None, - output_final_state: bool = False, - chunk_size: int = 64, - save_new_value: bool = True, - cu_seqlens: torch.Tensor | None = None, - chunk_indices: torch.Tensor | None = None, - persistent: bool = True, - _no_cp: bool = False, - cu_seqlens_cpu: torch.Tensor | None = None, -): - """FLA Triton delta-H adapter for the SM90 backward path.""" - del persistent, _no_cp - return _triton_chunk_gated_delta_rule_fwd_h( - k=k, - w=w, - u=u, - g=g, - gk=gk, - initial_state=initial_state, - output_final_state=output_final_state, - chunk_size=chunk_size, - save_new_value=save_new_value, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens_cpu, - chunk_indices=chunk_indices, - use_exp2=True, - transpose_state_layout=False, - ) - - -def _get_chunk_delta_h_fwd(device: torch.device): - major, minor = get_device_sm_version(device) - if major == 9 and minor == 0: - return _sm90_triton_chunk_gated_delta_rule_fwd_h - if major == 10 and minor in (0, 3): - return chunk_gated_delta_rule_fwd_h - raise RuntimeError( - f"Unsupported CUDA compute capability sm_{major}{minor}. " - "Only sm90a (Hopper) and Blackwell (SM100/SM103) are supported." - ) - - -def _select_recompute_w_u_backend(device: torch.device): - major, minor = get_device_sm_version(device) - if major == 9 and minor == 0: - return "triton" - if major == 10 and minor in (0, 3): - return "cutedsl" - raise RuntimeError( - f"Unsupported CUDA compute capability sm_{major}{minor}. " - "Only sm90a (Hopper) and Blackwell (SM100/SM103) are supported." - ) - - -def _select_chunk_kda_bwd_wy_dqkg_fused(q: torch.Tensor, v: torch.Tensor): - major, minor = get_device_sm_version(q.device) - if major == 9 and minor == 0: - if q.shape[2] == v.shape[2]: - return importlib.import_module("cula.ops.kda.sm90.bwd_wy_dqkg").chunk_kda_bwd_wy_dqkg_fused - # The SM90 CuTe DSL kernel currently supports MHA only. Keep FLA/Triton - # behavior for grouped-value attention until the kernel gains GVA. - return chunk_kda_bwd_wy_dqkg_fused - if major == 10 and minor in (0, 3): - return chunk_kda_bwd_wy_dqkg_fused_cutedsl - raise RuntimeError( - f"Unsupported CUDA compute capability sm_{major}{minor}. " - "Only sm90a (Hopper) and Blackwell (SM100/SM103) are supported." - ) - - -BK_LIST = [32, 64] if check_shared_mem() else [16, 32] -BV_LIST = [64, 128] if check_shared_mem("ampere") else [16, 32] -NUM_WARPS = [2, 4] if IS_NVIDIA_HOPPER else [2, 4, 8] - - -@triton.heuristics( - { - "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, - } -) -@triton.autotune( - configs=[ - triton.Config({}, num_warps=num_warps, num_stages=num_stages) for num_warps in NUM_WARPS for num_stages in [2, 3, 4] - ], - key=["H", "HV", "K", "V", "BT", "BK", "BV"], - **autotune_cache_kwargs, -) -@triton.jit(do_not_specialize=["T"]) -def chunk_kda_bwd_kernel_dAv( - q, - k, - v, - A, - do, - dv, - dA, - cu_seqlens, - chunk_indices, - scale, - T, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BT: tl.constexpr, - BK: tl.constexpr, - BV: tl.constexpr, - IS_VARLEN: tl.constexpr, -): - i_t, i_bh = tl.program_id(0), tl.program_id(1) - i_b, i_hv = i_bh // HV, i_bh % HV - i_h = i_hv // (HV // H) - if IS_VARLEN: - i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) - bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) - T = eos - bos - else: - bos, eos = i_b * T, i_b * T + T - - # offset calculation - q += (bos * H + i_h) * K - k += (bos * H + i_h) * K - v += (bos * HV + i_hv) * V - do += (bos * HV + i_hv) * V - dv += (bos * HV + i_hv) * V - dA += (bos * HV + i_hv) * BT - - p_A = tl.make_block_ptr(A + (bos * HV + i_hv) * BT, (BT, T), (1, HV * BT), (0, i_t * BT), (BT, BT), (0, 1)) - b_A = tl.load(p_A, boundary_check=(0, 1)) - - o_t = i_t * BT + tl.arange(0, BT) - m_t = o_t < T - m_A = (o_t[:, None] <= o_t[None, :]) & (m_t[:, None] & m_t) - b_A = tl.where(m_A, b_A, 0).to(do.dtype.element_ty) - - b_dA = tl.zeros([BT, BT], dtype=tl.float32) - for i_v in range(tl.cdiv(V, BV)): - p_v = tl.make_block_ptr(v, (V, T), (1, HV * V), (i_v * BV, i_t * BT), (BV, BT), (0, 1)) - p_do = tl.make_block_ptr(do, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - p_dv = tl.make_block_ptr(dv, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - # [BV, BT] - b_v = tl.load(p_v, boundary_check=(0, 1)) - # [BT, BV] - b_do = tl.load(p_do, boundary_check=(0, 1)) - # [BT, BT] - b_dA += tl.dot(b_do, b_v) - # [BT, BV] - b_dv = tl.dot(b_A.to(b_do.dtype), b_do) - tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1)) - - p_dA = tl.make_block_ptr(dA, (T, BT), (HV * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)) - b_dA = tl.where(o_t[:, None] >= o_t, b_dA * scale, 0.0) - tl.store(p_dA, b_dA.to(p_dA.dtype.element_ty), boundary_check=(0, 1)) - - -@triton.heuristics( - { - "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, - } -) -@triton.autotune( - configs=[ - triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages) - for BK in BK_LIST - for BV in BV_LIST - for num_warps in NUM_WARPS - for num_stages in [2, 3, 4] - if not (IS_NVIDIA_HOPPER and BK == 32 and num_warps == 4) - ], - key=["BT", "HV", "TRANSPOSE_STATE"], - **autotune_cache_kwargs, -) -@triton.jit(do_not_specialize=["T"]) -def chunk_kda_bwd_kernel_wy_dqkg_fused( - q, - k, - v, - v_new, - g, - beta, - A, - h, - do, - dh, - dq, - dk, - dv, - dv2, - dg, - db, - dA, - cu_seqlens, - chunk_indices, - scale, - T, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BT: tl.constexpr, - BK: tl.constexpr, - BV: tl.constexpr, - TRANSPOSE_STATE: tl.constexpr, - IS_VARLEN: tl.constexpr, -): - i_t, i_bh = tl.program_id(0), tl.program_id(1) - i_b, i_hv = i_bh // HV, i_bh % HV - i_h = i_hv // (HV // H) - - if IS_VARLEN: - i_tg = i_t.to(tl.int64) - i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) - bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load(cu_seqlens + i_n + 1).to(tl.int64) - T = (eos - bos).to(tl.int32) - NT = tl.cdiv(T, BT) - else: - NT = tl.cdiv(T, BT) - i_tg = (i_b * NT + i_t).to(tl.int64) - bos, eos = (i_b * T).to(tl.int64), (i_b * T + T).to(tl.int64) - - o_t = i_t * BT + tl.arange(0, BT) - m_t = o_t < T - m_last = o_t == min(T, i_t * BT + BT) - 1 - - q += (bos * H + i_h) * K - k += (bos * H + i_h) * K - v += (bos * HV + i_hv) * V - v_new += (bos * HV + i_hv) * V - g += (bos * HV + i_hv) * K - beta += bos * HV + i_hv - A += (bos * HV + i_hv) * BT - h += (i_tg * HV + i_hv) * K * V - do += (bos * HV + i_hv) * V - dh += (i_tg * HV + i_hv) * K * V - dq += (bos * HV + i_hv) * K - dk += (bos * HV + i_hv) * K - dv += (bos * HV + i_hv) * V - dv2 += (bos * HV + i_hv) * V - dg += (bos * HV + i_hv) * K - db += bos * HV + i_hv - dA += (bos * HV + i_hv) * BT - - p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_t * BT,), (BT,), (0,)) - b_beta = tl.load(p_beta, boundary_check=(0,)) - - p_A = tl.make_block_ptr(A, (BT, T), (1, HV * BT), (0, i_t * BT), (BT, BT), (0, 1)) - b_A = tl.load(p_A, boundary_check=(0, 1)) - - b_dA = tl.zeros([BT, BT], dtype=tl.float32) - b_db = tl.zeros([BT], dtype=tl.float32) - - for i_k in range(tl.cdiv(K, BK)): - o_k = i_k * BK + tl.arange(0, BK) - m_k = o_k < K - - p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) - p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) - b_k = tl.load(p_k, boundary_check=(0, 1)) - b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) - - p_gn = g + (min(T, i_t * BT + BT) - 1).to(tl.int64) * HV * K + o_k - b_gn = tl.load(p_gn, mask=m_k, other=0).to(tl.float32) - - b_dq = tl.zeros([BT, BK], dtype=tl.float32) - b_dk = tl.zeros([BT, BK], dtype=tl.float32) - b_dw = tl.zeros([BT, BK], dtype=tl.float32) - b_dgk = tl.zeros([BK], dtype=tl.float32) - - for i_v in range(tl.cdiv(V, BV)): - p_v_new = tl.make_block_ptr(v_new, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - p_do = tl.make_block_ptr(do, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - if TRANSPOSE_STATE: - p_h = tl.make_block_ptr(h, (V, K), (K, 1), (i_v * BV, i_k * BK), (BV, BK), (1, 0)) - p_dh = tl.make_block_ptr(dh, (V, K), (K, 1), (i_v * BV, i_k * BK), (BV, BK), (1, 0)) - else: - p_h = tl.make_block_ptr(h, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1)) - p_dh = tl.make_block_ptr(dh, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1)) - p_dv = tl.make_block_ptr(dv, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - # [BT, BV] - b_v_new = tl.load(p_v_new, boundary_check=(0, 1)) - b_do = tl.load(p_do, boundary_check=(0, 1)) - # [BV, BK] - b_h = tl.load(p_h, boundary_check=(0, 1)) - b_dh = tl.load(p_dh, boundary_check=(0, 1)) - # [BT, BV] - b_dv = tl.load(p_dv, boundary_check=(0, 1)) - - b_dgk += tl.sum(b_h * b_dh, axis=0) - b_dq += tl.dot(b_do, b_h.to(b_do.dtype)) - b_dk += tl.dot(b_v_new, b_dh.to(b_v_new.dtype)) - b_dw += tl.dot(b_dv.to(b_v_new.dtype), b_h.to(b_v_new.dtype)) - tl.debug_barrier() # DO NOT REMOVE THIS LINE! - if i_k == 0: - p_v = tl.make_block_ptr(v, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - p_dv2 = tl.make_block_ptr(dv2, (T, V), (HV * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) - - b_v = tl.load(p_v, boundary_check=(0, 1)) - - b_dA += tl.dot(b_dv, tl.trans(b_v)) - - b_dvb = tl.dot(b_A, b_dv) - b_dv2 = b_dvb * b_beta[:, None] - b_db += tl.sum(b_dvb * b_v, 1) - - tl.store(p_dv2, b_dv2.to(p_dv2.dtype.element_ty), boundary_check=(0, 1)) - - b_gk_exp = exp2(b_g) - b_gb = b_gk_exp * b_beta[:, None] - b_dgk *= exp2(b_gn) - b_dq = b_dq * b_gk_exp * scale - b_dk = b_dk * tl.where(m_t[:, None], exp2(b_gn[None, :] - b_g), 0) - - b_kg = b_k * b_gk_exp - - b_dw = -b_dw.to(b_A.dtype) - b_dA += tl.dot(b_dw, tl.trans(b_kg.to(b_A.dtype))) - - b_dkgb = tl.dot(b_A, b_dw) - b_db += tl.sum(b_dkgb * b_kg, 1) - - p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) - b_q = tl.load(p_q, boundary_check=(0, 1)) - b_kdk = b_k * b_dk - b_dgk += tl.sum(b_kdk, axis=0) - b_dg = b_q * b_dq - b_kdk + m_last[:, None] * b_dgk + b_kg * b_dkgb * b_beta[:, None] - b_dk = b_dk + b_dkgb * b_gb - - p_dq = tl.make_block_ptr(dq, (T, K), (HV * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) - p_dk = tl.make_block_ptr(dk, (T, K), (HV * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) - p_dg = tl.make_block_ptr(dg, (T, K), (HV * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) - tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_dg, b_dg.to(p_dg.dtype.element_ty), boundary_check=(0, 1)) - - m_A = (o_t[:, None] > o_t[None, :]) & (m_t[:, None] & m_t) - b_dA = tl.where(m_A, b_dA * b_beta[None, :], 0) - b_dA = tl.dot(b_dA.to(b_A.dtype), b_A) - b_dA = tl.dot(b_A, b_dA.to(b_A.dtype)) - b_dA = tl.where(m_A, -b_dA, 0) - - p_dA = tl.make_block_ptr(dA, (T, BT), (HV * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)) - p_db = tl.make_block_ptr(db, (T,), (HV,), (i_t * BT,), (BT,), (0,)) - tl.store(p_dA, b_dA.to(p_dA.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_db, b_db.to(p_db.dtype.element_ty), boundary_check=(0,)) - - -def chunk_kda_bwd_dAv( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - do: torch.Tensor, - A: torch.Tensor | None = None, - scale: float = None, - cu_seqlens: torch.LongTensor | None = None, - chunk_size: int = 64, - chunk_indices: torch.LongTensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - B, T, H, K, HV, V = *k.shape, do.shape[2], do.shape[-1] - BT = chunk_size - if chunk_indices is None and cu_seqlens is not None: - chunk_indices = prepare_chunk_indices(cu_seqlens, BT) - # H100 can have larger block size - if check_shared_mem("hopper", k.device.index): - CONST_TILING = 128 - elif check_shared_mem(): - CONST_TILING = 64 - else: - CONST_TILING = 32 - BK = min(max(triton.next_power_of_2(K), 16), CONST_TILING) - BV = min(max(triton.next_power_of_2(V), 16), CONST_TILING) - NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) - - dA = v.new_empty(B, T, HV, BT, dtype=torch.float) - dv = torch.empty_like(do) - grid = (NT, B * HV) - chunk_kda_bwd_kernel_dAv[grid]( - q=q, - k=k, - v=v, - A=A, - do=do, - dv=dv, - dA=dA, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - scale=scale, - T=T, - H=H, - HV=HV, - K=K, - V=V, - BT=BT, - BK=BK, - BV=BV, - ) - return dA, dv - - -def chunk_kda_bwd_wy_dqkg_fused( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - v_new: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - A: torch.Tensor, - h: torch.Tensor, - do: torch.Tensor, - dh: torch.Tensor, - dv: torch.Tensor, - scale: float | None = None, - cu_seqlens: torch.LongTensor | None = None, - chunk_size: int = 64, - chunk_indices: torch.LongTensor | None = None, - transpose_state_layout: bool = False, -): - B, T, H, K, HV, V = *k.shape, v.shape[2], v.shape[-1] - BT = chunk_size - - if chunk_indices is None and cu_seqlens is not None: - chunk_indices = prepare_chunk_indices(cu_seqlens, BT) - NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) - - # dq, dk are allocated at HV dimension; caller reduces to H if GVA - dq = g.new_empty(B, T, HV, K, dtype=torch.float) - dk = g.new_empty(B, T, HV, K, dtype=torch.float) - dv2 = torch.empty_like(v) - dg = torch.empty_like(g, dtype=torch.float) - db = torch.empty_like(beta, dtype=torch.float) - dA = torch.empty_like(A, dtype=torch.float) - - grid = (NT, B * HV) - chunk_kda_bwd_kernel_wy_dqkg_fused[grid]( - q=q, - k=k, - v=v, - v_new=v_new, - g=g, - beta=beta, - A=A, - h=h, - do=do, - dh=dh, - dq=dq, - dk=dk, - dv=dv, - dv2=dv2, - dg=dg, - db=db, - dA=dA, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - scale=scale, - T=T, - H=H, - HV=HV, - K=K, - V=V, - BT=BT, - TRANSPOSE_STATE=transpose_state_layout, - ) - dv = dv2 - return dq, dk, dv, db, dg, dA - - -def chunk_kda_bwd( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - Aqk: torch.Tensor, - Akk: torch.Tensor, - scale: float, - initial_state: torch.Tensor, - do: torch.Tensor, - dht: torch.Tensor, - g: torch.Tensor | None = None, - g_org: torch.Tensor | None = None, - cu_seqlens: torch.LongTensor | None = None, - chunk_indices: torch.LongTensor | None = None, - chunk_size: int = 64, - safe_gate: bool = False, - lower_bound: float | None = None, - use_gate_in_kernel: bool = False, - A_log: torch.Tensor | None = None, - dt_bias: torch.Tensor | None = None, - disable_recompute: bool = False, - cp_context: FLACPContext | None = None, - transpose_state_layout: bool = False, - **kwargs, -): - assert transpose_state_layout is False, "transpose_state_layout=True is not supported for training." - - H, HV = q.shape[2], v.shape[2] - G = HV // H - - if disable_recompute is False: - if use_gate_in_kernel: - g = kda_gate_chunk_cumsum( - g=g_org, - A_log=A_log, - dt_bias=dt_bias, - scale=RCP_LN2, - chunk_size=chunk_size, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - lower_bound=lower_bound, - ) - if cu_seqlens is not None and chunk_indices is None: - chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) - recompute_backend = _select_recompute_w_u_backend(q.device) - if recompute_backend == "cutedsl": - w, u, qg, kg = _get_sm100_cutedsl_recompute_wu()( - k, - v, - beta, - Akk, - g, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - q=q, - ) - else: - w, u, qg, kg = recompute_w_u_fwd_triton( - k=k, - v=v, - beta=beta, - A=Akk, - q=q, - gk=g, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - ) - if cp_context is not None: - # Restore the full initial_state tensor from the compressed version. - # Only the first sequence's state is non-zero as it's the only one that could be cross-rank. - initial_state = expand_h0(initial_state, context=cp_context) - # TODO: update to support only varlen (1,T,H,D) format - chunk_delta_h_fwd = _get_chunk_delta_h_fwd(q.device) - h, v_new, _ = chunk_delta_h_fwd( - k=kg, - w=w, - u=u, - gk=g, - initial_state=initial_state, - output_final_state=False, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - ) - else: - w, u, qg, kg, v_new, h = kwargs["w"], kwargs["u"], kwargs["qg"], kwargs["kg"], kwargs["v_new"], kwargs["h"] - if cp_context is not None: - # Restore the full initial_state tensor from the compressed version. - # Only the first sequence's state is non-zero as it's the only one that could be cross-rank. - initial_state = expand_h0(initial_state, context=cp_context) - - # dAqk = do @ v.T - # dv = A @ do - dAqk, dv = chunk_kda_bwd_dAv( - q=q, - k=k, - v=v_new, - do=do, - A=Aqk, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - ) - - if cp_context is not None: - # initial_state is None in the CP mode - # We only need to compute dht of current rank and pass it to the backward kernel - dht, initial_state = chunk_gated_delta_rule_bwd_dhu_pre_process( - q=qg, - k=kg, - w=w, - do=do, - dv=dv, - gk=g, - scale=scale, - cu_seqlens=cu_seqlens, - dht=dht, - initial_state=initial_state, - use_exp2=True, - context=cp_context, - transpose_state_layout=transpose_state_layout, - ) - - dh, dh0, dv = chunk_gated_delta_rule_bwd_dhu( - q=qg, - k=kg, - w=w, - gk=g, - h0=initial_state, - dht=dht, - do=do, - dv=dv, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - use_exp2=True, - transpose_state_layout=transpose_state_layout, - ) - - chunk_kda_bwd_wy_dqkg_fused_impl = _select_chunk_kda_bwd_wy_dqkg_fused(q, v) - dq, dk, dv, db, dg, dAkk = chunk_kda_bwd_wy_dqkg_fused_impl( - q=q, - k=k, - v=v, - v_new=v_new, - g=g, - beta=beta, - A=Akk, - h=h, - do=do, - dh=dh, - dv=dv, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - # transpose_state_layout=transpose_state_layout, - ) - - dq, dk, db, dg = chunk_kda_bwd_intra( - q=q, - k=k, - g=g, - beta=beta, - dAqk=dAqk, - dAkk=dAkk, - dq=dq, - dk=dk, - db=db, - dg=dg, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - safe_gate=safe_gate, - ) - - # For GVA, reduce dq and dk from [B, T, HV, K] back to [B, T, H, K] - if HV > H: - dq = dq.view(*dq.shape[:2], H, G, dq.shape[-1]).sum(dim=3) - dk = dk.view(*dk.shape[:2], H, G, dk.shape[-1]).sum(dim=3) - - dA, dbias = None, None - dg = chunk_local_cumsum( - dg, - chunk_size=chunk_size, - reverse=True, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - ) - if use_gate_in_kernel: - dg, dA, dbias = kda_gate_bwd(g=g_org, A_log=A_log, dt_bias=dt_bias, dyg=dg, lower_bound=lower_bound) - - return dq, dk, dv, db, dg, dh0, dA, dbias diff --git a/cula/kda/chunk_fwd.py b/cula/kda/chunk_fwd.py deleted file mode 100644 index 3694cedd..00000000 --- a/cula/kda/chunk_fwd.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SM90/SM100 modular chunk KDA forward orchestration.""" - -import functools - -import torch -from fla.ops.common.chunk_delta_h import chunk_gated_delta_rule_fwd_h as _triton_chunk_gated_delta_rule_fwd_h -from fla.ops.cp import FLACPContext -from fla.ops.cp.chunk_delta_h import ( - chunk_gated_delta_rule_fwd_h_pre_process, - compress_h0, -) -from fla.ops.gla.chunk import chunk_gla_fwd_o_gk as _triton_chunk_gla_fwd_o_gk -from fla.ops.kda.gate import kda_gate_chunk_cumsum -from fla.ops.utils import chunk_local_cumsum -from fla.ops.utils.constant import RCP_LN2 - -from cula.kda.chunk_intra import chunk_kda_fwd_intra -from cula.utils import get_device_sm_version - - -def _sm90_triton_chunk_gated_delta_rule_fwd_h( - k: torch.Tensor, - w: torch.Tensor, - u: torch.Tensor, - g: torch.Tensor | None = None, - gk: torch.Tensor | None = None, - initial_state: torch.Tensor | None = None, - output_final_state: bool = False, - chunk_size: int = 64, - save_new_value: bool = True, - cu_seqlens: torch.Tensor | None = None, - chunk_indices: torch.Tensor | None = None, - persistent: bool = True, - _no_cp: bool = False, - cu_seqlens_cpu: torch.Tensor | None = None, - use_intracard_cp=None, -): - """FLA Triton delta-H adapter for the SM90 forward path.""" - del persistent, _no_cp, use_intracard_cp - return _triton_chunk_gated_delta_rule_fwd_h( - k=k, - w=w, - u=u, - g=g, - gk=gk, - initial_state=initial_state, - output_final_state=output_final_state, - chunk_size=chunk_size, - save_new_value=save_new_value, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens_cpu, - chunk_indices=chunk_indices, - use_exp2=True, - transpose_state_layout=False, - ) - - -def _sm90_triton_chunk_gla_fwd_o( - q: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - h: torch.Tensor, - o: torch.Tensor | None, - A: torch.Tensor, - scale: float, - chunk_size: int = 64, - cu_seqlens: torch.Tensor | None = None, - chunk_indices: torch.Tensor | None = None, - is_varlen: bool = False, - persistent: bool = True, -): - """FLA Triton output adapter for the SM90 forward path.""" - del is_varlen, persistent - h_fla = h.flatten(0, 1) if h.dim() == 5 else h - out = _triton_chunk_gla_fwd_o_gk( - q=q, - v=v, - g=g, - A=A, - h=h_fla, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - use_exp2=True, - transpose_state_layout=False, - ) - if o is not None: - o.copy_(out) - return o - return out - - -@functools.cache -def _get_fwd_kernels_for_sm(major: int, minor: int): - if major == 9 and minor == 0: - return _sm90_triton_chunk_gated_delta_rule_fwd_h, _sm90_triton_chunk_gla_fwd_o - if major == 10 and minor in (0, 3): - from cula.ops.kda.sm100.delta_h import chunk_gated_delta_rule_fwd_h - from cula.ops.kda.sm100.fwd_o import chunk_gla_fwd_o - - return chunk_gated_delta_rule_fwd_h, chunk_gla_fwd_o - raise RuntimeError( - f"Unsupported CUDA compute capability sm_{major}{minor}. " - "Only sm90a (Hopper) and Blackwell (SM100/SM103) are supported." - ) - - -def chunk_kda_fwd( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - scale: float, - initial_state: torch.Tensor, - output_final_state: bool, - cu_seqlens: torch.IntTensor | None = None, - cu_seqlens_cpu: torch.IntTensor | None = None, - chunk_indices: torch.IntTensor | None = None, - chunk_size: int = 64, - safe_gate: bool = False, - lower_bound: float | None = None, - use_gate_in_kernel: bool = False, - A_log: torch.Tensor | None = None, - dt_bias: torch.Tensor | None = None, - disable_recompute: bool = False, - return_intermediate_states: bool = False, - cp_context: FLACPContext | None = None, - use_intracard_cp=None, - use_tf32_inverse: bool = True, - unified_gref: bool = False, # Set True for ~5% extra perf (slightly lower precision) -): - chunk_gated_delta_rule_fwd_h, chunk_gla_fwd_o = _get_fwd_kernels_for_sm(*get_device_sm_version(q.device)) - - # Apply gate activation - g_org = None - if use_gate_in_kernel: - g_org = g - g = kda_gate_chunk_cumsum( - g=g_org, - A_log=A_log, - dt_bias=dt_bias, - scale=RCP_LN2, - chunk_size=chunk_size, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - lower_bound=lower_bound, - ) - else: - g = chunk_local_cumsum(g=g, scale=RCP_LN2, chunk_size=chunk_size, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices) - - # qg = None if disable_recompute is False - w, u, qg, kg, Aqk, Akk = chunk_kda_fwd_intra( - q=q, - k=k, - v=v, - gk=g, - beta=beta, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - safe_gate=safe_gate, - disable_recompute=disable_recompute, - use_tf32_inverse=use_tf32_inverse, - unified_gref=unified_gref, - ) - - if cp_context is not None: - initial_state = chunk_gated_delta_rule_fwd_h_pre_process( - k=kg, - w=w, - u=u, - gk=g, - cu_seqlens=cu_seqlens, - initial_state=initial_state, - context=cp_context, - use_exp2=True, - ) - - h, v_new, final_state = chunk_gated_delta_rule_fwd_h( - k=kg, - w=w, - u=u, - gk=g, - initial_state=initial_state, - output_final_state=output_final_state, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - cu_seqlens_cpu=cu_seqlens_cpu, - use_intracard_cp=use_intracard_cp, - ) - - if cp_context is not None: - # In Context Parallel (CP) mode, global initial states are not supported at the entry point. - # The `initial_state` here is computed internally via inter-rank communication. - # Since only the first sequence in the local batch can be a continuation of a cross-rank sequence, - # only the first state in the tensor is relevant. We compress it to optimize memory for `save_for_backward`. - initial_state = compress_h0(initial_state, context=cp_context) - - # Please ensure zeros, since vllm will use padding v - o = torch.zeros_like(v) - chunk_gla_fwd_o( - q=q, - v=v_new, - g=g, - A=Aqk, - h=h, - o=o, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - is_varlen=cu_seqlens is not None, - ) - if disable_recompute is False: - # Delete to save memory - w, u, qg, kg, v_new = None, None, None, None, None - if not return_intermediate_states: - # Only delete h if not requested for inference - h = None - if use_gate_in_kernel: - g = None - return o, final_state, g, Aqk, Akk, w, u, qg, kg, v_new, h, initial_state diff --git a/cula/kda/chunk_intra.py b/cula/kda/chunk_intra.py index 1d62cf8b..07ddd674 100644 --- a/cula/kda/chunk_intra.py +++ b/cula/kda/chunk_intra.py @@ -1,322 +1,19 @@ # Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang +"""Standalone CuTeDSL composition of the SM100 KDA intra/WU kernels.""" -"""SM90/SM100 modular chunk KDA intra-chunk wrapper and helpers.""" - -import functools -import importlib +from __future__ import annotations import torch -import triton -import triton.language as tl -from fla.ops.kda.chunk_intra import chunk_kda_bwd_intra as chunk_kda_bwd_intra_triton -from fla.ops.kda.chunk_intra import chunk_kda_fwd_intra as chunk_kda_fwd_intra_triton -from fla.ops.utils import prepare_chunk_indices -from fla.ops.utils.op import exp2, gather -from fla.utils import IS_GATHER_SUPPORTED, autotune_cache_kwargs +from cula.ops.kda.index import prepare_chunk_indices +from cula.ops.kda.sm100.full_port import chunk_kda_fwd_full_sm100_from_gk +from cula.ops.kda.sm100.intra_fused import chunk_kda_fwd_intra_sm100_from_gk +from cula.ops.kda.sm100.recompute_wu import recompute_w_u_fwd from cula.utils import get_device_sm_version -@functools.cache -def _get_sm100_cutedsl_fwd(): - intra = importlib.import_module("cula.ops.kda.sm100.intra_fused") - recompute = importlib.import_module("cula.ops.kda.sm100.recompute_wu") - full = importlib.import_module("cula.ops.kda.sm100.full_port") - return intra.chunk_kda_fwd_intra_sm100_from_gk, recompute.recompute_w_u_fwd, full.chunk_kda_fwd_full_sm100_from_gk - - -@triton.heuristics( - { - "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, - } -) -@triton.autotune( - configs=[ - triton.Config({}, num_warps=num_warps, num_stages=num_stages) for num_warps in [1, 2, 4, 8] for num_stages in [2, 3, 4] - ], - key=["BK", "NC", "BT", "HV"], - **autotune_cache_kwargs, -) -@triton.jit(do_not_specialize=["B", "T"]) -def chunk_kda_bwd_kernel_intra( - q, - k, - g, - beta, - dAqk, - dAkk, - dq, - dq2, - dk, - dk2, - dg, - dg2, - db, - cu_seqlens, - chunk_indices, - B, - T, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - BT: tl.constexpr, - BC: tl.constexpr, - BK: tl.constexpr, - NC: tl.constexpr, - IS_VARLEN: tl.constexpr, - SAFE_GATE: tl.constexpr, - USE_GATHER: tl.constexpr, -): - i_kc, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) - i_b, i_hv = i_bh // HV, i_bh % HV - i_h = i_hv // (HV // H) - i_k, i_i = i_kc // NC, i_kc % NC - - all = B * T - if IS_VARLEN: - i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) - bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) - else: - bos, eos = i_b * T, i_b * T + T - T = eos - bos - - i_ti = i_t * BT + i_i * BC - if i_ti >= T: - return - - o_k = i_k * BK + tl.arange(0, BK) - m_k = o_k < K - - q += (bos * H + i_h) * K - k += (bos * H + i_h) * K - g += (bos * HV + i_hv) * K - beta += bos * HV + i_hv - - dAqk += (bos * HV + i_hv) * BT - dAkk += (bos * HV + i_hv) * BT - dq += (bos * HV + i_hv) * K - dq2 += (bos * HV + i_hv) * K - dk += (bos * HV + i_hv) * K - dk2 += (bos * HV + i_hv) * K - dg += (bos * HV + i_hv) * K - dg2 += (bos * HV + i_hv) * K - db += (i_k * all + bos) * HV + i_hv - - p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) - - p_b = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,)) - b_b = tl.load(p_b, boundary_check=(0,)) - - b_dq2 = tl.zeros([BC, BK], dtype=tl.float32) - b_dk2 = tl.zeros([BC, BK], dtype=tl.float32) - if i_i > 0: - p_gn = g + i_ti * HV * K + o_k - # [BK,] - b_gn = tl.load(p_gn, mask=m_k, other=0).to(tl.float32)[None, :] - for i_j in range(0, i_i): - p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0)) - p_gk = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0)) - p_dAqk = tl.make_block_ptr(dAqk, (T, BT), (HV * BT, 1), (i_ti, i_j * BC), (BC, BC), (1, 0)) - p_dAkk = tl.make_block_ptr(dAkk, (T, BT), (HV * BT, 1), (i_ti, i_j * BC), (BC, BC), (1, 0)) - # [BC, BK] - b_k = tl.load(p_k, boundary_check=(0, 1)) - b_gk = tl.load(p_gk, boundary_check=(0, 1)) - b_kg = b_k * exp2(b_gn - b_gk) - # [BC, BC] - b_dAqk = tl.load(p_dAqk, boundary_check=(0, 1)) - b_dAkk = tl.load(p_dAkk, boundary_check=(0, 1)) - # [BC, BK] - b_dq2 += tl.dot(b_dAqk, b_kg) - b_dk2 += tl.dot(b_dAkk, b_kg) - b_gqn = exp2(b_g - b_gn) - b_dq2 *= b_gqn - b_dk2 *= b_gqn - - o_i = tl.arange(0, BC) - m_dA = (i_ti + o_i) < T - o_dA = (i_ti + o_i) * HV * BT + i_i * BC - p_kj = k + i_ti * H * K + o_k - p_gkj = g + i_ti * HV * K + o_k - - p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - b_q = tl.load(p_q, boundary_check=(0, 1)) - b_k = tl.load(p_k, boundary_check=(0, 1)) - - if SAFE_GATE: - if USE_GATHER: - b_gn = gather(b_g, tl.full([1, BK], min(BC // 2, T - i_ti - 1), dtype=tl.int16), axis=0) - else: - p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV * K + o_k - b_gn = tl.load(p_gn, mask=m_k, other=0)[None, :] - - p_dAqk = tl.make_block_ptr(dAqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)) - p_dAkk = tl.make_block_ptr(dAkk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0)) - b_dAqk_diag_qk = tl.load(p_dAqk, boundary_check=(0, 1)).to(tl.float32) - b_dAkk_diag_qk = tl.load(p_dAkk, boundary_check=(0, 1)).to(tl.float32) - - m_i_diag_qk = (o_i[:, None] >= o_i[None, :]) & ((i_ti + o_i[:, None]) < T) & ((i_ti + o_i[None, :]) < T) - m_j_diag_qk = (i_ti + o_i[:, None]) < T - - b_dAqk_diag_qk = tl.where(m_i_diag_qk, b_dAqk_diag_qk, 0.0) - b_dAkk_diag_qk = tl.where(m_i_diag_qk, b_dAkk_diag_qk, 0.0) - b_g_diag_qk = tl.where(m_j_diag_qk, b_g - b_gn, 0.0) - exp_b_g_diag_qk = tl.where(m_j_diag_qk, exp2(b_g_diag_qk), 0.0) - exp_neg_b_g_diag_qk = tl.where(m_j_diag_qk, exp2(-b_g_diag_qk), 0.0) - - b_k_exp_diag_qk = b_k * exp_neg_b_g_diag_qk - b_dq2 += tl.dot(b_dAqk_diag_qk, b_k_exp_diag_qk) * exp_b_g_diag_qk - b_dk2 += tl.dot(b_dAkk_diag_qk, b_k_exp_diag_qk) * exp_b_g_diag_qk - else: - for j in range(0, min(BC, T - i_t * BT - i_i * BC)): - # [BC] - b_dAqk = tl.load(dAqk + o_dA + j, mask=m_dA, other=0) - b_dAkk = tl.load(dAkk + o_dA + j, mask=m_dA, other=0) - # [BK] - b_kj = tl.load(p_kj, mask=m_k, other=0).to(tl.float32) - b_gkj = tl.load(p_gkj, mask=m_k, other=0).to(tl.float32) - # [BC, BK] - m_i = o_i[:, None] >= j - # [BC, BK] - b_gqk = exp2(b_g - b_gkj[None, :]) - b_dq2 += tl.where(m_i, b_dAqk[:, None] * b_kj[None, :] * b_gqk, 0.0) - b_dk2 += tl.where(m_i, b_dAkk[:, None] * b_kj[None, :] * b_gqk, 0.0) - - p_kj += H * K - p_gkj += HV * K - - b_db = tl.sum(b_dk2 * b_k, 1) - b_dk2 *= b_b[:, None] - - p_dq = tl.make_block_ptr(dq, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - p_dq2 = tl.make_block_ptr(dq2, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - p_db = tl.make_block_ptr(db, (T,), (HV,), (i_ti,), (BC,), (0,)) - - b_dg2 = b_q * b_dq2 - b_dq2 = b_dq2 + tl.load(p_dq, boundary_check=(0, 1)) - tl.store(p_dq2, b_dq2.to(p_dq2.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_db, b_db.to(p_db.dtype.element_ty), boundary_check=(0,)) - - tl.debug_barrier() - b_dkt = tl.zeros([BC, BK], dtype=tl.float32) - - NC = min(NC, tl.cdiv(T - i_t * BT, BC)) - if i_i < NC - 1: - p_gn = g + (min(i_ti + BC, T) - 1) * HV * K + o_k - # [BK,] - b_gn = tl.load(p_gn, mask=m_k, other=0).to(tl.float32)[None, :] - for i_j in range(i_i + 1, NC): - p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0)) - p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0)) - p_gk = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_t * BT + i_j * BC, i_k * BK), (BC, BK), (1, 0)) - p_b = tl.make_block_ptr(beta, (T,), (HV,), (i_t * BT + i_j * BC,), (BC,), (0,)) - p_dAqk = tl.make_block_ptr(dAqk, (BT, T), (1, HV * BT), (i_i * BC, i_t * BT + i_j * BC), (BC, BC), (0, 1)) - p_dAkk = tl.make_block_ptr(dAkk, (BT, T), (1, HV * BT), (i_i * BC, i_t * BT + i_j * BC), (BC, BC), (0, 1)) - # [BC] - b_b = tl.load(p_b, boundary_check=(0,)) - # [BC, BK] - b_q = tl.load(p_q, boundary_check=(0, 1)) - b_kb = tl.load(p_k, boundary_check=(0, 1)) * b_b[:, None] - b_gk = tl.load(p_gk, boundary_check=(0, 1)).to(tl.float32) - # [BC, BC] - b_dAqk = tl.load(p_dAqk, boundary_check=(0, 1)) - b_dAkk = tl.load(p_dAkk, boundary_check=(0, 1)) - - o_j = i_t * BT + i_j * BC + o_i - m_j = o_j < T - # [BC, BK] - b_gkn = exp2(b_gk - b_gn) - b_qg = b_q * tl.where(m_j[:, None], b_gkn, 0) - b_kbg = b_kb * tl.where(m_j[:, None], b_gkn, 0) - # [BC, BK] - # (SY 09/17) important to not use bf16 here to have a good precision. - b_dkt += tl.dot(b_dAqk, b_qg) - b_dkt += tl.dot(b_dAkk, b_kbg) - b_dkt *= exp2(b_gn - b_g) - o_dA = i_ti * HV * BT + i_i * BC + o_i - p_qj = q + i_ti * H * K + o_k - p_kj = k + i_ti * H * K + o_k - p_gkj = g + i_ti * HV * K + o_k - p_bj = beta + i_ti * HV - - if SAFE_GATE: - if USE_GATHER: - b_gn = gather(b_g, tl.full([1, BK], min(BC // 2, T - i_ti - 1), dtype=tl.int16), axis=0) - else: - p_gn = g + (i_ti + min(BC // 2, T - i_ti - 1)) * HV * K + o_k - b_gn = tl.load(p_gn, mask=m_k, other=0).to(tl.float32)[None, :] - p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - b_q = tl.load(p_q, boundary_check=(0, 1)) - p_b = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,)) - b_b = tl.load(p_b, boundary_check=(0,)) - - p_dAqk = tl.make_block_ptr(dAqk, (BT, T), (1, HV * BT), (i_i * BC, i_ti), (BC, BC), (0, 1)) - p_dAkk = tl.make_block_ptr(dAkk, (BT, T), (1, HV * BT), (i_i * BC, i_ti), (BC, BC), (0, 1)) - b_dAqk_diag_kk = tl.load(p_dAqk, boundary_check=(0, 1)).to(tl.float32) - b_dAkk_diag_kk = tl.load(p_dAkk, boundary_check=(0, 1)).to(tl.float32) - - m_i_diag_kk = (o_i[:, None] <= o_i[None, :]) & ((i_ti + o_i[:, None]) < T) & ((i_ti + o_i[None, :]) < T) - m_j_diag_kk = (i_ti + o_i[:, None]) < T - - b_dAqk_diag_kk = tl.where(m_i_diag_kk, b_dAqk_diag_kk, 0.0) - b_dAkk_diag_kk = tl.where(m_i_diag_kk, b_dAkk_diag_kk, 0.0) - # ensure numerical stability - b_g_diag_kk = tl.where(m_j_diag_kk, b_g - b_gn, 0.0) - exp_b_g_diag_kk = tl.where(m_j_diag_kk, exp2(b_g_diag_kk), 0.0) - exp_neg_b_g_diag_kk = tl.where(m_j_diag_kk, exp2(-b_g_diag_kk), 0.0) - - b_q_exp = b_q * exp_b_g_diag_kk - b_kb_exp = b_k * b_b[:, None] * exp_b_g_diag_kk - - b_dkt += tl.dot(b_dAqk_diag_kk, b_q_exp) * exp_neg_b_g_diag_kk - b_dkt += tl.dot(b_dAkk_diag_kk, b_kb_exp) * exp_neg_b_g_diag_kk - else: - for j in range(0, min(BC, T - i_t * BT - i_i * BC)): - # [BC,] - b_dAqk = tl.load(dAqk + o_dA + j * HV * BT) - b_dAkk = tl.load(dAkk + o_dA + j * HV * BT) - # [BK,] - b_qj = tl.load(p_qj, mask=m_k, other=0).to(tl.float32) - b_kbj = tl.load(p_kj, mask=m_k, other=0).to(tl.float32) * tl.load(p_bj) - b_gkj = tl.load(p_gkj, mask=m_k, other=0).to(tl.float32) - # [BC, BK] - m_i = o_i[:, None] <= j - b_gkq = exp2(b_gkj[None, :] - b_g) - b_dkt += tl.where(m_i, b_dAqk[:, None] * b_qj[None, :] * b_gkq, 0.0) - b_dkt += tl.where(m_i, b_dAkk[:, None] * b_kbj[None, :] * b_gkq, 0.0) - - p_qj += H * K - p_kj += H * K - p_gkj += HV * K - p_bj += HV - p_dk = tl.make_block_ptr(dk, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - p_dk2 = tl.make_block_ptr(dk2, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - p_dg = tl.make_block_ptr(dg, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - p_dg2 = tl.make_block_ptr(dg2, (T, K), (HV * K, 1), (i_ti, i_k * BK), (BC, BK), (1, 0)) - - b_dg2 += (b_dk2 - b_dkt) * b_k + tl.load(p_dg, boundary_check=(0, 1)) - b_dk2 += tl.load(p_dk, boundary_check=(0, 1)) - b_dk2 += b_dkt - - tl.store(p_dk2, b_dk2.to(p_dk2.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_dg2, b_dg2.to(p_dg2.dtype.element_ty), boundary_check=(0, 1)) - - def chunk_kda_fwd_intra( q: torch.Tensor, k: torch.Tensor, @@ -330,61 +27,63 @@ def chunk_kda_fwd_intra( safe_gate: bool = False, disable_recompute: bool = False, use_tf32_inverse: bool = True, - unified_gref: bool = False, # Set True for ~5% extra perf (slightly lower precision) + unified_gref: bool = False, ): - assert safe_gate, "Only safe_gate=True is supported in chunk_kda_fwd_intra for now" + """Compose the standalone SM100 intra and W/U preparation kernels. + + ``gk`` is the already activated, chunk-cumulative gate. Gate activation, + cumsum, recurrent state propagation, and output projection are deliberately + outside this atomic API. + """ + + if not safe_gate: + raise ValueError("chunk_kda_fwd_intra requires safe_gate=True") major, minor = get_device_sm_version(q.device) - if major == 9 and minor == 0: - return chunk_kda_fwd_intra_triton( - q=q, - k=k, - v=v, - gk=gk, - beta=beta, - scale=scale, - cu_seqlens=cu_seqlens, - chunk_size=chunk_size, - chunk_indices=chunk_indices, - safe_gate=safe_gate, - disable_recompute=disable_recompute, - ) if not (major == 10 and minor in (0, 3)): raise RuntimeError( - f"Unsupported CUDA compute capability sm_{major}{minor}. " - "Only sm90a (Hopper) and Blackwell (SM100/SM103) are supported." + f"Unsupported CUDA compute capability sm_{major}{minor}; the modular intra path requires SM100 or SM103" ) - B, T, H, K = k.shape - # GVA: g/beta/v live in h_v head space; q/k live in h_qk head space. - HV = v.size(2) - assert H > 0 and HV > 0 and HV % H == 0, f"HV ({HV}) must be a positive multiple of HQK ({H})" - BT = chunk_size - if gk is None or beta is None: - raise ValueError("gk and beta are required for the SM100/SM103 CuTeDSL path") + raise ValueError("gk and beta are required") if unified_gref: - raise NotImplementedError("unified_gref=True is not part of the bitwise-aligned CuTeDSL path") + raise NotImplementedError("unified_gref=True is not supported by the exact CuTeDSL path") + if chunk_size != 64: + raise NotImplementedError(f"the SM100 modular kernels require chunk_size=64, got {chunk_size}") + + batch, length, qk_heads, key_dim = k.shape + value_heads = v.size(2) + if qk_heads <= 0 or value_heads <= 0 or value_heads % qk_heads: + raise ValueError(f"value heads ({value_heads}) must be a positive multiple of Q/K heads ({qk_heads})") if chunk_indices is None and cu_seqlens is not None: - chunk_indices = prepare_chunk_indices(cu_seqlens, BT) - + chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) if cu_seqlens is not None and (cu_seqlens.dtype != torch.int32 or chunk_indices.dtype != torch.int32): raise TypeError("cu_seqlens and chunk_indices must be int32") - cutedsl_intra, cutedsl_recompute_wu, cutedsl_full = _get_sm100_cutedsl_fwd() - if cu_seqlens is None and not disable_recompute and B >= 2 and T % (4 * BT) == 0 and K == 128 and v.shape[-1] == 128: - return cutedsl_full(q, k, v, gk, beta, K**-0.5 if scale is None else scale) - - Aqk, Akk = cutedsl_intra( + if not use_tf32_inverse: + raise NotImplementedError("use_tf32_inverse=False is not supported by the exact CuTeDSL path") + resolved_scale = key_dim**-0.5 if scale is None else scale + if ( + cu_seqlens is None + and not disable_recompute + and batch >= 2 + and length % (4 * chunk_size) == 0 + and key_dim == 128 + and v.shape[-1] == 128 + ): + return chunk_kda_fwd_full_sm100_from_gk(q, k, v, gk, beta, resolved_scale) + + Aqk, Akk = chunk_kda_fwd_intra_sm100_from_gk( q=q, k=k, gk=gk, beta=beta, - scale=scale, + scale=resolved_scale, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices, chunk_size=chunk_size, use_tf32_inverse=use_tf32_inverse, ) - w, u, qg, kg = cutedsl_recompute_wu( + w, u, qg, kg = recompute_w_u_fwd( k, v, beta, @@ -394,96 +93,7 @@ def chunk_kda_fwd_intra( chunk_indices=chunk_indices, q=q if disable_recompute else None, ) - return w, u, qg, kg, Aqk, Akk -def chunk_kda_bwd_intra( - q: torch.Tensor, - k: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - dAqk: torch.Tensor, - dAkk: torch.Tensor, - dq: torch.Tensor, - dk: torch.Tensor, - db: torch.Tensor, - dg: torch.Tensor, - cu_seqlens: torch.IntTensor | None = None, - chunk_indices: torch.IntTensor | None = None, - chunk_size: int = 64, - safe_gate: bool = False, -): - major, minor = get_device_sm_version(q.device) - if major == 9 and minor == 0: - return chunk_kda_bwd_intra_triton( - q=q, - k=k, - g=g, - beta=beta, - dAqk=dAqk, - dAkk=dAkk, - dq=dq, - dk=dk, - db=db, - dg=dg, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - chunk_size=chunk_size, - safe_gate=safe_gate, - ) - if not (major == 10 and minor in (0, 3)): - raise RuntimeError( - f"Unsupported CUDA compute capability sm_{major}{minor}. " - "Only sm90a (Hopper) and Blackwell (SM100/SM103) are supported." - ) - B, T, H, K, HV = *k.shape, g.shape[2] - BT = chunk_size - BC = min(16, BT) - BK = min(32, triton.next_power_of_2(K)) - - if chunk_indices is None and cu_seqlens is not None: - chunk_indices = prepare_chunk_indices(cu_seqlens, BT) - NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) - NC = triton.cdiv(BT, BC) - NK = triton.cdiv(K, BK) - - dq2 = torch.empty_like(dq) - dk2 = torch.empty_like(dk) - db2 = beta.new_empty(NK, *beta.shape, dtype=torch.float) - dg2 = torch.empty_like(dg, dtype=torch.float) - grid = (NK * NC, NT, B * HV) - chunk_kda_bwd_kernel_intra[grid]( - q=q, - k=k, - g=g, - beta=beta, - dAqk=dAqk, - dAkk=dAkk, - dq=dq, - dq2=dq2, - dk=dk, - dk2=dk2, - dg=dg, - dg2=dg2, - db=db2, - cu_seqlens=cu_seqlens, - chunk_indices=chunk_indices, - B=B, - T=T, - H=H, - HV=HV, - K=K, - BT=BT, - BC=BC, - BK=BK, - NC=NC, - SAFE_GATE=safe_gate, - USE_GATHER=IS_GATHER_SUPPORTED, - ) - dq = dq2 - dk = dk2 - db = db2.sum(0).add_(db) - dg = dg2 - - return dq, dk, db, dg +__all__ = ["chunk_kda_fwd_intra"] diff --git a/cula/kda/wy_intra.py b/cula/kda/wy_intra.py deleted file mode 100644 index 58e741c6..00000000 --- a/cula/kda/wy_intra.py +++ /dev/null @@ -1,354 +0,0 @@ -from __future__ import annotations - -import torch -import triton -import triton.language as tl - -from cula.kda.wy_recompute import kda_recompute_w_u - - -@triton.autotune( - configs=[ - triton.Config({}, num_warps=num_warps, num_stages=num_stages) for num_warps in [1, 2, 4] for num_stages in [2, 3, 4] - ], - key=["H", "K", "BT", "BC"], -) -@triton.jit(do_not_specialize=["T"]) -def _kda_intra_sub_chunk_kernel( - k, - g, - beta, - Akkd, - T, - H: tl.constexpr, - K: tl.constexpr, - BT: tl.constexpr, - BC: tl.constexpr, - BK: tl.constexpr, -): - i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) - i_b, i_h = i_bh // H, i_bh % H - bos = i_b * T - i_ti = i_t * BT + i_i * BC - if i_ti >= T: - return - - o_c = i_ti + tl.arange(0, BC) - m_c = o_c < T - - # Per-head pointer offsets - k_base = k + (bos * H + i_h) * K - g_base = g + (bos * H + i_h) * K - beta_base = beta + bos * H + i_h - Akkd_base = Akkd + (bos * H + i_h) * BC - - # Load k, g (BC × BK), beta (BC) for this sub-chunk - p_k = tl.make_block_ptr(k_base, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0)) - p_g = tl.make_block_ptr(g_base, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0)) - p_beta = tl.make_block_ptr(beta_base, (T,), (H,), (i_ti,), (BC,), (0,)) - b_k = tl.load(p_k, boundary_check=(0, 1)) - b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) - b_beta = tl.load(p_beta, boundary_check=(0,)) - - o_gn = i_ti + tl.minimum(BC // 2, T - i_ti - 1) - o_k = tl.arange(0, BK) - m_k = o_k < K - b_gn = tl.load(g + (bos * H + i_h) * K + o_gn * (H * K) + o_k, mask=m_k, other=0.0).to(tl.float32) - - b_gm = (b_g - b_gn[None, :]).to(tl.float32) - b_gq = tl.where(m_c[:, None], tl.math.exp2(b_gm), 0.0) - b_gk = tl.where(m_c[:, None], tl.math.exp2(-b_gm), 0.0) - - b_kgt = tl.trans(b_k * b_gk) - b_Akk = tl.dot(b_k * b_gq, b_kgt) * b_beta[:, None] - - o_i = tl.arange(0, BC) - m_Akk = o_i[:, None] > o_i[None, :] - m_I = o_i[:, None] == o_i[None, :] - b_Akk = tl.where(m_Akk, b_Akk, 0.0) - - p_Akkd = tl.make_block_ptr(Akkd_base, (T, BC), (H * BC, 1), (i_ti, 0), (BC, BC), (1, 0)) - tl.store(p_Akkd, b_Akk.to(Akkd.dtype.element_ty), boundary_check=(0, 1)) - tl.debug_barrier() - - b_Ai = -b_Akk - for i in range(2, tl.minimum(BC, T - i_ti)): - b_a = -tl.load(Akkd_base + (i_ti + i) * (H * BC) + o_i) - b_a = tl.where(o_i < i, b_a, 0.0) - b_a += tl.sum(b_a[:, None] * b_Ai, 0) - b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai) - - b_Ai += m_I - - tl.store(p_Akkd, b_Ai.to(Akkd.dtype.element_ty), boundary_check=(0, 1)) - - -_SOLVE_DOT_PRECISION = tl.constexpr("tf32") - - -@triton.autotune( - configs=[triton.Config({"BK": BK}, num_warps=num_warps) for BK in [32, 64] for num_warps in [1, 2, 4]], - key=["H", "K", "BC"], -) -@triton.jit(do_not_specialize=["T"]) -def _kda_intra_inter_solve_kernel( - k, - g, - beta, - Akkd, - Akk, - T, - H: tl.constexpr, - K: tl.constexpr, - BT: tl.constexpr, - BC: tl.constexpr, - BK: tl.constexpr, -): - i_t, i_bh = tl.program_id(0), tl.program_id(1) - i_b, i_h = i_bh // H, i_bh % H - bos = i_b * T - - if i_t * BT >= T: - return - - i_tc0 = i_t * BT - i_tc1 = i_t * BT + BC - i_tc2 = i_t * BT + 2 * BC - i_tc3 = i_t * BT + 3 * BC - - k_base = k + (bos * H + i_h) * K - g_base = g + (bos * H + i_h) * K - Akk_base = Akk + (bos * H + i_h) * BT - Akkd_base = Akkd + (bos * H + i_h) * BC - - o_i = tl.arange(0, BC) - m_tc1 = (i_tc1 + o_i) < T - m_tc2 = (i_tc2 + o_i) < T - m_tc3 = (i_tc3 + o_i) < T - - b_Akk10 = tl.zeros([BC, BC], dtype=tl.float32) - b_Akk20 = tl.zeros([BC, BC], dtype=tl.float32) - b_Akk21 = tl.zeros([BC, BC], dtype=tl.float32) - b_Akk30 = tl.zeros([BC, BC], dtype=tl.float32) - b_Akk31 = tl.zeros([BC, BC], dtype=tl.float32) - b_Akk32 = tl.zeros([BC, BC], dtype=tl.float32) - - for i_k in range(tl.cdiv(K, BK)): - o_k = i_k * BK + tl.arange(0, BK) - m_k = o_k < K - - p_k0 = tl.make_block_ptr(k_base, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)) - p_g0 = tl.make_block_ptr(g_base, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)) - b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32) - b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32) - - # sub-chunk 1 (vs sub-chunk 0) - if i_tc1 < T: - p_k1 = tl.make_block_ptr(k_base, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)) - p_g1 = tl.make_block_ptr(g_base, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)) - b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32) - b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32) - b_gn1 = tl.load(g + (bos * H + i_h) * K + i_tc1 * (H * K) + o_k, mask=m_k, other=0.0).to(tl.float32) - b_gqn = tl.where(m_tc1[:, None], tl.math.exp2(b_g1 - b_gn1[None, :]), 0.0) - b_kgt = tl.trans(b_k0 * tl.math.exp2(b_gn1[None, :] - b_g0)) - b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt) - - # sub-chunk 2 (vs 0 and 1) - if i_tc2 < T: - p_k2 = tl.make_block_ptr(k_base, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)) - p_g2 = tl.make_block_ptr(g_base, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)) - b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32) - b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32) - b_gn2 = tl.load(g + (bos * H + i_h) * K + i_tc2 * (H * K) + o_k, mask=m_k, other=0.0).to(tl.float32) - b_gqn2 = tl.where(m_tc2[:, None], tl.math.exp2(b_g2 - b_gn2[None, :]), 0.0) - b_kg2 = b_k2 * b_gqn2 - b_kgt0 = tl.trans(b_k0 * tl.math.exp2(b_gn2[None, :] - b_g0)) - b_Akk20 += tl.dot(b_kg2, b_kgt0) - b_kgt1 = tl.trans(b_k1 * tl.math.exp2(b_gn2[None, :] - b_g1)) - b_Akk21 += tl.dot(b_kg2, b_kgt1) - - # sub-chunk 3 (vs 0, 1, 2) - if i_tc3 < T: - p_k3 = tl.make_block_ptr(k_base, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)) - p_g3 = tl.make_block_ptr(g_base, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)) - b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32) - b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32) - b_gn3 = tl.load(g + (bos * H + i_h) * K + i_tc3 * (H * K) + o_k, mask=m_k, other=0.0).to(tl.float32) - b_gqn3 = tl.where(m_tc3[:, None], tl.math.exp2(b_g3 - b_gn3[None, :]), 0.0) - b_kg3 = b_k3 * b_gqn3 - b_kgt0 = tl.trans(b_k0 * tl.math.exp2(b_gn3[None, :] - b_g0)) - b_Akk30 += tl.dot(b_kg3, b_kgt0) - b_kgt1 = tl.trans(b_k1 * tl.math.exp2(b_gn3[None, :] - b_g1)) - b_Akk31 += tl.dot(b_kg3, b_kgt1) - b_kgt2 = tl.trans(b_k2 * tl.math.exp2(b_gn3[None, :] - b_g2)) - b_Akk32 += tl.dot(b_kg3, b_kgt2) - - beta_base = beta + bos * H + i_h - if i_tc1 < T: - p_b1 = tl.make_block_ptr(beta_base, (T,), (H,), (i_tc1,), (BC,), (0,)) - b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32) - b_Akk10 = b_Akk10 * b_b1[:, None] - if i_tc2 < T: - p_b2 = tl.make_block_ptr(beta_base, (T,), (H,), (i_tc2,), (BC,), (0,)) - b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32) - b_Akk20 = b_Akk20 * b_b2[:, None] - b_Akk21 = b_Akk21 * b_b2[:, None] - if i_tc3 < T: - p_b3 = tl.make_block_ptr(beta_base, (T,), (H,), (i_tc3,), (BC,), (0,)) - b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32) - b_Akk30 = b_Akk30 * b_b3[:, None] - b_Akk31 = b_Akk31 * b_b3[:, None] - b_Akk32 = b_Akk32 * b_b3[:, None] - - # Load 4 inverted diagonal blocks (from sub_chunk kernel) - p_Akk00 = tl.make_block_ptr(Akkd_base, (T, BC), (H * BC, 1), (i_tc0, 0), (BC, BC), (1, 0)) - p_Akk11 = tl.make_block_ptr(Akkd_base, (T, BC), (H * BC, 1), (i_tc1, 0), (BC, BC), (1, 0)) - p_Akk22 = tl.make_block_ptr(Akkd_base, (T, BC), (H * BC, 1), (i_tc2, 0), (BC, BC), (1, 0)) - p_Akk33 = tl.make_block_ptr(Akkd_base, (T, BC), (H * BC, 1), (i_tc3, 0), (BC, BC), (1, 0)) - b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32) - b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32) - b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32) - b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32) - - b_Ai10 = -tl.dot( - tl.dot(b_Ai11, b_Akk10, input_precision=_SOLVE_DOT_PRECISION), - b_Ai00, - input_precision=_SOLVE_DOT_PRECISION, - ) - b_Ai21 = -tl.dot( - tl.dot(b_Ai22, b_Akk21, input_precision=_SOLVE_DOT_PRECISION), - b_Ai11, - input_precision=_SOLVE_DOT_PRECISION, - ) - b_Ai32 = -tl.dot( - tl.dot(b_Ai33, b_Akk32, input_precision=_SOLVE_DOT_PRECISION), - b_Ai22, - input_precision=_SOLVE_DOT_PRECISION, - ) - - b_Ai20 = -tl.dot( - b_Ai22, - tl.dot(b_Akk20, b_Ai00, input_precision=_SOLVE_DOT_PRECISION) - + tl.dot(b_Akk21, b_Ai10, input_precision=_SOLVE_DOT_PRECISION), - input_precision=_SOLVE_DOT_PRECISION, - ) - b_Ai31 = -tl.dot( - b_Ai33, - tl.dot(b_Akk31, b_Ai11, input_precision=_SOLVE_DOT_PRECISION) - + tl.dot(b_Akk32, b_Ai21, input_precision=_SOLVE_DOT_PRECISION), - input_precision=_SOLVE_DOT_PRECISION, - ) - b_Ai30 = -tl.dot( - b_Ai33, - tl.dot(b_Akk30, b_Ai00, input_precision=_SOLVE_DOT_PRECISION) - + tl.dot(b_Akk31, b_Ai10, input_precision=_SOLVE_DOT_PRECISION) - + tl.dot(b_Akk32, b_Ai20, input_precision=_SOLVE_DOT_PRECISION), - input_precision=_SOLVE_DOT_PRECISION, - ) - - # Store 10 blocks to the full BT×BT Akk buffer. - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc0, 0), (BC, BC), (1, 0)) - tl.store(p, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc1, 0), (BC, BC), (1, 0)) - tl.store(p, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc1, BC), (BC, BC), (1, 0)) - tl.store(p, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc2, 0), (BC, BC), (1, 0)) - tl.store(p, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc2, BC), (BC, BC), (1, 0)) - tl.store(p, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)) - tl.store(p, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc3, 0), (BC, BC), (1, 0)) - tl.store(p, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc3, BC), (BC, BC), (1, 0)) - tl.store(p, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)) - tl.store(p, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - p = tl.make_block_ptr(Akk_base, (T, BT), (H * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)) - tl.store(p, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - - -def kda_intra_native( - k: torch.Tensor, # bf16 [1, T, H, K] - v: torch.Tensor, # bf16 [1, T, H, V] - gk: torch.Tensor, # fp32 [1, T, H, K] - beta: torch.Tensor, # bf16 [1, T, H] - chunk_size: int = 64, - q: torch.Tensor | None = None, # bf16 [1, T, H, K], for qg (optional) - need_qg: bool = False, - out_w: torch.Tensor | None = None, # bf16 [1, T, H, K] - out_u: torch.Tensor | None = None, # bf16 [1, T, H, V] - out_kg: torch.Tensor | None = None, # bf16 [1, T, H, K] - out_Akkd: torch.Tensor | None = None, # fp32 [1, T, H, BC] - out_Akk: torch.Tensor | None = None, # bf16 [1, T, H, BT] -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor]: - B, T, H, K = k.shape - V = v.shape[-1] - assert B == 1, "intra expects packed [1, T, H, K] input" - assert K == 128 and V == 128, f"specialized for K=V=128, got K={K} V={V}" - BT = chunk_size - BC = 16 - NT = (T + BT - 1) // BT - NC = BT // BC - - if out_Akkd is not None: - Akkd = out_Akkd[:B, :T] - else: - Akkd = torch.empty(B, T, H, BC, device=k.device, dtype=torch.float32) - if out_Akk is not None: - Akk = out_Akk[:B, :T] - Akk.zero_() - else: - Akk = torch.zeros(B, T, H, BT, device=k.device, dtype=k.dtype) - - # Step 1: per-sub-chunk diagonal Akk inversion - BK_sub = triton.next_power_of_2(K) # =128 for K=128 - grid_sub = (NT, NC, B * H) - _kda_intra_sub_chunk_kernel[grid_sub]( - k=k, - g=gk, - beta=beta, - Akkd=Akkd, - T=T, - H=H, - K=K, - BT=BT, - BC=BC, - BK=BK_sub, - ) - - # Step 2: per-chunk off-diagonal + assemble full Akk_inv - grid_inter = (NT, B * H) - _kda_intra_inter_solve_kernel[grid_inter]( - k=k, - g=gk, - beta=beta, - Akkd=Akkd, - Akk=Akk, - T=T, - H=H, - K=K, - BT=BT, - BC=BC, - ) - if out_Akkd is None: - del Akkd - - # Step 3: recompute w, u, kg (and optionally qg) from Akk - w, u, qg, kg = kda_recompute_w_u( - k=k, - v=v, - beta=beta, - A=Akk, - q=q if need_qg else None, - gk=gk, - chunk_size=chunk_size, - out_w=out_w, - out_u=out_u, - out_kg=out_kg, - ) - # Akk is similarly dead after recompute's kernel is queued. - if out_Akk is None: - del Akk - return w, u, qg, kg diff --git a/cula/kda/wy_recompute.py b/cula/kda/wy_recompute.py deleted file mode 100644 index edfb9693..00000000 --- a/cula/kda/wy_recompute.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import torch -import triton -import triton.language as tl - - -@triton.autotune( - configs=[ - triton.Config({}, num_warps=num_warps, num_stages=num_stages) for num_warps in [2, 4, 8] for num_stages in [2, 3, 4] - ], - key=["H", "K", "V", "BT"], -) -@triton.heuristics( - { - "STORE_QG": lambda args: args["qg"] is not None, - "STORE_KG": lambda args: args["kg"] is not None, - } -) -@triton.jit(do_not_specialize=["T"]) -def _kda_recompute_wuk_kernel( - q, - k, - qg, - kg, - v, - beta, - w, - u, - A, - gk, - T, - H: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BT: tl.constexpr, - STORE_QG: tl.constexpr, - STORE_KG: tl.constexpr, -): - """K = V = 128, BT = 64 specialized. BK = K, BV = V (no inner loop).""" - i_t, i_bh = tl.program_id(0), tl.program_id(1) - i_b, i_h = i_bh // H, i_bh % H - bos = i_b * T - - # Per-head pointer offsets - p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) - b_b = tl.load(p_b, boundary_check=(0,)) - - p_A = tl.make_block_ptr(A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)) - b_A = tl.load(p_A, boundary_check=(0, 1)) - - # ----- u = A @ (β · v) ----- - p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, 0), (BT, V), (1, 0)) - b_v = tl.load(p_v, boundary_check=(0, 1)) - b_vb = (b_v * b_b[:, None]).to(b_v.dtype) - b_u = tl.dot(b_A, b_vb) - p_u = tl.make_block_ptr(u + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, 0), (BT, V), (1, 0)) - tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) - - # ----- Load k, gk, compute β·exp2(gk)·k and kg ----- - p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, 0), (BT, K), (1, 0)) - b_k = tl.load(p_k, boundary_check=(0, 1)) - p_gk = tl.make_block_ptr(gk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, 0), (BT, K), (1, 0)) - b_gk = tl.load(p_gk, boundary_check=(0, 1)).to(tl.float32) - b_exp_gk = tl.math.exp2(b_gk) - - # w = A @ (β · exp2(gk) · k) - b_kb = b_k * b_b[:, None] * b_exp_gk - b_w = tl.dot(b_A, b_kb.to(b_k.dtype)) - p_w = tl.make_block_ptr(w + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, 0), (BT, K), (1, 0)) - tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) - - # qg = q · exp2(gk) (optional) - if STORE_QG: - p_q = tl.make_block_ptr(q + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, 0), (BT, K), (1, 0)) - b_q = tl.load(p_q, boundary_check=(0, 1)) - b_qg = b_q * b_exp_gk - p_qg = tl.make_block_ptr(qg + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, 0), (BT, K), (1, 0)) - tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1)) - - # kg = β · exp2(g_chunk_end - gk) · k (optional, needed by cp_h0 path) - if STORE_KG: - last_idx = tl.minimum(i_t * BT + BT, T) - 1 - o_k = tl.arange(0, K) - b_gn = tl.load(gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=o_k < K, other=0.0).to(tl.float32) - m_t = (i_t * BT + tl.arange(0, BT)) < T - b_kg = b_k * tl.where(m_t[:, None], tl.math.exp2(b_gn[None, :] - b_gk), 0.0) - p_kg = tl.make_block_ptr(kg + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, 0), (BT, K), (1, 0)) - tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1)) - - -def kda_recompute_w_u( - k: torch.Tensor, # bf16 [B, T, H, K] - v: torch.Tensor, # bf16 [B, T, H, V] - beta: torch.Tensor, # bf16 [B, T, H] - A: torch.Tensor, # bf16 [B, T, H, BT] - q: torch.Tensor | None, # bf16 [B, T, H, K], or None - gk: torch.Tensor, # fp32 [B, T, H, K] - chunk_size: int = 64, - out_w: torch.Tensor | None = None, - out_u: torch.Tensor | None = None, - out_kg: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: - B, T, H, K = k.shape - V = v.shape[-1] - BT = chunk_size - assert K == 128 and V == 128, f"specialized for K=V=128, got K={K} V={V}" - assert A.shape[-1] == BT, f"expected A.shape[-1]={BT}, got {A.shape[-1]}" - - w = out_w[:B, :T] if out_w is not None else torch.empty_like(k) - u = out_u[:B, :T] if out_u is not None else torch.empty_like(v) - qg = torch.empty_like(q) if q is not None else None - if gk is not None: - kg = out_kg[:B, :T] if out_kg is not None else torch.empty_like(k) - else: - kg = None - - NT = triton.cdiv(T, BT) - grid = (NT, B * H) - _kda_recompute_wuk_kernel[grid]( - q=q, - k=k, - qg=qg, - kg=kg, - v=v, - beta=beta, - w=w, - u=u, - A=A, - gk=gk, - T=T, - H=H, - K=K, - V=V, - BT=BT, - ) - return w, u, qg, kg diff --git a/cula/kernels/__init__.py b/cula/kernels/__init__.py new file mode 100644 index 00000000..5631d0dc --- /dev/null +++ b/cula/kernels/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Framework-independent public surface for cuLA's CuTeDSL kernels. + +The submodules use lazy exports so importing :mod:`cula.kernels` does not +compile kernels or import an external linear-attention framework. +""" + +from cula.kernels.registry import KernelSpec, get_kernel, list_kernels + +__all__ = ["KernelSpec", "get_kernel", "list_kernels"] diff --git a/cula/kernels/_lazy.py b/cula/kernels/_lazy.py new file mode 100644 index 00000000..4fd543d0 --- /dev/null +++ b/cula/kernels/_lazy.py @@ -0,0 +1,17 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared lazy-export support for public kernel namespaces.""" + +from __future__ import annotations + +import importlib +from collections.abc import Mapping +from typing import Any + + +def resolve(mapping: Mapping[str, tuple[str, str]], module_name: str, name: str) -> Any: + target = mapping.get(name) + if target is None: + raise AttributeError(f"module {module_name!r} has no attribute {name!r}") + return getattr(importlib.import_module(target[0]), target[1]) diff --git a/cula/kernels/kda.py b/cula/kernels/kda.py new file mode 100644 index 00000000..6c47a848 --- /dev/null +++ b/cula/kernels/kda.py @@ -0,0 +1,64 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Standalone KDA operator and atomic-kernel exports.""" + +from cula.kernels._lazy import resolve + +_LAZY = { + # Complete inference operators. + "kda_prefill": ("cula.kda.prefill", "kda_prefill"), + "kda_decode": ("cula.ops.kda.decode.cute", "kda_decode"), + "kda_packed_decode": ("cula.ops.kda.decode.cute", "kda_packed_decode"), + "kda_decode_mtp": ("cula.ops.kda.decode.mtp", "kda_decode_mtp"), + "kda_decode_mtp_recurrent": ("cula.ops.kda.decode.mtp", "kda_decode_mtp_recurrent"), + "kda_decode_mtp_recurrent_ws": ("cula.ops.kda.decode.mtp", "kda_decode_mtp_recurrent_ws"), + "kda_conv_decode_mtp_verify": ("cula.ops.kda.decode.mtp_conv", "kda_conv_decode_mtp_verify"), + "kda_conv_decode_mtp_kvbuffer": ( + "cula.ops.kda.decode.mtp_conv_kvbuffer", + "kda_conv_decode_mtp_kvbuffer", + ), + "kda_flush_kvbuffer": ("cula.ops.kda.decode.mtp_kvbuffer", "kda_flush_kvbuffer"), + "kda_flush_kvbuffer_all_layers": ( + "cula.ops.kda.decode.mtp_kvbuffer", + "kda_flush_kvbuffer_all_layers", + ), + # SM100 modular forward/training primitives. + "chunk_kda_fwd_intra": ("cula.kda.chunk_intra", "chunk_kda_fwd_intra"), + "chunk_kda_fwd_intra_sm100_from_gk": ( + "cula.ops.kda.sm100.intra_fused", + "chunk_kda_fwd_intra_sm100_from_gk", + ), + "chunk_kda_fwd_full_sm100_from_gk": ( + "cula.ops.kda.sm100.full_port", + "chunk_kda_fwd_full_sm100_from_gk", + ), + "recompute_w_u_fwd": ("cula.ops.kda.sm100.recompute_wu", "recompute_w_u_fwd"), + "recompute_w_u_from_preprocessed": ( + "cula.ops.kda.sm100.recompute_wu", + "recompute_w_u_from_preprocessed", + ), + "chunk_gated_delta_rule_fwd_h": ( + "cula.ops.kda.sm100.delta_h", + "chunk_gated_delta_rule_fwd_h", + ), + "chunk_gla_fwd_o": ("cula.ops.kda.sm100.fwd_o", "chunk_gla_fwd_o"), + "kda_bwd_wy_dqkg_sm100": ( + "cula.ops.kda.sm100.bwd_wy_dqkg", + "chunk_kda_bwd_wy_dqkg_fused", + ), + "kda_bwd_wy_dqkg_sm90": ( + "cula.ops.kda.sm90.bwd_wy_dqkg", + "chunk_kda_bwd_wy_dqkg_fused", + ), +} + +__all__ = sorted(_LAZY) + + +def __getattr__(name): + return resolve(_LAZY, __name__, name) + + +def __dir__(): + return __all__ diff --git a/cula/kernels/lightning.py b/cula/kernels/lightning.py new file mode 100644 index 00000000..b6308753 --- /dev/null +++ b/cula/kernels/lightning.py @@ -0,0 +1,35 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Standalone Lightning Attention operator exports.""" + +from cula.kernels._lazy import resolve + +_LAZY = { + "lightning_attn_fwd": ("cula.ops.lightning.prefill", "lightning_attn_fwd"), + "lightning_attn_fwd_varlen": ("cula.ops.lightning.prefill", "lightning_attn_fwd_varlen"), + "linear_attention_decode": ("cula.ops.lightning.decode", "linear_attention_decode"), + "linear_attention_decode_mtp": ("cula.lightning.la_decode_mtp", "linear_attention_decode_mtp"), + "linear_attention_verify_kvbuffer": ( + "cula.lightning.la_verify_kvbuffer", + "linear_attention_verify_kvbuffer", + ), + "linear_attention_state_update_kvbuffer": ( + "cula.lightning.la_state_update_kvbuffer", + "linear_attention_state_update_kvbuffer", + ), + "linear_attention_state_update_kvbuffer_fused": ( + "cula.lightning.la_state_update_kvbuffer", + "linear_attention_state_update_kvbuffer_fused", + ), +} + +__all__ = sorted(_LAZY) + + +def __getattr__(name): + return resolve(_LAZY, __name__, name) + + +def __dir__(): + return __all__ diff --git a/cula/kernels/registry.py b/cula/kernels/registry.py new file mode 100644 index 00000000..99f85655 --- /dev/null +++ b/cula/kernels/registry.py @@ -0,0 +1,229 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Static inventory for the supported standalone kernel surface.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +KernelFamily = Literal["kda", "lightning"] +KernelLevel = Literal["operator", "composite", "primitive"] +KernelStatus = Literal["stable", "specialized", "experimental"] + + +@dataclass(frozen=True, slots=True) +class KernelSpec: + name: str + family: KernelFamily + level: KernelLevel + architectures: tuple[str, ...] + status: KernelStatus + summary: str + + +_KERNELS = ( + KernelSpec("kda_prefill", "kda", "operator", ("sm90", "sm100", "sm103"), "stable", "Forward-only FlashKDA prefill."), + KernelSpec("kda_decode", "kda", "operator", ("sm90+",), "stable", "Single-token and packed recurrent decode."), + KernelSpec("kda_packed_decode", "kda", "operator", ("sm90+",), "stable", "Packed recurrent decode."), + KernelSpec("kda_decode_mtp", "kda", "operator", ("sm90+",), "specialized", "Multi-token prediction verify/decode."), + KernelSpec( + "kda_decode_mtp_recurrent", + "kda", + "operator", + ("sm90+",), + "specialized", + "Recurrent multi-token prediction decode.", + ), + KernelSpec( + "kda_decode_mtp_recurrent_ws", + "kda", + "operator", + ("sm90+",), + "specialized", + "Workspace-managed recurrent multi-token prediction decode.", + ), + KernelSpec( + "kda_conv_decode_mtp_verify", + "kda", + "operator", + ("sm90+",), + "specialized", + "Fused short-convolution and KDA MTP verification.", + ), + KernelSpec( + "kda_conv_decode_mtp_kvbuffer", + "kda", + "operator", + ("sm90+",), + "specialized", + "Fused short-convolution, KDA MTP, and KV-buffer update.", + ), + KernelSpec( + "kda_flush_kvbuffer", + "kda", + "operator", + ("sm90+",), + "specialized", + "Flush accepted speculative tokens into one KDA KV buffer.", + ), + KernelSpec( + "kda_flush_kvbuffer_all_layers", + "kda", + "operator", + ("sm90+",), + "specialized", + "Flush accepted speculative tokens across KDA layer buffers.", + ), + KernelSpec( + "chunk_kda_fwd_intra", + "kda", + "composite", + ("sm100", "sm103"), + "specialized", + "Chunk intra matrices plus W/U preparation from preprocessed gates.", + ), + KernelSpec( + "chunk_kda_fwd_intra_sm100_from_gk", + "kda", + "primitive", + ("sm100", "sm103"), + "specialized", + "Bitwise-oriented Aqk/Akk construction from cumulative FP32 gates.", + ), + KernelSpec( + "chunk_kda_fwd_full_sm100_from_gk", + "kda", + "composite", + ("sm100", "sm103"), + "specialized", + "Optimized fixed-length KDA intra and W/U composition.", + ), + KernelSpec( + "recompute_w_u_fwd", + "kda", + "primitive", + ("sm100", "sm103"), + "specialized", + "W/U/KG preparation for modular chunk KDA.", + ), + KernelSpec( + "recompute_w_u_from_preprocessed", + "kda", + "primitive", + ("sm100", "sm103"), + "specialized", + "W/U recomputation from preprocessed KDA intermediates.", + ), + KernelSpec( + "chunk_gated_delta_rule_fwd_h", + "kda", + "primitive", + ("sm100", "sm103"), + "specialized", + "Inter-chunk recurrent-state propagation.", + ), + KernelSpec( + "chunk_gla_fwd_o", + "kda", + "primitive", + ("sm100", "sm103"), + "specialized", + "Chunk output projection from recurrent state and intra attention.", + ), + KernelSpec( + "kda_bwd_wy_dqkg_sm100", + "kda", + "primitive", + ("sm100", "sm103"), + "experimental", + "Fused W/Y-related KDA backward gradients.", + ), + KernelSpec( + "kda_bwd_wy_dqkg_sm90", + "kda", + "primitive", + ("sm90",), + "experimental", + "Hopper fused W/Y-related KDA backward gradients.", + ), + KernelSpec( + "lightning_attn_fwd", + "lightning", + "operator", + ("sm90", "sm100", "sm103"), + "stable", + "Fixed-length Lightning Attention prefill.", + ), + KernelSpec( + "lightning_attn_fwd_varlen", + "lightning", + "operator", + ("sm90", "sm100", "sm103"), + "stable", + "Packed variable-length Lightning Attention prefill.", + ), + KernelSpec( + "linear_attention_decode", + "lightning", + "operator", + ("sm90+",), + "stable", + "Recurrent Lightning Attention decode.", + ), + KernelSpec( + "linear_attention_decode_mtp", + "lightning", + "operator", + ("sm90+",), + "specialized", + "Multi-token Lightning Attention verification.", + ), + KernelSpec( + "linear_attention_verify_kvbuffer", + "lightning", + "operator", + ("sm90+",), + "specialized", + "MTP verification against a pooled state/KV buffer.", + ), + KernelSpec( + "linear_attention_state_update_kvbuffer", + "lightning", + "operator", + ("sm90+",), + "specialized", + "Update a Lightning Attention state/KV buffer.", + ), + KernelSpec( + "linear_attention_state_update_kvbuffer_fused", + "lightning", + "operator", + ("sm90+",), + "specialized", + "Fused Lightning Attention state/KV-buffer update.", + ), +) + +_BY_NAME = {spec.name: spec for spec in _KERNELS} + + +def list_kernels(*, family: KernelFamily | None = None, level: KernelLevel | None = None) -> tuple[KernelSpec, ...]: + """Return immutable kernel metadata, optionally filtered by family/level.""" + + return tuple( + spec for spec in _KERNELS if (family is None or spec.family == family) and (level is None or spec.level == level) + ) + + +def get_kernel(name: str) -> KernelSpec: + """Return one kernel specification by public name.""" + + try: + return _BY_NAME[name] + except KeyError as error: + raise KeyError(f"unknown standalone kernel {name!r}") from error + + +__all__ = ["KernelSpec", "get_kernel", "list_kernels"] diff --git a/cula/ops/kda/decode/reference_fla.py b/cula/ops/kda/decode/reference_fla.py deleted file mode 100644 index 2c4d0cfc..00000000 --- a/cula/ops/kda/decode/reference_fla.py +++ /dev/null @@ -1,234 +0,0 @@ -import torch -import triton -import triton.language as tl - - -@triton.jit(do_not_specialize=["T"]) -def fused_sigmoid_gating_delta_rule_update_kernel( - A_log, - a, - dt_bias, - softplus_beta, - softplus_threshold, - q, - k, - v, - b, - o, - h0_source, - h0_indices, - cu_seqlens, - scale, - T, - B: tl.constexpr, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BK: tl.constexpr, - BV: tl.constexpr, - USE_INITIAL_STATE: tl.constexpr, - USE_QK_L2NORM_IN_KERNEL: tl.constexpr, - IS_VARLEN: tl.constexpr, - IS_KDA: tl.constexpr, -): - """ - Fused kernel that combines sigmoid gating computation with recurrent delta rule update. - """ - i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2) - i_n, i_hv = i_nh // HV, i_nh % HV - i_h = i_hv // (HV // H) - - if IS_VARLEN: - bos, eos = ( - tl.load(cu_seqlens + i_n).to(tl.int64), - tl.load(cu_seqlens + i_n + 1).to(tl.int64), - ) - all = T - T = eos - bos - else: - bos, eos = i_n * T, i_n * T + T - all = B * T - - o_k = i_k * BK + tl.arange(0, BK) - o_v = i_v * BV + tl.arange(0, BV) - - p_q = q + (bos * H + i_h) * K + o_k - p_k = k + (bos * H + i_h) * K + o_k - p_v = v + (bos * HV + i_hv) * V + o_v - p_b = b + bos * HV + i_hv - p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v - - # Gating computation pointers - p_A_log = A_log + i_hv - if IS_KDA: - # For KDA, a has shape [T, HV*K] (flattened), so stride is HV*K per token - p_a = a + bos * HV * K + i_hv * K + o_k - p_dt_bias = dt_bias + i_hv * K + o_k - else: - p_a = a + bos * HV + i_hv - p_dt_bias = dt_bias + i_hv - - mask_k = o_k < K - mask_v = o_v < V - mask_h = mask_k[:, None] & mask_v[None, :] - - b_h = tl.zeros([BK, BV], dtype=tl.float32) - if USE_INITIAL_STATE: - idx = tl.load(h0_indices + i_n) - if idx >= 0: - p_h0 = h0_source + idx * HV * K * V + i_hv * K * V + o_k[:, None] * V + o_v[None, :] - b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) - - for _ in range(0, T): - # Load inputs - b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32) - b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32) - b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32) - b_b = tl.load(p_b).to(tl.float32) - - # Compute sigmoid gating - # Load gating parameters - b_A_log = tl.load(p_A_log).to(tl.float32) - b_a = tl.load(p_a).to(tl.float32) - b_dt_bias = tl.load(p_dt_bias).to(tl.float32) - - # Compute g = -exp(A_log) * softplus(a + dt_bias) - x = b_a + b_dt_bias - beta_x = softplus_beta * x - # Apply softplus with numerical stability - softplus_x = tl.where( - beta_x <= softplus_threshold, - (1.0 / softplus_beta) * tl.log(1.0 + tl.exp(beta_x)), - x, - ) - b_g = -tl.exp(b_A_log) * softplus_x - - # Compute beta = sigmoid(b) - b_beta = 1.0 / (1.0 + tl.exp(-b_b)) - - # Apply L2 normalization if enabled - if USE_QK_L2NORM_IN_KERNEL: - b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q) + 1e-6)) - b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k) + 1e-6)) - - b_q = b_q * scale - - # Apply gating to hidden state: h *= exp(g) - if IS_KDA: - b_h *= tl.exp(b_g[:, None]) - else: - b_h *= tl.exp(b_g) - - # Delta rule: v -= sum(h * k, dim=0) - b_v -= tl.sum(b_h * b_k[:, None], 0) - - # Apply beta gating: v *= beta - b_v *= b_beta - - # Update hidden state: h += k[:, None] * v[None, :] - b_h += b_k[:, None] * b_v[None, :] - - # Compute output: o = sum(h * q, dim=0) - b_o = tl.sum(b_h * b_q[:, None], 0) - tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v) - - # Update pointers for next timestep - p_q += H * K - p_k += H * K - p_o += HV * V - p_v += HV * V - p_b += HV - if IS_KDA: - p_a += HV * K - else: - p_a += HV - - # Store final state back to h0_source with bounds checking - if USE_INITIAL_STATE: - idx = tl.load(h0_indices + i_n) - if idx >= 0: - p_h0 = h0_source + idx * HV * K * V + i_hv * K * V + o_k[:, None] * V + o_v[None, :] - tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) - - -def fused_sigmoid_gating_delta_rule_update( - A_log: torch.Tensor, - a: torch.Tensor, - dt_bias: torch.Tensor, - softplus_beta: float, - softplus_threshold: float, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - b: torch.Tensor, - initial_state_source: torch.Tensor, - initial_state_indices: torch.Tensor, - scale: float | None = None, - use_qk_l2norm_in_kernel: bool = False, - cu_seqlens: torch.Tensor | None = None, - is_kda: bool = False, -): - """ - Fused triton implementation of sigmoid gating delta rule update. - This function uses a single fused kernel that combines both sigmoid gating computation - and the recurrent delta rule update for better performance. - """ - B, T, H, K, V = *k.shape, v.shape[-1] - HV = v.shape[2] - N = B if cu_seqlens is None else len(cu_seqlens) - 1 - BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 32) - NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) - assert NK == 1, "NK > 1 is not supported yet" - num_stages = 3 - num_warps = 1 - - if scale is None: - scale = k.shape[-1] ** -0.5 - else: - assert scale > 0, "scale must be positive" - - # Kernel needs valid tensors for h0_source and h0_indices even when not using initial state. - # Create dummy tensors if needed. - use_initial_state = initial_state_source is not None - if initial_state_indices is None: - initial_state_indices = torch.full((N,), -1, dtype=torch.int32, device=q.device) - if initial_state_source is None: - # Create a dummy state buffer with minimal shape - initial_state_source = torch.zeros(1, HV, K, V, dtype=q.dtype, device=q.device) - - o = q.new_empty(NK, *v.shape) - grid = (NK, NV, N * HV) - - fused_sigmoid_gating_delta_rule_update_kernel[grid]( - A_log=A_log, - a=a, - dt_bias=dt_bias, - softplus_beta=softplus_beta, - softplus_threshold=softplus_threshold, - q=q, - k=k, - v=v, - b=b, - o=o, - h0_source=initial_state_source, - h0_indices=initial_state_indices, - cu_seqlens=cu_seqlens, - scale=scale, - T=T, - B=B, - H=H, - HV=HV, - K=K, - V=V, - BK=BK, - BV=BV, - USE_INITIAL_STATE=use_initial_state, - USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, - IS_VARLEN=cu_seqlens is not None, - IS_KDA=is_kda, - num_warps=num_warps, - num_stages=num_stages, - ) - o = o.squeeze(0) - return o diff --git a/cula/ops/kda/index.py b/cula/ops/kda/index.py new file mode 100644 index 00000000..814b6af9 --- /dev/null +++ b/cula/ops/kda/index.py @@ -0,0 +1,116 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Small tensor-index helpers shared by the standalone KDA kernels. + +These functions intentionally use only PyTorch. Keeping this module local to +cuLA prevents low-level CuTeDSL kernels from acquiring a runtime dependency on +a higher-level linear-attention framework. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from typing import Any + +import torch + + +def single_entry_tensor_cache(fn: Callable[..., Any]) -> Callable[..., Any]: + """Cache the most recent call by object identity. + + Metadata helpers are commonly invoked several times with the same + ``cu_seqlens`` tensor during one operator dispatch. An identity cache is + sufficient for that case and, unlike a value cache, never synchronizes a + CUDA tensor merely to build a key. + """ + + entry: tuple[tuple[Any, ...], dict[str, Any], tuple[int | None, ...], Any] | None = None + + def tensor_versions(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[int | None, ...]: + versions = [] + for value in (*args, *kwargs.values()): + if not isinstance(value, torch.Tensor): + versions.append(None) + continue + try: + versions.append(value._version) + except RuntimeError: + # Inference tensors do not expose a version counter. Treat + # them as uncacheable rather than risking stale metadata. + versions.append(None) + return tuple(versions) + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + nonlocal entry + cached = entry + if cached is not None: + cached_args, cached_kwargs, cached_versions, cached_result = cached + same_args = len(args) == len(cached_args) and all( + arg is cached_arg for arg, cached_arg in zip(args, cached_args, strict=True) + ) + same_kwargs = len(kwargs) == len(cached_kwargs) and all( + key in cached_kwargs and value is cached_kwargs[key] for key, value in kwargs.items() + ) + versions = tensor_versions(args, kwargs) + versions_available = all( + version is not None + for value, version in zip((*args, *kwargs.values()), versions, strict=True) + if isinstance(value, torch.Tensor) + ) + if same_args and same_kwargs and versions_available and versions == cached_versions: + return cached_result + + result = fn(*args, **kwargs) + entry = (args, kwargs, tensor_versions(args, kwargs), result) + return result + + return wrapper + + +@single_entry_tensor_cache +def prepare_lens(cu_seqlens: torch.Tensor) -> torch.Tensor: + """Return per-sequence lengths for a cumulative-length tensor.""" + + if cu_seqlens.ndim != 1: + raise ValueError(f"cu_seqlens must be one-dimensional, got shape {tuple(cu_seqlens.shape)}") + return torch.diff(cu_seqlens) + + +@single_entry_tensor_cache +def prepare_chunk_indices( + cu_seqlens: torch.Tensor, + chunk_size: int, + cu_seqlens_cpu: torch.Tensor | None = None, +) -> torch.Tensor: + """Map each logical chunk to ``(sequence_id, local_chunk_id)``. + + The output is an integer tensor of shape ``[num_chunks, 2]`` on the same + device and with the same dtype as ``cu_seqlens``. ``cu_seqlens_cpu`` may + be supplied to avoid a device-to-host synchronization while determining + how many chunks belong to each sequence. + """ + + if not isinstance(chunk_size, int) or isinstance(chunk_size, bool) or chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}") + source = cu_seqlens if cu_seqlens_cpu is None else cu_seqlens_cpu + if source.ndim != 1: + raise ValueError(f"cu_seqlens must be one-dimensional, got shape {tuple(source.shape)}") + if source.numel() == 0: + raise ValueError("cu_seqlens must contain at least one element") + + counts = torch.div(prepare_lens(source) + chunk_size - 1, chunk_size, rounding_mode="floor") + repeat_counts = counts.to(torch.int64) + sequence_ids = torch.repeat_interleave( + torch.arange(counts.numel(), dtype=source.dtype, device=source.device), repeat_counts + ) + sequence_starts = torch.cumsum(counts, dim=0) - counts + local_chunk_ids = torch.arange(sequence_ids.numel(), dtype=source.dtype, device=source.device) - ( + torch.repeat_interleave(sequence_starts, repeat_counts) + ) + return torch.stack((sequence_ids, local_chunk_ids), dim=-1).to(dtype=cu_seqlens.dtype, device=cu_seqlens.device) + + +__all__ = ["prepare_chunk_indices", "prepare_lens", "single_entry_tensor_cache"] diff --git a/cula/ops/kda/sm100/bwd_wy_dqkg.py b/cula/ops/kda/sm100/bwd_wy_dqkg.py index 4ab6bf4c..8124099b 100644 --- a/cula/ops/kda/sm100/bwd_wy_dqkg.py +++ b/cula/ops/kda/sm100/bwd_wy_dqkg.py @@ -22,8 +22,8 @@ from cutlass.cute.runtime import make_fake_compact_tensor, make_fake_stream from cutlass.cute.tensor import TensorSSA from cutlass.cute.typing import BFloat16, Float32, Int32, Int64 -from fla.ops.utils import prepare_chunk_indices +from cula.ops.kda.index import prepare_chunk_indices from cula.ops.ptx import reinterpret_cast, store_256b, subvec from cula.ops.sm100.ptx import ( Tcgen05SmemDescriptor, diff --git a/cula/ops/kda/sm100/cp/pre_scan.py b/cula/ops/kda/sm100/cp/pre_scan.py index 709086fa..559ec4ab 100644 --- a/cula/ops/kda/sm100/cp/pre_scan.py +++ b/cula/ops/kda/sm100/cp/pre_scan.py @@ -1279,57 +1279,6 @@ def run_bench(): elapsed_ms = start_event.elapsed_time(end_event) / n_iter print(f" cuLA pre_scan: {elapsed_ms:.3f} ms") - # FLA Triton kernel reference (call raw kernel directly) - try: - import triton - from fla.ops.cp.chunk_delta_h import pre_process_fwd_kernel_merged as fla_kernel - - BLOCK_SIZE_FLA = 32 if K <= 64 else 64 - BK1_FLA = triton.next_power_of_2(K) - fla_grid = (triton.cdiv(V, BLOCK_SIZE_FLA) + triton.cdiv(K, BLOCK_SIZE_FLA), S_split * H) - - # FLA expects [T, H, K/V] layout (no batch dim), HV=H for this case - k_fla = k_b[0] # [T, H, K] - w_fla = w_b[0] # [T, H, K] - u_fla = u_b[0] # [T, H, V] - gk_fla = gk_b[0] # [T, H, K] - hm_fla = torch.empty(S_split, H, K, V + K, device=device, dtype=torch.float32) - - def run_fla(): - fla_kernel[fla_grid]( - k=k_fla, - v=u_fla, - w=w_fla, - g=None, - gk=gk_fla, - hm=hm_fla, - cu_seqlens=cu_seqlens, - T=T, - H=H, - HV=H, - K=K, - V=V, - BT=BT, - BK1=BK1_FLA, - BLOCK_SIZE=BLOCK_SIZE_FLA, - USE_EXP2=True, - MULTI_SEQS=True, - ) - - for _ in range(3): - run_fla() - torch.cuda.synchronize() - start_event.record() - for _ in range(n_iter): - run_fla() - end_event.record() - torch.cuda.synchronize() - fla_ms = start_event.elapsed_time(end_event) / n_iter - print(f" FLA pre_scan: {fla_ms:.3f} ms") - print(f" Speedup vs FLA: {fla_ms / elapsed_ms:.2f}x") - except Exception as e: - print(f" FLA not available: {e}") - if __name__ == "__main__": main() diff --git a/cula/ops/kda/sm100/delta_h.py b/cula/ops/kda/sm100/delta_h.py index be573f9e..455552e6 100644 --- a/cula/ops/kda/sm100/delta_h.py +++ b/cula/ops/kda/sm100/delta_h.py @@ -26,13 +26,11 @@ import cutlass.utils.blackwell_helpers as sm100_utils import torch import torch.nn.functional as F -import triton from cutlass.cute.nvgpu import cpasync, tcgen05 from cutlass.cute.runtime import make_fake_compact_tensor, make_fake_stream from cutlass.cute.typing import Float32, Int32, Int64 -from fla.ops.utils import prepare_chunk_indices, prepare_lens -from fla.utils import tensor_cache +from cula.ops.kda.index import prepare_chunk_indices, prepare_lens, single_entry_tensor_cache from cula.ops.kda.sm100.policy import sm100_intracard_cp_decision from cula.utils import USE_FAST_MATH, assert_blackwell, get_device_sm_count @@ -40,12 +38,13 @@ # in FLA, cumsum returns int64 tensor by default -@tensor_cache +@single_entry_tensor_cache def prepare_chunk_offsets_i32( cu_seqlens: torch.Tensor, chunk_size: int, ) -> torch.Tensor: - return F.pad(triton.cdiv(prepare_lens(cu_seqlens), chunk_size), (1, 0), value=0).cumsum(-1).int() + chunk_counts = torch.div(prepare_lens(cu_seqlens) + chunk_size - 1, chunk_size, rounding_mode="floor") + return F.pad(chunk_counts, (1, 0), value=0).cumsum(-1).int() PRINT_DEBUG = False @@ -2491,45 +2490,6 @@ def run_bench(): elapsed_ms = start_event.elapsed_time(end_event) / n_iter print(f" V2 kernel: {elapsed_ms:.3f} ms") - # FLA h-kernel reference (apples-to-apples: h-state recurrence only) - try: - from fla.ops.common.chunk_delta_h import chunk_gated_delta_rule_fwd_h as fla_fwd_h - - # Warmup - for _ in range(3): - fla_fwd_h( - k=kb, - w=wb, - u=ub, - g=None, - gk=None, - initial_state=None, - output_final_state=False, - chunk_size=BT, - save_new_value=True, - ) - torch.cuda.synchronize() - start_event.record() - for _ in range(n_iter): - fla_fwd_h( - k=kb, - w=wb, - u=ub, - g=None, - gk=None, - initial_state=None, - output_final_state=False, - chunk_size=BT, - save_new_value=True, - ) - end_event.record() - torch.cuda.synchronize() - fla_ms = start_event.elapsed_time(end_event) / n_iter - print(f" FLA h-kernel: {fla_ms:.3f} ms") - print(f" Speedup vs FLA h-kernel: {fla_ms / elapsed_ms:.2f}x") - except Exception as e: - print(f" FLA not available: {e}") - if __name__ == "__main__": main() diff --git a/cula/ops/kda/sm100/fwd_o.py b/cula/ops/kda/sm100/fwd_o.py index 0fd4bbd8..bc3219aa 100644 --- a/cula/ops/kda/sm100/fwd_o.py +++ b/cula/ops/kda/sm100/fwd_o.py @@ -77,8 +77,8 @@ from cutlass.cute.nvgpu import cpasync, tcgen05 from cutlass.cute.runtime import make_fake_compact_tensor, make_fake_stream from cutlass.cute.typing import Float32, Int32, Int64 -from fla.ops.utils import prepare_chunk_indices +from cula.ops.kda.index import prepare_chunk_indices from cula.utils import USE_FAST_MATH, assert_blackwell PRINT_DEBUG = False @@ -2092,26 +2092,6 @@ def main(): ms = start.elapsed_time(end) / N_iters print(f" CuTe DSL T={bench_T}: {ms:.3f} ms") - try: - from fla.ops.gla.chunk import chunk_gla_fwd_o_gk - - for _ in range(10): - chunk_gla_fwd_o_gk( - q=q_b, v=v_b, g=g_b, A=A_b, h=h_b.flatten(0, 1), scale=scale, chunk_size=BT, use_exp2=True - ) - torch.cuda.synchronize() - start.record() - for _ in range(N_iters): - chunk_gla_fwd_o_gk( - q=q_b, v=v_b, g=g_b, A=A_b, h=h_b.flatten(0, 1), scale=scale, chunk_size=BT, use_exp2=True - ) - end.record() - torch.cuda.synchronize() - ms_fla = start.elapsed_time(end) / N_iters - print(f" Triton T={bench_T}: {ms_fla:.3f} ms speedup={ms_fla / ms:.2f}x") - except Exception as e: - print(f" Triton benchmark failed: {e}") - if __name__ == "__main__": main() diff --git a/cula/ops/kda/sm100/intra_fused.py b/cula/ops/kda/sm100/intra_fused.py index a1fbeb5b..cb34f6eb 100644 --- a/cula/ops/kda/sm100/intra_fused.py +++ b/cula/ops/kda/sm100/intra_fused.py @@ -74,8 +74,7 @@ from cutlass._mlir.dialects import llvm from cutlass.cute.nvgpu import cpasync, tcgen05 from cutlass.cutlass_dsl import T, dsl_user_op -from fla.ops.utils import prepare_chunk_indices - +from cula.ops.kda.index import prepare_chunk_indices from cula.ops.kda.sm100.akk_inv_fp32 import akk_inv_fp32_physical_host as _akk_inv_fp32_physical_host from cula.ops.kda.sm100.akk_inv_tf32 import _add_store_C16_smem as _tf32_add_store_C16_smem from cula.ops.kda.sm100.akk_inv_tf32 import _invert_diag_forward16 as _tf32_invert_diag_forward16 @@ -2036,8 +2035,6 @@ def host_fn( # High-level Python wrappers for training usage # ============================================================================ -from fla.ops.utils import prepare_chunk_indices - _INTRA_CACHE = {} _VARLEN_PAD_CACHE = {} _EXACT_PORT_OUTPUT_POOLS = {} diff --git a/cula/ops/kda/sm100/recompute_wu.py b/cula/ops/kda/sm100/recompute_wu.py index 0775c0e8..a0c5c433 100644 --- a/cula/ops/kda/sm100/recompute_wu.py +++ b/cula/ops/kda/sm100/recompute_wu.py @@ -43,8 +43,8 @@ from cutlass.cute.runtime import make_fake_compact_tensor, make_fake_stream from cutlass.cute.tensor import TensorSSA from cutlass.cute.typing import Int32, Int64 -from fla.ops.utils import prepare_chunk_indices +from cula.ops.kda.index import prepare_chunk_indices from cula.ops.ptx import reinterpret_cast, store_256b, subvec from cula.ops.sm100.ptx import tcgen05_fence_after, tcgen05_fence_before, tcgen05_ld_32x32b from cula.utils import USE_FAST_MATH, assert_blackwell diff --git a/cula/ops/kda/sm90/bwd_wy_dqkg.py b/cula/ops/kda/sm90/bwd_wy_dqkg.py index be6fd334..3e63a964 100644 --- a/cula/ops/kda/sm90/bwd_wy_dqkg.py +++ b/cula/ops/kda/sm90/bwd_wy_dqkg.py @@ -3543,8 +3543,7 @@ def chunk_kda_bwd_wy_dqkg_fused( Returns: (dq, dk, dv, db, dg, dA), matching FLA's output order. """ - from fla.ops.utils.index import prepare_chunk_indices - + from cula.ops.kda.index import prepare_chunk_indices from cula.utils import prepare_uniform_cu_seqlens B, T, H, K = q.shape diff --git a/docs/KERNEL_VALUE.md b/docs/KERNEL_VALUE.md new file mode 100644 index 00000000..32de12c3 --- /dev/null +++ b/docs/KERNEL_VALUE.md @@ -0,0 +1,60 @@ +# Standalone kernel inventory and overlap assessment + +This document records why each cuLA kernel family exists after separating the +CuTeDSL runtime from FLA. It reflects upstream `main` as inspected on +2026-10-26; upstream coverage and dispatch policy can change quickly. + +## Decision summary + +| cuLA surface | Keep? | Overlap | Why it still matters | +|---|---:|---|---| +| Lightning Attention prefill, decode, MTP and state-buffer kernels | **Yes — primary** | No production implementation was found in FlashInfer; its Lightning issue remains open. No cuDNN linear-attention engine documents Lightning Attention. | This is the clearest independent product surface: SM90 + SM100, fixed/packed prefill, decode and serving-oriented MTP/state-buffer primitives. | +| SM90 FlashKDA prefill and intracard CP | **Yes — primary** | FlashInfer's current KDA prefill dispatch documents SM100/SM103/SM120 paths; cuDNN FROST KDA starts at SM100. | Maintains a Hopper path and a single-GPU CP implementation for long sequences. | +| KDA MTP/KV-buffer/conv decode | **Keep selectively** | FlashInfer now exposes recurrent, packed and fused speculative KDA decode, including fused short convolution and accepted-token cache updates. cuDNN does not document a KDA decode engine. | Useful as decomposed kernels, for custom state layouts, and as an integration-independent reference. It is no longer differentiated as a complete serving path. | +| SM100 KDA prefill | **Keep as an alternative** | FlashInfer has CuTeDSL/Cake prefill and can dispatch to cuDNN; cuDNN FROST is the native full-operator implementation on SM100/SM103. | cuLA remains smaller, source-readable, and supports explicit atomic composition and CP experiments. It should not claim broad backend superiority without shape-by-shape data. | +| SM100 modular `intra`, `recompute_w_u`, `delta_h`, `fwd_o` | **Yes — primitives** | Their combined mathematical operator is covered by cuDNN FROST KDA forward/backward. | Stable, individually benchmarkable building blocks are valuable for algorithm research, precision regression, scheduling experiments, and embedding into a larger fused pipeline. | +| KDA WY backward primitives | **Research only** | cuDNN documents KDA forward and backward, with checkpoint recomputation; FlashInfer intentionally moved training ownership to cuDNN Frontend. | Retain only while they provide measurable value for kernel research or a needed boundary that cuDNN does not expose. Do not position them as a complete training stack. | +| FLA/Triton orchestration and vendored FLA submodule | **Remove** | FLA was framework glue and a reference implementation rather than a CuTeDSL kernel. | It made the runtime import graph larger and obscured which code was actually owned by cuLA. Reference comparisons can install FLA separately. | + +## Upstream evidence + +- [FlashInfer's current KDA facade](https://github.com/flashinfer-ai/flashinfer/blob/main/flashinfer/kda.py) + covers prefill, decode and speculative decode, with CuTeDSL, Cake and cuDNN + backends selected by contract and shape. +- [FlashInfer v0.7.1 release notes](https://github.com/flashinfer-ai/flashinfer/releases/tag/v0.7.1) + describe packed fused KDA speculative verification with short convolution, + accepted-token cache updates, CUDA Graph capture and FP32/BF16 state. +- [cuDNN Frontend's Python engine documentation](https://docs.nvidia.com/cudnn/latest/utilities/python_graph_and_execution_backends/) + documents `KdaFrostEngine` and `KdaCuTileEngine`, including KDA forward, + backward and checkpoint recomputation on Blackwell. +- [FlashInfer Lightning Attention issue #739](https://github.com/flashinfer-ai/flashinfer/issues/739) + is still an open feature request, which is the strongest evidence that the + cuLA Lightning family is not duplicated there today. + +## Public boundary + +Use `cula.kernels.kda` and `cula.kernels.lightning` for the standalone API. +The registry is intentionally static and import-light: + +```python +from cula.kernels import list_kernels + +for spec in list_kernels(family="kda"): + print(spec.name, spec.level, spec.architectures, spec.status) +``` + +`operator` entries are complete calls, `composite` entries compose several +owned kernels, and `primitive` entries expose a deliberate intermediate +boundary. Inputs such as the cumulative FP32 KDA gate are part of a primitive's +contract; cuLA does not silently import another framework to construct them. + +## Maintenance policy + +1. A public kernel must import and run without FLA or Triton. +2. New primitives need a correctness test for their tensor contract and a + benchmark that justifies preserving the boundary. +3. An overlapped complete operator is retained only if cuLA wins an important + architecture/shape, exposes a missing primitive, or materially simplifies + integration. +4. Reference frameworks belong in optional tests and benchmarks, never in + `cula/` or as a git submodule. diff --git a/pyproject.toml b/pyproject.toml index 9ecfeb42..e2fb2e9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ include = ["cula", "cula.*"] [tool.ruff] target-version = "py312" line-length = 127 -exclude = ["third_party/"] [tool.ruff.lint] select = [ diff --git a/scripts/modal_validate.py b/scripts/modal_validate.py index 55b3579b..fb4f3973 100644 --- a/scripts/modal_validate.py +++ b/scripts/modal_validate.py @@ -42,7 +42,7 @@ f"torch=={TORCH_VERSION}", index_url=f"https://download.pytorch.org/whl/{CUDA_TAG}", ) - .pip_install(CUTLASS_SPEC, "flash-linear-attention", "pytest") + .pip_install(CUTLASS_SPEC, "pytest") ) app = modal.App("cula-validate-v5") diff --git a/tests/conftest.py b/tests/conftest.py index 84017b1c..541deb55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,26 @@ +import importlib.util import re import pytest import torch +_FLA_REFERENCE_TESTS = { + "test_chunk_wy_dqkg_sm90.py", + "test_kda_flashkda_sm100.py", + "test_kda_sm100_intracard_cp.py", + "test_kda_sm100_output.py", + "test_kda_sm100_recurrence.py", + "test_kda_sm90_intracard_cp.py", + "test_kda_sm90_prefill_vs_fla.py", +} + + +def pytest_ignore_collect(collection_path, config): + """Keep external-reference tests optional in a standalone installation.""" + + del config + return collection_path.name in _FLA_REFERENCE_TESTS and importlib.util.find_spec("fla") is None + def _is_sm100() -> bool: return torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10 diff --git a/tests/test_kda_chunk_sm90.py b/tests/test_kda_chunk_sm90.py deleted file mode 100644 index a1ac933e..00000000 --- a/tests/test_kda_chunk_sm90.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SM90 correctness tests for the public ``cula.kda.chunk_kda`` entrypoint.""" - -import pytest -import torch - -from benchmarks import bench_kda_fwd_bwd_e2e as bench -from benchmarks.utils import ( - SEED, - exclusive_cumsum, - prepare_safe_gate_inputs, - relative_rms_error_rel_max_mean_abs, - set_seed, -) - -pytestmark = pytest.mark.sm90_only - -D = 128 -OUT_NAMES = ("o", "ht", "dq", "dk", "dv", "dg", "dbeta", "dh0") - -SM90_CHUNK_CASES = ( - ("fixed_recompute_beta_fp32", 2, 64, (64, 64), torch.float32, False), - ("fixed_saved_intermediates_beta_fp32", 1, 64, (64,), torch.float32, True), - ("varlen_recompute_beta_bf16", 1, 96, (31, 65), torch.bfloat16, False), -) - - -def _make_inputs(batch_size, length, seq_lens, beta_dtype): - device = torch.device("cuda") - cu_seqlens = torch.tensor(exclusive_cumsum(list(seq_lens)), dtype=torch.int32, device=device) - inputs = prepare_safe_gate_inputs( - batch_size, - length, - 2, - D, - device, - cu_seqlens=cu_seqlens, - has_init_state=True, - num_v_heads=2, - ) - inputs["beta"] = inputs["beta"].to(beta_dtype) - set_seed(SEED + 1) - return { - "q": inputs["q"], - "k": inputs["k"], - "v": inputs["v"], - "g": inputs["g"], - "beta": inputs["beta"], - "scale": inputs["scale"], - "A_log": inputs["A_log"], - "dt_bias": inputs["dt_bias"], - "init_state": inputs["init_state"], - "cu_seqlens": cu_seqlens, - "lower_bound": inputs["lower_bound"], - "do": torch.randn_like(inputs["v"]), - "dht": torch.randn_like(inputs["init_state"]), - } - - -def _run_case(case): - case_id, batch_size, length, seq_lens, beta_dtype, disable_recompute = case - inputs = _make_inputs(batch_size, length, seq_lens, beta_dtype) - previous = bench.DISABLE_RECOMPUTE - bench.DISABLE_RECOMPUTE = disable_recompute - try: - reference = bench.run_kda_e2e_with_grads(**inputs, fn=bench.fla_chunk_kda) - actual = bench.run_kda_e2e_with_grads(**inputs, fn=bench.cula_chunk_kda) - torch.cuda.synchronize() - finally: - bench.DISABLE_RECOMPUTE = previous - - for name in OUT_NAMES: - assert reference[name].shape == actual[name].shape - assert torch.isfinite(actual[name]).all(), f"{case_id}: {name} contains non-finite values" - rel_rms, rel_max, mean_abs = relative_rms_error_rel_max_mean_abs(reference[name], actual[name]) - assert rel_rms < 0.05, f"{case_id}: {name} rel_rms={rel_rms:.6f}, mean_abs={mean_abs:.6e}" - assert rel_max < 0.25, f"{case_id}: {name} rel_max={rel_max:.6f}, mean_abs={mean_abs:.6e}" - - -@pytest.mark.parametrize("case", SM90_CHUNK_CASES, ids=[case[0] for case in SM90_CHUNK_CASES]) -def test_chunk_kda_sm90_entry_matches_fla(case): - _run_case(case) - torch.cuda.empty_cache() diff --git a/tests/test_kda_sm100_chunk_vs_fla.py b/tests/test_kda_sm100_chunk_vs_fla.py deleted file mode 100644 index ac8326b5..00000000 --- a/tests/test_kda_sm100_chunk_vs_fla.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang - -# Precision Tests for FlashKDA implementation compared with FLA Triton baseline -# Test modes (see issue #78): -# default: fast subset (kda_fast cases only) -# pytest -m kda_slow: slow stress traces and wider parameter grids -# pytest -m kda_full: full sweep (fast + slow) - -import pytest -import torch -import torch.nn.functional as F -from fla.ops.kda.chunk import chunk_kda as fla_chunk_kda -from fla.utils import assert_close, device - -from cula.kda import chunk_kda as cula_chunk_kda - -pytestmark = pytest.mark.sm100_only - -_FAST = [pytest.mark.kda_fast] -_FAST_NR = [pytest.mark.kda_fast, pytest.mark.kda_fast_norecomp] -_SLOW = [pytest.mark.kda_slow] - -# (B, T, H, HV, D, gln, mask_p, l2norm, gate, safe_gate, dtype), marks -# FLA chunk KDA requires H == HV; dedicated decode/conv tests cover GVA. -_FIXED_CONFIGS = [ - ((1, 63, 1, 1, 128, 1, 0, False, False, True, torch.bfloat16), _FAST_NR), # small fixed (+ no_recomp) - ((2, 500, 3, 3, 128, 1, 0, False, False, True, torch.bfloat16), _SLOW), - ((2, 1000, 3, 3, 128, 1, 0.5, False, False, True, torch.bfloat16), _SLOW), - ((3, 1024, 4, 4, 128, 0.1, 0, False, False, True, torch.bfloat16), _SLOW), - ((4, 1024, 4, 4, 128, 1, 0, False, False, True, torch.bfloat16), _SLOW), - ((4, 1024, 4, 4, 128, 1, 0, True, False, True, torch.bfloat16), _FAST), # l2norm medium - ((2, 1500, 4, 4, 128, 10, 0, False, True, True, torch.bfloat16), _FAST_NR), # gated (+ no_recomp) - ((4, 2048, 8, 8, 128, 1, 0, False, True, True, torch.bfloat16), _SLOW), -] - -# (H, HV, D, mask_p, cu_seqlens, dtype, safe_gate), marks -_VARLEN_CONFIGS = [ - ((4, 4, 128, 0.1, [0, 15], torch.bfloat16, True), _SLOW), - ((4, 4, 128, 0.9, [0, 256, 500, 1000], torch.bfloat16, True), _SLOW), - ((4, 4, 128, 0.5, [0, 256, 500, 1000], torch.bfloat16, True), _SLOW), - ((4, 4, 128, 0, [0, 15, 100, 300, 1200, 2000], torch.bfloat16, True), _SLOW), - ((4, 4, 128, 0, [0, 100, 300, 1200, 3000, 4096], torch.bfloat16, True), _SLOW), - # ======Varlen test with simulated trace======= - ( - ( - 32, - 32, - 128, - 0, - [0, 247, 699, 982, 1688, 1985, 2383, 3081, 3526, 3973, 4096, 4824, 5101, 5919, 6426, 7137, 7392, 7800, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), - ( - ( - 32, - 32, - 128, - 0, - [0, 652, 1255, 1600, 2083, 2345, 2756, 3172, 3767, 4096, 4891, 5236, 5543, 6255, 6480, 6947, 7616, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), - ( - ( - 32, - 32, - 128, - 0, - [0, 315, 973, 1283, 2162, 2459, 2678, 2998, 3781, 4096, 4503, 5459, 6318, 6669, 6979, 7583, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), - ( - ( - 32, - 32, - 128, - 0, - [0, 494, 1004, 1561, 1908, 2240, 2849, 3116, 4096, 4986, 5626, 6090, 6718, 7244, 7870, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), -] - - -@pytest.mark.parametrize( - "beta_dtype", - [ - pytest.param(torch.float32, id="beta_fp32", marks=pytest.mark.kda_slow), - pytest.param(torch.bfloat16, id="beta_bf16"), - ], -) -@pytest.mark.parametrize("disable_recompute", [True, False], ids=["no_recomp", "recomp"]) -@pytest.mark.parametrize( - ( - "B", - "T", - "H", - "HV", - "D", - "gate_logit_normalizer", - "mask_p", - "use_qk_l2norm_in_kernel", - "use_gate_in_kernel", - "safe_gate", - "dtype", - ), - [ - pytest.param( - *params, - id="B{}-T{}-H{}-HV{}-D{}-gln{}-mask_p{}-l2norm{}-gate{}-safe_gate{}-{}".format(*params), - marks=marks, - ) - for params, marks in _FIXED_CONFIGS - ], -) -def test_safe_gate_chunk( - B: int, - T: int, - H: int, - HV: int, - D: int, - gate_logit_normalizer: float, - mask_p: float, - use_qk_l2norm_in_kernel: bool, - use_gate_in_kernel: bool, - safe_gate: bool, - dtype: torch.dtype, - disable_recompute: bool, - beta_dtype: torch.dtype, -): - torch.manual_seed(42) - q = torch.rand(B, T, H, D, dtype=dtype) - k = torch.rand(B, T, H, D, dtype=dtype) - v = torch.rand(B, T, HV, D, dtype=dtype) - g = torch.randn(B, T, HV, D, dtype=torch.float if not use_gate_in_kernel else dtype) - if use_gate_in_kernel: - A_log = torch.randn(HV, dtype=torch.float) - dt_bias = torch.randn(HV * D, dtype=torch.float) - else: - g = F.logsigmoid(g) / gate_logit_normalizer - g = g * (torch.rand_like(g) > mask_p) - if safe_gate: - lower_bound = -5.0 - if not use_gate_in_kernel: - g = g.clamp(-5, 0) - else: - lower_bound = None - - beta = torch.randn(B, T, HV, dtype=torch.float32).sigmoid().to(beta_dtype) - h0 = torch.randn(B, HV, D, D, dtype=torch.float32) - if use_gate_in_kernel: - A_log, dt_bias = map(lambda x: x.to(device).requires_grad_(True), (A_log, dt_bias)) - q, k, v, g, beta, h0 = map(lambda x: x.to(device).requires_grad_(True), (q, k, v, g, beta, h0)) - - do = torch.randn_like(v) - dht = torch.randn_like(h0) - - ref, ref_ht = fla_chunk_kda( - q=(F.normalize(q.clone(), p=2, dim=-1) if not use_qk_l2norm_in_kernel else q.clone()), - k=(F.normalize(k.clone(), p=2, dim=-1) if not use_qk_l2norm_in_kernel else k.clone()), - v=v.clone(), - g=g.clone(), - beta=beta.clone(), - A_log=(A_log.clone() if use_gate_in_kernel else None), - dt_bias=(dt_bias.clone() if use_gate_in_kernel else None), - initial_state=h0.clone(), - output_final_state=True, - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - use_gate_in_kernel=use_gate_in_kernel, - safe_gate=safe_gate, - lower_bound=lower_bound, - disable_recompute=disable_recompute, - ) - ((ref * do).sum() + (ref_ht * dht).sum()).backward() - if use_gate_in_kernel: - ref_dA, A_log.grad = A_log.grad, None - ref_dbias, dt_bias.grad = dt_bias.grad, None - ref_dq, ref_dk, ref_dv, ref_dg, ref_db, ref_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - q.grad = k.grad = v.grad = g.grad = beta.grad = h0.grad = None - - tri, tri_ht = cula_chunk_kda( - q=(F.normalize(q.clone(), p=2, dim=-1) if not use_qk_l2norm_in_kernel else q.clone()), - k=(F.normalize(k.clone(), p=2, dim=-1) if not use_qk_l2norm_in_kernel else k.clone()), - v=v.clone(), - g=g.clone(), - beta=beta.clone(), - A_log=(A_log.clone() if use_gate_in_kernel else None), - dt_bias=(dt_bias.clone() if use_gate_in_kernel else None), - initial_state=h0.clone(), - output_final_state=True, - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - use_gate_in_kernel=use_gate_in_kernel, - safe_gate=safe_gate, - lower_bound=lower_bound, - disable_recompute=disable_recompute, - ) - ((tri * do).sum() + (tri_ht * dht).sum()).backward() - if use_gate_in_kernel: - tri_dA, A_log.grad = A_log.grad, None - tri_dbias, dt_bias.grad = dt_bias.grad, None - tri_dq, tri_dk, tri_dv, tri_dg, tri_db, tri_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - q.grad = k.grad = v.grad = g.grad = beta.grad = h0.grad = None - - assert_close("o", ref, tri, 0.005) - assert_close("ht", ref_ht, tri_ht, 0.005) - assert_close("dq", ref_dq, tri_dq, 0.008) - assert_close("dk", ref_dk, tri_dk, 0.008) - assert_close("dv", ref_dv, tri_dv, 0.008) - assert_close("dg", ref_dg, tri_dg, 0.02) - assert_close("db", ref_db, tri_db, 0.02) - if use_gate_in_kernel: - assert_close("dA", ref_dA, tri_dA, 0.003, warning=True) - assert_close("dbias", ref_dbias, tri_dbias, 0.008) - assert_close("dh0", ref_dh0, tri_dh0, 0.008) - - -@pytest.mark.parametrize( - "beta_dtype", - [ - pytest.param(torch.float32, id="beta_fp32", marks=pytest.mark.kda_slow), - pytest.param(torch.bfloat16, id="beta_bf16"), - ], -) -@pytest.mark.parametrize("disable_recompute", [True, False], ids=["no_recomp", "recomp"]) -@pytest.mark.parametrize( - ("H", "HV", "D", "mask_p", "cu_seqlens", "dtype", "safe_gate"), - [ - pytest.param( - *params, - id="H{}-HV{}-D{}-mask_p{}-cu_seqlens{}-{}-safe_gate{}".format(*params), - marks=marks, - ) - for params, marks in _VARLEN_CONFIGS - ], -) -def test_safe_gate_chunk_varlen( - H: int, - HV: int, - D: int, - mask_p: float, - cu_seqlens: list[int], - dtype: torch.dtype, - safe_gate: bool, - disable_recompute: bool, - beta_dtype: torch.dtype, -): - torch.manual_seed(42) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) - cu_seqlens_cpu = cu_seqlens.cpu() - T = cu_seqlens[-1] - N = len(cu_seqlens) - 1 - - q = torch.randn((1, T, H, D), dtype=dtype) - k = F.normalize(torch.randn(1, T, H, D, dtype=torch.float32), p=2, dim=-1).to(dtype) - v = torch.randn((1, T, HV, D), dtype=dtype) - g = F.logsigmoid(torch.randn(1, T, HV, D, dtype=torch.float)) - mask = torch.rand_like(g) > mask_p - g = g * mask + (~mask) * (-1000) - if safe_gate: - g = g.clamp(-5, 0) - - beta = torch.randn(1, T, HV, dtype=torch.float32).sigmoid().to(beta_dtype) - h0 = torch.randn((N, HV, D, D), dtype=torch.float32) - - q, k, v, g, beta, h0 = map(lambda x: x.to(device).requires_grad_(), (q, k, v, g, beta, h0)) - do = torch.randn_like(v) - dht = torch.rand_like(h0) - - tri, tri_ht = cula_chunk_kda( - q=F.normalize(q.clone(), p=2, dim=-1), - k=k.clone(), - v=v.clone(), - g=g.clone(), - beta=beta.clone(), - initial_state=h0.clone(), - output_final_state=True, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens_cpu, - safe_gate=safe_gate, - lower_bound=-5.0 if safe_gate else None, - disable_recompute=disable_recompute, - ) - ((tri * do).sum() + (tri_ht * dht).sum()).backward() - tri_dq, tri_dk, tri_dv, tri_dg, tri_db, tri_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - q.grad = k.grad = v.grad = g.grad = beta.grad = h0.grad = None - - ref, ref_ht = fla_chunk_kda( - q=F.normalize(q.clone(), p=2, dim=-1), - k=k.clone(), - v=v.clone(), - g=g.clone(), - beta=beta.clone(), - initial_state=h0.clone(), - output_final_state=True, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens_cpu, - safe_gate=safe_gate, - lower_bound=-5.0 if safe_gate else None, - disable_recompute=disable_recompute, - ) - ((ref * do).sum() + (ref_ht * dht).sum()).backward() - ref_dq, ref_dk, ref_dv, ref_dg, ref_db, ref_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - - assert_close("o", ref, tri, 0.005) - assert_close("ht", ref_ht, tri_ht, 0.005) - assert_close("dq", ref_dq, tri_dq, 0.007) - assert_close("dk", ref_dk, tri_dk, 0.008) - assert_close("dv", ref_dv, tri_dv, 0.007) - assert_close("dg", ref_dg, tri_dg, 0.015) - assert_close("db", ref_db, tri_db, 0.015) - assert_close("dh0", ref_dh0, tri_dh0, 0.007) diff --git a/tests/test_kda_sm100_chunk_vs_naive.py b/tests/test_kda_sm100_chunk_vs_naive.py deleted file mode 100644 index e5cbb574..00000000 --- a/tests/test_kda_sm100_chunk_vs_naive.py +++ /dev/null @@ -1,334 +0,0 @@ -# Copyright 2025-2026 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang - -# Tests for the chunk-decomposed KDA implementation (python/kda/chunk.py) -# Test modes (see issue #78): -# default: fast subset (kda_fast cases only) -# pytest -m kda_slow: slow stress traces and wider parameter grids -# pytest -m kda_full: full sweep (fast + slow) - -import pytest -import torch -import torch.nn.functional as F -from fla.ops.kda.gate import naive_kda_gate, naive_kda_lowerbound_gate -from fla.ops.kda.naive import naive_recurrent_kda -from fla.utils import assert_close, device - -from cula.kda import chunk_kda - -pytestmark = pytest.mark.sm100_only - -_FAST = [pytest.mark.kda_fast] -_FAST_NR = [pytest.mark.kda_fast, pytest.mark.kda_fast_norecomp] -_SLOW = [pytest.mark.kda_slow] - -# (B, T, H, HV, D, gln, mask_p, l2norm, gate, safe_gate, dtype), marks -# The FLA naive reference requires H == HV; dedicated decode/conv tests cover GVA. -_FIXED_CONFIGS = [ - ((1, 63, 1, 1, 128, 1, 0, False, False, True, torch.bfloat16), _FAST_NR), # small fixed (+ no_recomp) - ((2, 500, 3, 3, 128, 1, 0, False, False, True, torch.bfloat16), _SLOW), - ((2, 1000, 3, 3, 128, 1, 0.5, False, False, True, torch.bfloat16), _SLOW), - ((3, 1024, 4, 4, 128, 0.1, 0, False, False, True, torch.bfloat16), _SLOW), - ((4, 1024, 4, 4, 128, 1, 0, False, False, True, torch.bfloat16), _SLOW), - ((4, 1024, 4, 4, 128, 1, 0, True, False, True, torch.bfloat16), _FAST), # l2norm medium - ((2, 1500, 4, 4, 128, 10, 0, False, True, True, torch.bfloat16), _FAST_NR), # gated (+ no_recomp) - ((4, 2048, 8, 8, 128, 1, 0, False, True, True, torch.bfloat16), _SLOW), -] - -# (H, HV, D, mask_p, cu_seqlens, dtype, safe_gate), marks -_VARLEN_CONFIGS = [ - ((4, 4, 128, 0.1, [0, 15], torch.bfloat16, True), _FAST), # short varlen smoke - ((4, 4, 128, 0.9, [0, 256, 500, 1000], torch.bfloat16, True), _SLOW), - ((4, 4, 128, 0.5, [0, 256, 500, 1000], torch.bfloat16, True), _SLOW), - ((4, 4, 128, 0, [0, 15, 100, 300, 1200, 2000], torch.bfloat16, True), _FAST), # multi-batch varlen - ((4, 4, 128, 0, [0, 100, 300, 1200, 3000, 4096], torch.bfloat16, True), _SLOW), - # ======Varlen test with simulated trace======= - ( - ( - 32, - 32, - 128, - 0, - [0, 247, 699, 982, 1688, 1985, 2383, 3081, 3526, 3973, 4096, 4824, 5101, 5919, 6426, 7137, 7392, 7800, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), - ( - ( - 32, - 32, - 128, - 0, - [0, 652, 1255, 1600, 2083, 2345, 2756, 3172, 3767, 4096, 4891, 5236, 5543, 6255, 6480, 6947, 7616, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), - ( - ( - 32, - 32, - 128, - 0, - [0, 315, 973, 1283, 2162, 2459, 2678, 2998, 3781, 4096, 4503, 5459, 6318, 6669, 6979, 7583, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), - ( - ( - 32, - 32, - 128, - 0, - [0, 494, 1004, 1561, 1908, 2240, 2849, 3116, 4096, 4986, 5626, 6090, 6718, 7244, 7870, 8192], - torch.bfloat16, - True, - ), - _SLOW, - ), -] - - -@pytest.mark.parametrize( - "beta_dtype", - [ - pytest.param(torch.float32, id="beta_fp32", marks=pytest.mark.kda_slow), - pytest.param(torch.bfloat16, id="beta_bf16"), - ], -) -@pytest.mark.parametrize("disable_recompute", [True, False], ids=["no_recomp", "recomp"]) -@pytest.mark.parametrize( - ( - "B", - "T", - "H", - "HV", - "D", - "gate_logit_normalizer", - "mask_p", - "use_qk_l2norm_in_kernel", - "use_gate_in_kernel", - "safe_gate", - "dtype", - ), - [ - pytest.param( - *params, - id="B{}-T{}-H{}-HV{}-D{}-gln{}-mask_p{}-l2norm{}-gate{}-safe_gate{}-{}".format(*params), - marks=marks, - ) - for params, marks in _FIXED_CONFIGS - ], -) -def test_safe_gate_chunk( - B: int, - T: int, - H: int, - HV: int, - D: int, - gate_logit_normalizer: float, - mask_p: float, - use_qk_l2norm_in_kernel: bool, - use_gate_in_kernel: bool, - safe_gate: bool, - dtype: torch.dtype, - disable_recompute: bool, - beta_dtype: torch.dtype, -): - torch.manual_seed(42) - q = torch.rand(B, T, H, D, dtype=dtype) - k = torch.rand(B, T, H, D, dtype=dtype) - v = torch.rand(B, T, HV, D, dtype=dtype) - g = torch.randn(B, T, HV, D, dtype=torch.float if not use_gate_in_kernel else dtype) - if use_gate_in_kernel: - A_log = torch.randn(HV, dtype=torch.float) - dt_bias = torch.randn(HV * D, dtype=torch.float) - else: - g = F.logsigmoid(g) / gate_logit_normalizer - g = g * (torch.rand_like(g) > mask_p) - if safe_gate: - lower_bound = -5.0 - if not use_gate_in_kernel: - g = g.clamp(-5, 0) - naive_kda_gate_fn = naive_kda_lowerbound_gate - else: - lower_bound = None - naive_kda_gate_fn = naive_kda_gate - - beta = torch.randn(B, T, HV, dtype=torch.float32).sigmoid().to(beta_dtype) - h0 = torch.randn(B, HV, D, D, dtype=torch.float32) - if use_gate_in_kernel: - A_log, dt_bias = map(lambda x: x.to(device).requires_grad_(True), (A_log, dt_bias)) - q, k, v, g, beta, h0 = map(lambda x: x.to(device).requires_grad_(True), (q, k, v, g, beta, h0)) - - do = torch.randn_like(v) - dht = torch.randn_like(h0) - - ref, ref_ht = naive_recurrent_kda( - q=F.normalize(q.clone(), p=2, dim=-1), - k=F.normalize(k.clone(), p=2, dim=-1), - v=v.clone(), - g=(naive_kda_gate_fn(g, A_log, dt_bias) if use_gate_in_kernel else g.clone()), - beta=beta.clone(), - initial_state=h0.clone(), - output_final_state=True, - ) - ((ref * do).sum() + (ref_ht * dht).sum()).backward() - if use_gate_in_kernel: - ref_dA, A_log.grad = A_log.grad, None - ref_dbias, dt_bias.grad = dt_bias.grad, None - ref_dq, ref_dk, ref_dv, ref_dg, ref_db, ref_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - q.grad = k.grad = v.grad = g.grad = beta.grad = h0.grad = None - - tri, tri_ht = chunk_kda( - q=F.normalize(q.clone(), p=2, dim=-1) if not use_qk_l2norm_in_kernel else q.clone(), - k=F.normalize(k.clone(), p=2, dim=-1) if not use_qk_l2norm_in_kernel else k.clone(), - v=v.clone(), - g=g.clone(), - beta=beta.clone(), - A_log=(A_log.clone() if use_gate_in_kernel else None), - dt_bias=(dt_bias.clone() if use_gate_in_kernel else None), - initial_state=h0.clone(), - output_final_state=True, - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - use_gate_in_kernel=use_gate_in_kernel, - safe_gate=safe_gate, - lower_bound=lower_bound, - disable_recompute=disable_recompute, - ) - ((tri * do).sum() + (tri_ht * dht).sum()).backward() - if use_gate_in_kernel: - tri_dA, A_log.grad = A_log.grad, None - tri_dbias, dt_bias.grad = dt_bias.grad, None - tri_dq, tri_dk, tri_dv, tri_dg, tri_db, tri_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - q.grad = k.grad = v.grad = g.grad = beta.grad = h0.grad = None - - assert_close("o", ref, tri, 0.005) - assert_close("ht", ref_ht, tri_ht, 0.005) - assert_close("dq", ref_dq, tri_dq, 0.008) - assert_close("dk", ref_dk, tri_dk, 0.008) - assert_close("dv", ref_dv, tri_dv, 0.008) - assert_close("dg", ref_dg, tri_dg, 0.02) - assert_close("db", ref_db, tri_db, 0.02) - if use_gate_in_kernel: - assert_close("dA", ref_dA, tri_dA, 0.003, warning=True) - assert_close("dbias", ref_dbias, tri_dbias, 0.008) - assert_close("dh0", ref_dh0, tri_dh0, 0.008) - - -@pytest.mark.parametrize( - "beta_dtype", - [ - pytest.param(torch.float32, id="beta_fp32", marks=pytest.mark.kda_slow), - pytest.param(torch.bfloat16, id="beta_bf16"), - ], -) -@pytest.mark.parametrize("disable_recompute", [True, False], ids=["no_recomp", "recomp"]) -@pytest.mark.parametrize( - ("H", "HV", "D", "mask_p", "cu_seqlens", "dtype", "safe_gate"), - [ - pytest.param( - *params, - id="H{}-HV{}-D{}-mask_p{}-cu_seqlens{}-{}-safe_gate{}".format(*params), - marks=marks, - ) - for params, marks in _VARLEN_CONFIGS - ], -) -def test_safe_gate_chunk_varlen( - H: int, - HV: int, - D: int, - mask_p: float, - cu_seqlens: list[int], - dtype: torch.dtype, - safe_gate: bool, - disable_recompute: bool, - beta_dtype: torch.dtype, -): - torch.manual_seed(42) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) - cu_seqlens_cpu = cu_seqlens.cpu() - T = cu_seqlens[-1] - N = len(cu_seqlens) - 1 - - q = torch.randn((1, T, H, D), dtype=dtype) - k = F.normalize(torch.randn(1, T, H, D, dtype=torch.float32), p=2, dim=-1).to(dtype) - v = torch.randn((1, T, HV, D), dtype=dtype) - g = F.logsigmoid(torch.randn(1, T, HV, D, dtype=torch.float)) - mask = torch.rand_like(g) > mask_p - g = g * mask + (~mask) * (-1000) - if safe_gate: - g = g.clamp(-5, 0) - - beta = torch.randn(1, T, HV, dtype=torch.float32).sigmoid().to(beta_dtype) - h0 = torch.randn((N, HV, D, D), dtype=torch.float32) - - q, k, v, g, beta, h0 = map(lambda x: x.to(device).requires_grad_(), (q, k, v, g, beta, h0)) - do = torch.randn_like(v) - dht = torch.rand_like(h0) - - tri, tri_ht = chunk_kda( - q=F.normalize(q.clone(), p=2, dim=-1), - k=k.clone(), - v=v.clone(), - g=g.clone(), - beta=beta.clone(), - initial_state=h0.clone(), - output_final_state=True, - cu_seqlens=cu_seqlens, - cu_seqlens_cpu=cu_seqlens_cpu, - safe_gate=safe_gate, - lower_bound=-5.0 if safe_gate else None, - disable_recompute=disable_recompute, - ) - ((tri * do).sum() + (tri_ht * dht).sum()).backward() - tri_dq, tri_dk, tri_dv, tri_dg, tri_db, tri_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - q.grad = k.grad = v.grad = g.grad = beta.grad = h0.grad = None - - ref = [] - ref_ht = [] - for i in range(N): - ref_i, ref_ht_i = naive_recurrent_kda( - q=F.normalize(q[:, cu_seqlens[i] : cu_seqlens[i + 1]], p=2, dim=-1), - k=k[:, cu_seqlens[i] : cu_seqlens[i + 1]], - v=v[:, cu_seqlens[i] : cu_seqlens[i + 1]], - beta=beta[:, cu_seqlens[i] : cu_seqlens[i + 1]], - g=g[:, cu_seqlens[i] : cu_seqlens[i + 1]], - initial_state=h0[i], - output_final_state=True, - ) - ref.append(ref_i) - ref_ht.append(ref_ht_i) - ref = torch.cat(ref, 1) - ref_ht = torch.cat(ref_ht, 0) - - ((ref * do).sum() + (ref_ht * dht).sum()).backward() - ref_dq, ref_dk, ref_dv, ref_dg, ref_db, ref_dh0 = q.grad, k.grad, v.grad, g.grad, beta.grad, h0.grad - - assert_close("o", ref, tri, 0.005) - assert_close("ht", ref_ht, tri_ht, 0.005) - assert_close("dq", ref_dq, tri_dq, 0.007) - assert_close("dk", ref_dk, tri_dk, 0.008) - assert_close("dv", ref_dv, tri_dv, 0.007) - assert_close("dg", ref_dg, tri_dg, 0.015) - assert_close("db", ref_db, tri_db, 0.015) - assert_close("dh0", ref_dh0, tri_dh0, 0.007) diff --git a/tests/test_kda_sm100_intracard_cp.py b/tests/test_kda_sm100_intracard_cp.py index 9de4a4ff..6f348b6b 100644 --- a/tests/test_kda_sm100_intracard_cp.py +++ b/tests/test_kda_sm100_intracard_cp.py @@ -48,11 +48,10 @@ # * Same-kernel (CP scheduling only): torch.testing.assert_close(atol=1e-2, rtol=1e-2) # — matches tests/test_kda_sm100_recurrence.py CP block # * Cross-impl / vs ref: fla.utils.assert_close(ratio=...) -# — matches tests/test_kda_sm100_chunk_vs_fla.py ATOL_SAME_KERNEL = 1e-2 RTOL_SAME_KERNEL = 1e-2 RATIO_VS_REF = 0.005 # RMSE / RMS(ref) — matches FLA test_gated_delta.py fwd -RATIO_VS_FLA = 0.015 # cross-impl gap measured ~1.27% (TF32 MMA vs Triton fp32) +RATIO_VS_FLA = 0.015 # cross-implementation gap measured ~1.27% (TF32 MMA vs Triton fp32) RATIO_STRESS = 1e-6 # deterministic re-run: drift would indicate race diff --git a/tests/test_standalone_kernel_surface.py b/tests/test_standalone_kernel_surface.py new file mode 100644 index 00000000..87084bb3 --- /dev/null +++ b/tests/test_standalone_kernel_surface.py @@ -0,0 +1,80 @@ +# Copyright 2025-2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest +import torch + +import cula.kernels.kda as kda +import cula.kernels.lightning as lightning +from cula.kernels import get_kernel, list_kernels +from cula.ops.kda.index import prepare_chunk_indices, prepare_lens + + +def test_runtime_package_has_no_fla_or_triton_imports(): + package_root = Path(__file__).parents[1] / "cula" + forbidden: list[tuple[Path, int, str]] = [] + for path in package_root.rglob("*.py"): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + names = [node.module or ""] + else: + continue + for name in names: + if name == "fla" or name.startswith("fla.") or name == "triton" or name.startswith("triton."): + forbidden.append((path.relative_to(package_root), node.lineno, name)) + assert forbidden == [] + + +def test_registry_names_are_exported(): + namespaces = {"kda": kda, "lightning": lightning} + registered_by_family = {family: set() for family in namespaces} + for spec in list_kernels(): + assert spec.name in namespaces[spec.family].__all__ + assert get_kernel(spec.name) is spec + registered_by_family[spec.family].add(spec.name) + for family, namespace in namespaces.items(): + assert registered_by_family[family] == set(namespace.__all__) + + +def test_registry_filters_are_stable(): + kda_primitives = list_kernels(family="kda", level="primitive") + assert kda_primitives + assert all(spec.family == "kda" and spec.level == "primitive" for spec in kda_primitives) + with pytest.raises(KeyError, match="unknown standalone kernel"): + get_kernel("not_a_kernel") + + +def test_prepare_chunk_indices_cpu_contract(): + cu_seqlens = torch.tensor([0, 1, 65, 193], dtype=torch.int32) + actual = prepare_chunk_indices(cu_seqlens, 64) + expected = torch.tensor([[0, 0], [1, 0], [2, 0], [2, 1]], dtype=torch.int32) + assert torch.equal(actual, expected) + assert torch.equal(prepare_lens(cu_seqlens), torch.tensor([1, 64, 128], dtype=torch.int32)) + + +def test_prepare_chunk_indices_invalidates_cache_after_in_place_update(): + cu_seqlens = torch.tensor([0, 64, 128], dtype=torch.int32) + first = prepare_chunk_indices(cu_seqlens, 64) + + cu_seqlens[1] = 128 + cu_seqlens[2] = 192 + second = prepare_chunk_indices(cu_seqlens, 64) + + assert first.tolist() == [[0, 0], [1, 0]] + assert second.tolist() == [[0, 0], [0, 1], [1, 0]] + + +def test_prepare_chunk_indices_rejects_invalid_metadata(): + for chunk_size in (None, -1, 0, True, 64.0): + with pytest.raises(ValueError, match="positive"): + prepare_chunk_indices(torch.tensor([0, 4], dtype=torch.int32), chunk_size) + with pytest.raises(ValueError, match="one-dimensional"): + prepare_chunk_indices(torch.zeros((2, 2), dtype=torch.int32), 64) diff --git a/third_party/flash-linear-attention b/third_party/flash-linear-attention deleted file mode 160000 index 3a9ce1c8..00000000 --- a/third_party/flash-linear-attention +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3a9ce1c83a13994d824dbb3421e2989d330bb38b