From af4d9c2d5bf547eaeff550e1aec6dcb9e196b7ae Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 00:23:39 +0800 Subject: [PATCH 01/10] feat(ws1): land C1 four-judgment numerical contract (#267) Freeze the WS1 numerical SSOT for issue #267: four-judgment tolerance rows, dtype/TF32/FP8 policy, comparison roles, chain logprob aggregates, shared resolver, and op_checks wiring so forward and gradient accuracy no longer share one threshold path. Add schema tests, usage docs, and a migration checklist for remaining private-atol call sites (C3/C4/C8). Closes #267 --- docs/contributing/gtest-usage.md | 389 +++++++ docs/contributing/testing.md | 21 + docs/design/ws1-gtest-migration-checklist.md | 248 +++++ docs/design/ws1-numerical-contract.md | 187 ++++ rl_engine/kernels/gtest/__init__.py | 12 + rl_engine/kernels/gtest/op_checks.py | 121 ++- rl_engine/kernels/gtest/tolerance.py | 967 +++++++++++++++++- .../kernels/gtest/tolerance_contract.json | 239 ++++- tests/test_op_checks.py | 99 +- tests/test_tolerance_contract.py | 446 +++++++- 10 files changed, 2716 insertions(+), 13 deletions(-) create mode 100644 docs/contributing/gtest-usage.md create mode 100644 docs/design/ws1-gtest-migration-checklist.md create mode 100644 docs/design/ws1-numerical-contract.md diff --git a/docs/contributing/gtest-usage.md b/docs/contributing/gtest-usage.md new file mode 100644 index 00000000..036c1b65 --- /dev/null +++ b/docs/contributing/gtest-usage.md @@ -0,0 +1,389 @@ +# gtest usage guide (operator candidate vs gold) + +> **Audience:** contributors implementing train–inference / batch-invariant operators +> **Entry point:** `scripts/check_operator.py` + `rl_engine/kernels/gtest/*` +> **Numerical SSOT:** [#267](https://github.com/RL-Align/RL-Kernel/issues/267) four-judgment contract +> **Related:** [WS1 numerical contract](../design/ws1-numerical-contract.md) · [migration checklist](../design/ws1-gtest-migration-checklist.md) + +This is the official how-to for the gtest harness: register an op, build inputs, run the CLI for forward/backward checks, and obtain tolerances from the shared contract (not private `atol`/`rtol`). + +--- + +## 1. What gtest is for + +gtest validates a **single operator**: + +| Capability | Meaning | +|------------|---------| +| Gold | Usually a PyTorch / `forward_fp32` reference path | +| Candidate | CUDA / Triton / arch-specific implementation | +| Forward check | Outputs within contract tolerance (`forward_accuracy`) | +| Backward check | Selected input gradients within contract tolerance (`gradient_accuracy`, **independent of forward**) | + +It is **not**: + +- The full Qwen3-8B model-level gate (#266 C9/C10) +- The final cross-config invariance harness (C3/C4 build on the same contract) +- Real vLLM vs Megatron engine alignment + +The CLI primarily covers **accuracy** (candidate vs gold). +**Invariance** (bitwise across configs) and **train/infer aggregates** use the contract APIs / later harnesses—do not invent private gate thresholds in tests. + +--- + +## 2. End-to-end flow + +```text +1) (Optional) register the op in the runtime registry + ↓ +2) gtest/operator_specs.py → OP_SPECS: gold + candidates + ↓ +3) gtest/operator_inputs.py → build input shapes / values + ↓ +4) scripts/check_operator.py → run suite, load tolerance_contract.json + ↓ +5) report max_abs / tol / passed +``` + +### 2.1 Key files + +| Path | Role | +|------|------| +| `rl_engine/kernels/gtest/operator_specs.py` | `OP_SPECS`: name, `op_class`, gold, candidates, grad inputs | +| `rl_engine/kernels/gtest/operator_inputs.py` | Default Qwen3-8B dims + `make_operator_inputs` | +| `rl_engine/kernels/gtest/op_checks.py` | Suite execution and comparison | +| `rl_engine/kernels/gtest/tolerance_contract.json` | Numerical contract SSOT | +| `rl_engine/kernels/gtest/tolerance.py` | `load_contract` / `resolve_tolerance` / chain aggregates | +| `scripts/check_operator.py` | **CLI entry** | + +--- + +## 3. Step 1: register the op in `OP_SPECS` + +Edit `rl_engine/kernels/gtest/operator_specs.py` and add an entry to `OP_SPECS`. Example shape (logp / linear_logp): + +```python +"logp": OperatorSpec( + name="logp", + op_class="logprob", # selects the contract op_class row + gold_path="rl_engine.kernels.ops.pytorch.loss.logp.NativeLogpOp", + gold_method="forward_fp32", # method invoked on the gold instance + candidate_paths={ + "pytorch": "rl_engine.kernels.ops.pytorch.loss.logp.NativeLogpOp", + "cuda": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", + "cuda-sm90": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpSM90Op", + }, + grad_input_names=("logits",), # inputs compared under --check-grad +), +``` + +### 3.1 `OperatorSpec` fields + +| Field | Meaning | +|-------|---------| +| `name` | Value for CLI `--op` | +| `op_class` | Contract class: `elementwise` / `reduction` / `logprob` / `attention` | +| `gold_path` | Gold class path `module.Class` | +| `gold_method` | Method name, e.g. `forward_fp32`, `apply`, `__call__` | +| `candidate_paths` | Map `candidate name → implementation class`; CLI `--candidate cuda` looks up this map | +| `grad_input_names` | With `--check-grad`, enable grads and compare these inputs; missing config errors | + +**Only ops registered in `OP_SPECS` can be invoked via `check_operator.py`.** + +Currently registered (source of truth is the code): + +```text +rms_norm, attention, logp, linear_logp, embedding, lm_head, +det_gemm, rope, silu, swiglu, batch_invariant_logp +``` + +--- + +## 4. Step 2: build inputs + +File: `rl_engine/kernels/gtest/operator_inputs.py`. + +### 4.1 Default model dims (Qwen3-8B Dense semantics) + +Macros at the top of the file (local experiments may change them; WS1 full-model EXIT uses the official config fingerprint): + +```text +DEFAULT_HIDDEN = 4096 +DEFAULT_N_HEADS = 32 +DEFAULT_N_KV_HEADS = 8 +DEFAULT_HEAD_DIM = 128 +DEFAULT_INTERMEDIATE = 12288 +DEFAULT_VOCAB = 151936 +DEFAULT_ROPE_THETA = 1.0e6 +DEFAULT_RMS_EPS = 1.0e-6 +``` + +### 4.2 Shape names and input builders + +- `operator_shape_name(op_name, args)` — human-readable case name (e.g. `2x16x257`) +- `_make_*_inputs` / `make_operator_inputs` — build the input dict from `--op` and CLI args + - `random`: reproducible randomness (`--seed` plus per-tensor offsets) + - `constant`: fixed values for debugging (`--constant-value` / `--token-value`) + +When adding an op: extend the shape map and implement the matching `_make_xxx_inputs`. + +### 4.3 Suggested GRPO-oriented shapes (local sweeps) + +For GRPO, `B = P × G`. With `G=8`, batch is often a multiple of 8. +`B=1` is fine for smoke; fuller sweeps may use: + +```text +B ∈ {1, 8, 16, 32, 64} +S ∈ {1, 31, 33, 127, 129, 255, 256, 257, 512, 1024, 4096, 8192} +``` + +Prefer short `S` when VRAM is tight; full-model gates are owned by #266 / C2. + +--- + +## 5. Step 3: run the CLI + +```bash +# From the repo root; prefer an editable install: pip install -e . +python scripts/check_operator.py --op logp --candidate pytorch --device cpu --dtype fp32 --batch 1 --seq 2 --vocab 17 +``` + +### 5.1 Common examples + +**Smoke (CPU / PyTorch self-check)** + +```bash +python scripts/check_operator.py \ + --op logp --candidate pytorch --device cpu --dtype fp32 \ + --batch 1 --seq 2 --vocab 17 +``` + +**Triton `linear_logp` + backward (BF16)** + +```bash +python scripts/check_operator.py \ + --op linear_logp --candidate triton --device cuda --dtype bf16 \ + --batch 1 --seq 2 --vocab 1024 --normalized-dim 4096 \ + --check-grad +``` + +**CUDA deterministic attention + gradients** + +```bash +python scripts/check_operator.py \ + --op attention --candidate cuda --device cuda --dtype bf16 \ + --batch 2 --seq 64 --check-grad --grad-mode random +``` + +**Full JSON report** + +```bash +python scripts/check_operator.py --op rms_norm --candidate cuda --dtype bf16 --device cuda --json +``` + +### 5.2 CLI flags + +| Flag | Meaning | +|------|---------| +| `--op` | Operator name from `OP_SPECS` | +| `--candidate` | Backend: `pytorch` / `cuda` / `cuda-generic` / `cuda-sm90` / `triton` / … (see that op’s `candidate_paths`) | +| `--dtype` | `fp32` / `bf16` / `fp16`; selects input dtype and contract row | +| `--device` | `auto` / `cpu` / `cuda` | +| `--batch` / `--seq` | Batch size and sequence length for inputs | +| `--vocab` | Vocab size; logp logits `[B,S,V]`; linear_logp weight `[V,H]` | +| `--input-mode` | `random` (default) or `constant` | +| `--constant-value` | Float fill in constant mode | +| `--token-value` | Token id in constant mode | +| `--normalized-dim` | Hidden dim for rms_norm / linear_logp, etc. | +| `--k-dim` / `--n-dim` | Matmul / det_gemm dims | +| `--theta` | RoPE theta | +| `--eps` | RMSNorm epsilon | +| `--seed` | Input RNG seed (per-tensor offsets still apply) | +| `--arch-key` | Arch override key, e.g. `sm90` (contract `arch_overrides`) | +| `--check-grad` | Also compare gradients (requires `grad_input_names`) | +| `--grad-mode` | `random` (default, stricter) / `ones` (≈ `output.sum().backward()`) | +| `--grad-seed` | Seed for random upstream gradients | +| `--json` | Print the full structured report | + +--- + +## 6. Where tolerances come from (after #267) + +### 6.1 Before vs after C1 + +| Before | After (C1 / #267) | +|--------|-------------------| +| Mostly `accuracy[op_class][dtype]` | **Four judgments**: forward/gradient × accuracy/invariance | +| Forward and grad often shared one tol | **Grad uses `gradient_accuracy` only** (no silent forward inheritance) | +| Flat threshold table | Plus dtype policy, comparison roles, chain logprob aggregates | + +### 6.2 Which judgments the CLI / `op_checks` use + +`run_operator_suite` / `check_operator.py`: + +| Comparison | Judgment | +|------------|----------| +| Output vs gold | `forward_accuracy` | +| Gradient vs gold | `gradient_accuracy` | + +Batch/chunk **bitwise invariance** and train/infer **three aggregates** are not separate CLI switches. Use: + +```python +from rl_engine.kernels.gtest.tolerance import ( + load_contract, + resolve_tolerance, + compute_logprob_aggregates, + judge_logprob_aggregates, + default_clip_interval, +) + +contract = load_contract() +# Cross-config invariance (gate path) +inv = resolve_tolerance( + contract, + judgment="forward_invariance", # or gradient_invariance + op_class="attention", + dtype="bfloat16", + backend_profile="cuda_bf16", +) +# inv.mode == "bitwise", inv.atol == inv.rtol == 0 + +# Train vs infer selected-logprob +agg = compute_logprob_aggregates( + train_logp, + rollout_logp, + active_mask, + contract=contract, + report_kind="train_infer_logprob_parity", + clip_interval=default_clip_interval(contract), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", +) +verdict = judge_logprob_aggregates(agg, contract, execution_dtype="bfloat16") +``` + +### 6.3 Policy locks (WS1) + +| Item | Value | +|------|--------| +| Execution | BF16 mandatory for EXIT (CLI may still exercise fp32/fp16) | +| Reference / accumulation | FP32 | +| FP8 | Out of scope (resolve hard-fails) | +| TF32 | Disabled | + +WS1 evidence must attach checked provenance to its candidate report: + +```python +from rl_engine.kernels.gtest import BackendProvenance, CandidateSpec + +provenance = BackendProvenance( + backend_profile="cuda_bf16", # use triton_cuda_bf16 + triton for Triton + requested_backend="cuda", + actual_backend="cuda", + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, +) +candidate = CandidateSpec( + name="cuda-candidate", + backend="cuda", + fn=op, + provenance=provenance, +) +``` + +The suite rejects backend fallback, dtype drift, TF32 enablement, and observed output +dtypes that disagree with this provenance before producing a passing report. +| Profiles | `cuda_bf16` and `triton_cuda_bf16` share **the same** thresholds | + +**Do not** use private `atol=1e-5` (etc.) as WS1 gate evidence. Inventory of legacy private thresholds: [migration checklist](../design/ws1-gtest-migration-checklist.md). + +### 6.4 Report `tol=(atol=..., rtol=...)` + +The CLI summary line: + +```text +tol=(atol=..., rtol=...) +``` + +comes from the shared resolver—not hard-coded constants inside `check_operator.py`. + +--- + +## 7. Recommended local test order + +```text +1. --candidate pytorch --device cpu --dtype fp32 + → registration / inputs / plumbing smoke + +2. Same shape with --dtype bf16 --device cuda --candidate triton|cuda + → real candidate forward + +3. Add --check-grad --grad-mode random + → gradients (random upstream grads catch more bugs than ones) + +4. --arch-key sm90 only when you need arch-specific contract overrides + +5. Cross batch/layout: not CLI-only; use invariance judgments + dedicated tests +``` + +--- + +## 8. Common failures + +| Symptom | Likely cause | +|---------|----------------| +| Unsupported / missing `--op` choice | Not registered in `OP_SPECS` | +| `--check-grad` missing grad inputs | Empty/wrong `grad_input_names` vs input keys | +| Candidate import error | Bad `candidate_paths` or extension not built | +| BF16 over tolerance | Confirm gold is `forward_fp32`; check contract row; do not loosen private atol | +| Missing SM90 symbols | Build without SM90 / non-sm90 GPU; pick another candidate or rebuild | +| Want FP8 | Hard-fail under WS1 contract; out of scope | + +--- + +## 9. Relationship to pytest + +| Path | Use | +|------|-----| +| `python scripts/check_operator.py ...` | Fast single-op shape/debug loops | +| `pytest tests/test_*.py` | Regression, invariance, integration | +| `pytest tests/test_tolerance_contract.py` | Contract schema / resolver | + +Both paths should take thresholds from `tolerance_contract.json`. +New pytest code should call `resolve_tolerance` instead of copying magic numbers. + +--- + +## 10. Minimal checklist for a new operator + +- [ ] Implementation under `rl_engine/kernels/ops/{pytorch,cuda,triton}/...` +- [ ] (Optional) runtime `registry` registration +- [ ] `OP_SPECS` entry: gold + candidates + `op_class` + `grad_input_names` +- [ ] `operator_inputs` shape name + input builder +- [ ] `check_operator.py` smoke + bf16 + `--check-grad` green +- [ ] Contract already has the `op_class` row (extend schema + `test_tolerance_contract` if not) +- [ ] No new private `atol`/`rtol` as gate evidence +- [ ] Operator docs point at the contract for thresholds (do not restate ad-hoc numbers) + +--- + +## 11. Further reading + +| Doc | Content | +|-----|---------| +| [ws1-numerical-contract.md](../design/ws1-numerical-contract.md) | Four judgments, roles, aggregate formulas | +| [ws1-gtest-migration-checklist.md](../design/ws1-gtest-migration-checklist.md) | Which tests still use private thresholds and when to migrate | +| [testing.md](testing.md) | Short testing entry points | +| Issues [#266](https://github.com/RL-Align/RL-Kernel/issues/266) / [#267](https://github.com/RL-Align/RL-Kernel/issues/267) | WS1 closeout and C1 contract | + +--- + +## 12. Changelog + +| Date | Notes | +|------|--------| +| 2026-08-11 | Initial English guide aligned with C1; documents CLI, `OP_SPECS`, inputs, and contract usage | diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index a96c0014..b0924ba0 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -2,6 +2,21 @@ RL-Kernel uses focused tests for dispatch behavior and operator accuracy. +## gtest (operator candidate vs gold) + +Primary entry for single-operator forward/backward checks against a PyTorch gold path: + +```bash +python scripts/check_operator.py --op logp --candidate pytorch --device cpu --dtype fp32 +``` + +Full usage (register `OP_SPECS`, build inputs, CLI flags, and the WS1 four-judgment +tolerance contract after #267): + +- **[gtest usage guide](gtest-usage.md)** (operator CLI + `OP_SPECS` + contract; English) +- [WS1 numerical contract](../design/ws1-numerical-contract.md) +- [gtest private-threshold migration checklist](../design/ws1-gtest-migration-checklist.md) + ## Dispatch Tests ```bash @@ -14,6 +29,12 @@ python -m pytest rl_engine/tests/test_dispatch.py -v python tests/test_op_accuracy.py ``` +Contract schema / resolver: + +```bash +python -m pytest tests/test_tolerance_contract.py tests/test_op_checks.py -q +``` + ## Documentation Build ```bash diff --git a/docs/design/ws1-gtest-migration-checklist.md b/docs/design/ws1-gtest-migration-checklist.md new file mode 100644 index 00000000..3b0a9346 --- /dev/null +++ b/docs/design/ws1-gtest-migration-checklist.md @@ -0,0 +1,248 @@ +# WS1 gtest 阈值迁移清单 + +> **关联：** [#266](https://github.com/RL-Align/RL-Kernel/issues/266) 父收尾 · [#267](https://github.com/RL-Align/RL-Kernel/issues/267) C1 契约 · [数值契约说明](ws1-numerical-contract.md) +> **目的：** 盘点「哪些测试仍用私有 `atol`/`rtol`、哪些已走 SSOT、何时必须迁到 `resolve_tolerance`」。 +> **快照：** 基于 `feat/ws1-c1-tolerance-contract-267` 落地 C1 后的仓库状态；文件增减时请更新本表。 + +--- + +## 0. 迁移总原则 + +### 0.1 SSOT 入口（改后唯一推荐） + +```python +from rl_engine.kernels.gtest.tolerance import ( + load_contract, + resolve_tolerance, + compute_logprob_aggregates, + judge_logprob_aggregates, + default_clip_interval, +) + +contract = load_contract() +spec = resolve_tolerance( + contract, + judgment="forward_accuracy", # 或 forward_invariance / gradient_* + op_class="logprob", # elementwise | reduction | logprob | attention + dtype="bfloat16", + backend_profile="cuda_bf16", # 与 triton_cuda_bf16 同阈值 +) +# assert_close(..., atol=spec.atol, rtol=spec.rtol) +# 不变性：spec.mode == "bitwise" 且 atol=rtol=0 → 优先 torch.equal +``` + +| Judgment | 用于 | +|----------|------| +| `forward_accuracy` | BF16 candidate vs FP32 reference | +| `forward_invariance` | 同逻辑 workload 跨 batch/chunk/layout（**bitwise**） | +| `gradient_accuracy` | 梯度 vs FP32 参考（**不得**读 forward 行） | +| `gradient_invariance` | 梯度跨 config（**bitwise**） | +| 三聚合 API | 链级 / 训推 selected-logprob（`max_abs_dlogp` / `approx_kl0` / `clipfrac0`） | + +### 0.2 什么叫「私有阈值」（禁止作为 WS1 gate 证据） + +- 测试文件内字面量：`atol=1e-5`、`atol=5e-2`、模块常量 `_DECODE_ATOL` 等 +- 文档里写死但未从 `tolerance_contract.json` resolve 的数 +- 从 `contract["accuracy"]...` 手抄数值后本地再改（应用 resolve，不要复制常量） +- 用非零 `atol` 充当 Batch/Chunk **invariance** 通过条件 + +### 0.3 什么可以保留（不必硬迁） + +| 场景 | 处理 | +|------|------| +| **bitwise 身份断言**（`torch.equal`） | 合法；对应 invariance judgment 的 `mode=bitwise` | +| **非数值语义**（mask 形状、版本单调、manifest 字段） | 不迁 | +| **框架/集成单测**（bridge、vLLM mock、DeepSpeed worker 编排） | 非 WS1 op gate；可保留宽松 `allclose`，但**不能**当作 #266 EXIT 证据 | +| **生产 FA / SDPA 对齐**（`test_attention_correctness`） | 非 BI 候选路径；阈值可独立，**不得**写进 WS1 EXIT claim | +| **legacy `accuracy` 键** | 仅兼容；新代码禁止新增依赖，应改 `resolve_tolerance` | + +### 0.4 建议迁移句式 + +```python +# BAD — 私有阈值 +torch.testing.assert_close(out, ref, atol=1e-5, rtol=1e-5) + +# GOOD — accuracy +spec = resolve_tolerance(contract, judgment="forward_accuracy", op_class="reduction", dtype=dtype) +torch.testing.assert_close(out, ref, atol=spec.atol, rtol=spec.rtol) + +# GOOD — invariance +ispec = resolve_tolerance(contract, judgment="forward_invariance", op_class="attention", dtype=dtype) +assert ispec.mode == "bitwise" and ispec.atol == 0.0 +assert torch.equal(a, b) # 或 assert_close(..., atol=0, rtol=0) + +# GOOD — gradient accuracy（独立 judgment） +gspec = resolve_tolerance(contract, judgment="gradient_accuracy", op_class="logprob", dtype=dtype) +``` + +--- + +## 1. 状态总表（`tests/`） + +图例： + +| 标记 | 含义 | +|------|------| +| **A** | 已走 resolver / gtest suite（目标态） | +| **B** | 走 `load_contract` 旧键或 gtest 间接路径（过渡） | +| **C** | WS1 相关但 **私有 atol**（应迁） | +| **D** | 多为 `torch.equal` / 结构断言（ok 或仅需声明 judgment） | +| **E** | 非 WS1 门禁（框架/产品路径，低优先级） | + +### 1.1 已对齐或接近 SSOT + +| 文件 | 状态 | 说明 | 下一步 | +|------|------|------|--------| +| `test_tolerance_contract.py` | **A** | C1 schema + resolve + 聚合 | 保持；契约变更必跑 | +| `test_op_checks.py` | **A/B** | suite 已按 judgment 解析；部分用例注入最小 contract | 新 fixture 尽量带 `judgments` | +| `test_operator_inputs.py` | **B** | 输入/规格，无数值阈值主责 | 无需迁阈值 | +| `test_swiglu.py` | **B/D** | issue-108 harness + 大量 `torch.equal` | accuracy 路径确认走 suite；字面量 atol 清零 | +| `test_det_gemm.py` | **B** | `load_contract()["accuracy"]...` | **优先迁**：改为 `resolve_tolerance(..., forward/gradient_accuracy)` | +| `test_deterministic_attention_cuda.py` | **B/C** | 部分用 suite；仍见 `5e-2/2e-2` 字面量 | 字面量改为 resolve；invariance 保持 equal | + +### 1.2 WS1 算子测试 — 私有阈值（应迁，按优先级） + +| 优先级 | 文件 | op_class 建议 | 现状摘要 | 何时必须迁 | +|--------|------|---------------|----------|------------| +| **P0** | `test_batch_invariant_logp.py` | `logprob` | 大量 `1e-6`…`1e-2` 私有；含 bwd | 接 C3/C4/C8 证据前 | +| **P0** | `test_linear_logp.py` | `logprob` | `1e-5`…`1.5e-1` 混用；bf16 松阈值 | 同上；链级改用三聚合 API | +| **P0** | `test_logp.py` / `test_deterministic_logp.py` | `logprob` | 私有 atol | 关 #148 residual / C8 前 | +| **P0** | `test_rms_norm.py` | `reduction` | `1e-5`…`8e-2`；bwd 混用 | C8 RMSNorm 证据前 | +| **P0** | `test_triton_batch_invariant_attention.py` | `attention` | 混 `1e-5` 与 `5e-2/2e-2` | C8 Attention 证据前 | +| **P0** | `test_attention.py` | `attention` | native GT；`1e-4`/`2e-6` 等 | 与 contract `attention` 行对齐 | +| **P1** | `test_kv_cache_attention.py` | `attention` | 含 `2e-6` 等；#152 相关 | **C6/C7 前必须**消私有 decode 阈值 | +| **P1** | `test_issue151_embedding_lm_head_invariance.py` | emb + lm_head + logp | bf16 `5e-2` 手写 | C8 emb/lm_head 证据前 | +| **P1** | `test_lm_head.py` | `reduction` | 多 equal；grad `1e-5` 私有 | 迁 grad → `gradient_accuracy` | +| **P1** | `test_embedding.py` | `elementwise` | 多为 equal | 若有 tolerance 路径再 resolve | +| **P1** | `test_rope.py` | `elementwise` | `1e-3`…`2e-2` | C5 RoPE 证据前 | +| **P1** | `test_matmul.py` | `reduction` | 私有 `1e-4/1e-5` | 与 det_gemm 统一 | +| **P2** | `test_pack.py` | `elementwise` | 几乎 equal；gradcheck `1e-6` | packing 纳入 #150 时 | +| **P2** | `test_grpo_loss.py` / `test_ratio_kl.py` | （loss，契约暂无独立 class） | `1e-4` 等 | 若进 chain 则扩展 op_class 或显式 N/A | +| **P3** | `test_attention_correctness.py` | 非 BI EXIT | FA/SDPA 私有表 | **不迁入 WS1 SSOT**；文档标明 out of WS1 claim | +| **P3** | `test_op_accuracy.py` | 杂项 harness | `1e-3` | 废弃或改走 `check_operator` + contract | + +### 1.3 非 WS1 门禁（低优先级 / 不阻塞 #267） + +| 文件 | 状态 | 说明 | +|------|------|------| +| `test_deepspeed_training_worker.py` | **E** | 训练 worker；`atol=1e-5` 编排级 | +| `test_stateless_training_contract.py` | **E** | 契约字段/数值 smoke | +| `test_rl_kernel_loss_step.py` | **E** | 端到端 loss 步 | +| `test_sampler_temperature.py` | **E** | 采样 | +| `test_weight_sync_bridge.py` 等 | **D/E** | bridge / IPC | +| `test_vllm_rollout_sampler.py` | **D/E** | vLLM mock | +| `test_alignment_model_wrappers.py` | **D/E** | wrapper 行为 | +| `test_rl_batch_fixture.py` | **D** | fixture 身份 | +| `test_stateless_executor.py` / `*_hf_integration*` | **D/E** | 执行器集成 | + +这些**不**作为 #266 Full WS1 EXIT 的数值证据来源；C10/C11 不得引用其私有阈值刷绿。 + +--- + +## 2. 按 #266 子 issue 的「何时必须迁」 + +| 子 issue | 阻塞迁移范围 | 完成信号 | +|----------|--------------|----------| +| **C1 #267** | 契约 + resolver + schema/报告测试 | **实现完成；待 CI / issue evidence** | +| **C3 #269** forward harness | 所有 **forward_accuracy / forward_invariance** 的 op 单测证据路径 | 无 private forward atol 作为 gate | +| **C4 #270** grad harness | 所有 **gradient_*** 证据路径 | 无「grad 抄 forward 字面量」 | +| **C5 #271** RoPE/elementwise | `test_rope.py`、activation/swiglu residual | audit 报告阈值均来自 resolve | +| **C6/C7 #272/#273** KV | `test_kv_cache_attention.py` 及后续 kv harness | **禁止** `_DECODE_ATOL` 类私有常量 | +| **C8 #274** closed-op 矩阵 | rmsnorm / gemm / attn / logp / emb / lm_head 测试 | 每格 `requested/actual backend` + resolve 阈值 | +| **C10 #276** 全模型 gate | 仅用 resolver + 三聚合；禁止任何测试内字面量阈值 | gate 报告无 private tol 字段 | +| **C11 #277** CI | CI 只跑 resolve 路径 | fail-closed | + +**规则：** 某 op 的 PR 若声称「满足 #266/C8」，则该 PR 触达的 assert **必须**来自 `resolve_tolerance` / 聚合 API，而不是文件顶部的魔法数。 + +--- + +## 3. 文件级迁移清单（可勾选） + +### 3.1 P0 — 直接挡 C3/C4/C8 + +- [ ] `tests/test_batch_invariant_logp.py` — fwd/bwd accuracy + invariance 拆 judgment +- [ ] `tests/test_linear_logp.py` — 同上；训推/链级改用三聚合 +- [ ] `tests/test_logp.py` +- [ ] `tests/test_deterministic_logp.py` +- [ ] `tests/test_rms_norm.py` +- [ ] `tests/test_triton_batch_invariant_attention.py` +- [ ] `tests/test_attention.py` +- [ ] `tests/test_det_gemm.py` — 去掉 `contract["accuracy"]` 直读 + +### 3.2 P1 — C5/C6/C7/C8 residual + +- [ ] `tests/test_kv_cache_attention.py` +- [ ] `tests/test_issue151_embedding_lm_head_invariance.py` +- [ ] `tests/test_lm_head.py` +- [ ] `tests/test_embedding.py`（若有 non-bitwise 路径） +- [ ] `tests/test_rope.py` +- [ ] `tests/test_matmul.py` +- [ ] `tests/test_deterministic_attention_cuda.py` 中剩余字面量 +- [ ] `tests/test_swiglu.py` 中任何 residual 字面量 + +### 3.3 P2 — 进 chain 时 + +- [ ] `tests/test_pack.py` +- [ ] `tests/test_grpo_loss.py` / `tests/test_ratio_kl.py`（先扩 contract op_class 或标 N/A） +- [ ] `scripts/check_operator.py` 报告字段确认只回传 resolve 结果（已间接） + +### 3.4 明确不迁入 WS1 SSOT + +- [x] `tests/test_attention_correctness.py` — 生产 FA；文档标注非 EXIT +- [x] bridge / vLLM / DeepSpeed / sampler 类 **E** 组 + +--- + +## 4. 推荐落地动作（每个测试文件） + +1. **分类每条 assert** + - identity / batch-invariance → `forward_invariance` 或 `gradient_invariance` + `torch.equal` + - vs fp32 gold → `*_accuracy` + - train vs infer logp → 三聚合，不用单点 atol 冒充 +2. **删除模块级 `_ATOL` / `_RTOL`** +3. **dtype 参数化** 时用 `resolve_tolerance(..., dtype=dtype)`，禁止 bf16 写死 `5e-2` +4. **报告**（若有）写上 `comparison_lhs_role` / `comparison_rhs_role`（从 spec 取） +5. **禁止** 为让 invariance 通过而调大 atol + +### 4.1 与契约行不一致时怎么办 + +| 情况 | 动作 | +|------|------| +| 测试私有更松，契约更紧 → 测试红 | **修 kernel** 或开 Blocker；**禁止**在测试放宽 | +| 测试私有更紧，契约更松 | 迁到契约后可能变绿；可保留额外严格 assert 但须标注 *non-gate* | +| 需要新 op_class（如 `grpo_loss`） | 先改 `tolerance_contract.json` + schema 测试，再迁测试 | +| decode vs prefill 无法 bitwise | 用 contract 已声明的 semantic 行 / 三聚合；**不要**私设 `_DECODE_ATOL` | + +--- + +## 5. 工具与 CI 建议（后续，非 C1 范围） + +| 建议 | 作用 | +|------|------| +| 简单 lint：`tests/**/*.py` 禁止 `atol=\d`（allowlist 契约测试与 FA 测试） | 防回流 | +| `pytest` marker：`ws1_gate` 仅收集 resolve 路径 | C11 门禁清晰 | +| 在 `check_operator.py` 输出中强制打印 `judgment` + roles | 证据可检索 | + +C1 **不**强制上 lint；C8/C10 前建议至少做 allowlist 扫描。 + +--- + +## 6. 现状一句话 + +| 层 | 状态 | +|----|------| +| **契约 + resolver（gtest 核心）** | 已就绪；待 CI / issue evidence（#267） | +| **op_checks 接入** | 已按 judgment 分叉，并持久化 roles / provenance | +| **存量 op 单测** | **多数仍私有 atol**（上表 P0/P1） | +| **#266 EXIT** | 依赖后续把 P0/P1 迁完，而不是只合 C1 | + +**C1 的价值是「唯一入口已存在」；清单的价值是「知道还欠哪些文件」。** +未完成 P0/P1 迁移前，**不得**声称「全仓测试已统一走 WS1 数值契约」。 + +--- + +## 7. 修订记录 + +| 日期 | 说明 | +|------|------| +| 2026-08-11 | 初版：C1 落地后基于 `tests/` 扫描的迁移清单与优先级 | diff --git a/docs/design/ws1-numerical-contract.md b/docs/design/ws1-numerical-contract.md new file mode 100644 index 00000000..76410fcb --- /dev/null +++ b/docs/design/ws1-numerical-contract.md @@ -0,0 +1,187 @@ +# WS1 Numerical Contract (C1 / #267) + +> **Parent:** [#266](https://github.com/RL-Align/RL-Kernel/issues/266) +> **Issue:** [#267](https://github.com/RL-Align/RL-Kernel/issues/267) +> **SSOT files:** `rl_engine/kernels/gtest/tolerance_contract.json`, `rl_engine/kernels/gtest/tolerance.py` + +This document freezes the **sole** numerical judgment source for WS1 ablations and +gates. New gates must obtain thresholds only through the shared resolver APIs; private +`atol` / `rtol` constants are forbidden. + +## Scope boundary + +**Allowed WS1 claim after full exit (#266):** single-GPU model-level train–inference +consistency for full Qwen3-8B Dense under required CUDA BF16 and Triton-on-CUDA BF16 +profiles (in-repo BI stack). + +**Not claimed here:** multi-GPU (WS2), real vLLM vs Megatron / vime product alignment +(WS3), or kernel bug fixes (open Blockers). + +## Dtype policy + +| Field | Lock | +| --- | --- | +| `execution_dtype` | **BF16** (mandatory) | +| `accumulation_dtype` | **FP32** | +| `reference_dtype` | **FP32** | +| `output_dtype.default` | follows execution | +| logprob aggregates compute dtype | **FP32** | +| FP8 | **out of scope** (request → hard fail) | +| FP16 | optional; rows complete when declared | +| TF32 (reference + candidate) | **disabled** (repo-wide single policy) | +| Backend profiles | `cuda_bf16`, `triton_cuda_bf16` (same thresholds) | +| Backend-private tolerance relaxation | **forbidden** | + +Execution, accumulation, output, and reference dtypes resolve **independently** via +`resolve_dtype_policy()`. + +## Four judgments + +| Judgment | What it compares | Default mode | +| --- | --- | --- | +| `forward_accuracy` | BF16 candidate vs FP32 reference outputs | tolerance | +| `forward_invariance` | transformed vs canonical config, same backend/dtype/logical workload | **bitwise** (`atol=0`, `rtol=0`) | +| `gradient_accuracy` | candidate gradient/VJP vs FP32 reference gradient/VJP | tolerance (independent of forward) | +| `gradient_invariance` | transformed vs canonical gradients, same logical workload | **bitwise** (`atol=0`, `rtol=0`) | + +Every declared-applicable `(judgment, op_class, dtype)` tuple must resolve. Missing +applicable cells hard-fail. Explicit `not_applicable` / `out_of_scope` is allowed only +when present in the schema. Use `resolve_tolerance_support()` to persist the explicit +support status; requesting thresholds for an N/A or out-of-scope cell still hard-fails. + +**Batch/Chunk invariance** (issue #150 / C10 matrix) **must** use the invariance +judgments in bitwise mode. Nonzero tolerance cannot satisfy that gate. + +Op classes: `elementwise`, `reduction`, `logprob`, `attention`. + +## Comparison roles + +Reports must record `comparison_lhs_role` / `comparison_rhs_role`. A bare `baseline` +field is forbidden. C2 `singleton_aggregate` is an **execution/aggregation mode**, not +a comparison role. + +| Report kind | `comparison_lhs_role` | `comparison_rhs_role` | +| --- | --- | --- | +| `forward_accuracy` | `bf16_candidate` | `fp32_reference` | +| `forward_invariance` | `transformed_config` | `canonical_config` | +| `train_infer_logprob_parity` | `training_style_teacher_forcing` | `inference_style_rollout_decode` | +| `gradient_accuracy` | `bf16_candidate` | `fp32_reference` | +| `gradient_invariance` | `transformed_config` | `canonical_config` | + +Direction is locked so train/infer preserves: + +```text +dlogp = train_logp - rollout_logp +ratio0 = exp(dlogp) +``` + +Swapping lhs/rhs without a different declared contract row hard-fails. + +API: `resolve_comparison_roles()`, `assert_comparison_roles()`. + +Aggregate callers must also provide `contract`, `report_kind`, and both roles; the +compute API validates the direction before calculating any metric. gtest reports +persist these roles on every output verdict. Backend reports must include +`BackendProvenance` (requested/actual backend, all four dtypes, and TF32 state), +which `validate_backend_provenance()` checks against the selected profile. + +## Chain-level logprob aggregates + +These three metrics are the **only** chain-level logprob / ablation aggregates for WS1 +pass/fail. Gradients use independent `gradient_*` tensor verdicts and **do not** use +these aggregates. + +Computed in FP32 on **active selected tokens only**: + +```text +dlogp = comparison_lhs_logp - comparison_rhs_logp +max_abs_dlogp = max(abs(dlogp)) +approx_kl0 = mean(exp(dlogp) - 1 - dlogp) +clipfrac0 = mean(1[exp(dlogp) outside clip_interval]) +``` + +Rules: + +- **All three** must pass (`require_all=true`). +- Empty active-token set → hard fail. +- NaN / Inf in `dlogp`, `ratio0`, or aggregates → hard fail. +- Clip interval is pinned by the workload manifest (C2); the contract stores a default + and the field name `clip_interval`. + +API: `compute_logprob_aggregates()`, `judge_logprob_aggregates()`, +`resolve_chain_aggregate_thresholds()`. + +## Resolver usage + +```python +from rl_engine.kernels.gtest.tolerance import ( + load_contract, + resolve_tolerance, + resolve_dtype_policy, + compute_logprob_aggregates, + judge_logprob_aggregates, + default_clip_interval, +) + +contract = load_contract() +policy = resolve_dtype_policy(contract) + +fwd = resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="logprob", + dtype="bfloat16", + backend_profile="cuda_bf16", +) +bwd = resolve_tolerance( + contract, + judgment="gradient_accuracy", + op_class="logprob", + dtype="bfloat16", + backend_profile="triton_cuda_bf16", # same thresholds as cuda_bf16 +) +inv = resolve_tolerance( + contract, + judgment="forward_invariance", + op_class="attention", + dtype="bfloat16", +) +# inv.mode == "bitwise", inv.atol == 0.0, inv.rtol == 0.0 +``` + +`op_checks.run_operator_suite` resolves **forward_accuracy** for outputs and +**gradient_accuracy** for gradients. + +## Compatibility keys + +For older tests that still dig into: + +- `contract["accuracy"]["default"][op_class][dtype]` — mirror of `forward_accuracy` +- `contract["batch_invariance"]` — `{atol: 0, rtol: 0}` + +New code should call the resolvers above. Schema validation fails if the compatibility +mirror drifts from `forward_accuracy` or if invariance rows leave bitwise mode. + +## Related issues + +| ID | Role | +| --- | --- | +| #266 | WS1 closeout parent | +| #267 | This contract (C1) | +| #268 | Full-model workload / clip interval pin in manifest | +| #269 / #270 | Forward / gradient invariance harnesses | +| #276 | Full-model train/infer gate consuming this contract | +| #154 / #108 | Historical contract owners (superseded remaining work → C1) | + +## Migration of existing tests + +Most operator tests still use **private** `atol` / `rtol` literals. That is expected +after C1: the SSOT exists, but call sites have not all moved. + +See the full inventory, priority, and “when it must migrate” map: + +- [WS1 gtest 阈值迁移清单](ws1-gtest-migration-checklist.md) + +How to register ops and run the CLI (post-#267): + +- [gtest usage guide](../contributing/gtest-usage.md) diff --git a/rl_engine/kernels/gtest/__init__.py b/rl_engine/kernels/gtest/__init__.py index c3fc3665..61b43218 100644 --- a/rl_engine/kernels/gtest/__init__.py +++ b/rl_engine/kernels/gtest/__init__.py @@ -2,9 +2,21 @@ # Copyright (c) 2026 RL-Kernel Contributors from .op_checks import CandidateSpec, OperatorCase, run_operator_suite +from .tolerance import ( + BackendProvenance, + ContractResolveError, + ContractSchemaError, + resolve_tolerance_support, + validate_backend_provenance, +) __all__ = [ "CandidateSpec", "OperatorCase", "run_operator_suite", + "BackendProvenance", + "ContractResolveError", + "ContractSchemaError", + "resolve_tolerance_support", + "validate_backend_provenance", ] diff --git a/rl_engine/kernels/gtest/op_checks.py b/rl_engine/kernels/gtest/op_checks.py index efea31a3..085b5f89 100644 --- a/rl_engine/kernels/gtest/op_checks.py +++ b/rl_engine/kernels/gtest/op_checks.py @@ -9,7 +9,13 @@ import torch -from rl_engine.kernels.gtest.tolerance import load_contract +from rl_engine.kernels.gtest.tolerance import ( + BackendProvenance, + ContractResolveError, + load_contract, + resolve_tolerance, + validate_backend_provenance, +) @dataclass(frozen=True) @@ -32,6 +38,7 @@ class CandidateSpec: fn: Callable[..., Any] | Any backend: str = "unknown" arch_key: str | None = None + provenance: BackendProvenance | None = None @dataclass(frozen=True) @@ -48,6 +55,9 @@ class OutputCheck: mean_abs_error: float max_rel_error: float passed: bool + judgment: str + comparison_lhs_role: str + comparison_rhs_role: str message: str = "" @@ -73,6 +83,7 @@ class CandidateReport: pass_rate: float passed: bool cases: list[CaseCheck] + backend_provenance: BackendProvenance | None = None @dataclass(frozen=True) @@ -140,6 +151,19 @@ def _run_candidate( grad_mode: str, grad_seed: int, ) -> CandidateReport: + if candidate.provenance is not None: + validate_backend_provenance(contract, candidate.provenance) + if candidate.backend != candidate.provenance.actual_backend: + raise ContractResolveError( + f"candidate backend {candidate.backend!r} disagrees with reported actual backend " + f"{candidate.provenance.actual_backend!r}" + ) + for case in cases: + if _dtype_name(case.dtype) != candidate.provenance.execution_dtype: + raise ContractResolveError( + f"case {case.name!r} dtype {case.dtype} does not match " + f"provenance execution_dtype {candidate.provenance.execution_dtype!r}" + ) if check_grad: case_checks = [ _run_case_backward( @@ -164,6 +188,7 @@ def _run_candidate( pass_rate=pass_rate, passed=passed_outputs == total_outputs, cases=case_checks, + backend_provenance=candidate.provenance, ) @@ -216,14 +241,29 @@ def _run_case_backward( candidate_outputs, gold_outputs, ).outputs - # Reuse the same tolerance class for gradients as for values. This is a - # first conservative default; operator-specific gradient tolerances can be - # split out later if a real backend shows different numerical behavior. + # Gradient thresholds come from the independent gradient_accuracy judgment + # (#267); they must not silently inherit forward_accuracy rows. atol, rtol = _resolve_tolerance( contract, op_class=case.op_class, dtype=case.dtype, arch_key=candidate.arch_key, + backend_profile=(candidate.provenance.backend_profile if candidate.provenance else None), + judgment="gradient_accuracy", + ) + gradient_spec = ( + resolve_tolerance( + contract, + judgment="gradient_accuracy", + op_class=case.op_class, + dtype=case.dtype, + arch_key=candidate.arch_key, + backend_profile=( + candidate.provenance.backend_profile if candidate.provenance else None + ), + ) + if "judgments" in contract + else None ) grad_checks = [ _compare_output( @@ -232,6 +272,13 @@ def _run_case_backward( output_index=len(output_checks) + index, atol=atol, rtol=rtol, + judgment="gradient_accuracy", + comparison_lhs_role=( + gradient_spec.comparison_lhs_role if gradient_spec is not None else "bf16_candidate" + ), + comparison_rhs_role=( + gradient_spec.comparison_rhs_role if gradient_spec is not None else "fp32_reference" + ), message=f"gradient:{name}", ) for index, (name, candidate_grad, gold_grad) in enumerate( @@ -265,7 +312,37 @@ def _compare_case_outputs( op_class=case.op_class, dtype=case.dtype, arch_key=candidate.arch_key, + backend_profile=(candidate.provenance.backend_profile if candidate.provenance else None), + judgment="forward_accuracy", ) + forward_spec = ( + resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class=case.op_class, + dtype=case.dtype, + arch_key=candidate.arch_key, + backend_profile=( + candidate.provenance.backend_profile if candidate.provenance else None + ), + ) + if "judgments" in contract + else None + ) + if candidate.provenance is not None: + for candidate_output, gold_output in zip(candidate_outputs, gold_outputs, strict=True): + candidate_dtype = _dtype_name(candidate_output.dtype) + gold_dtype = _dtype_name(gold_output.dtype) + if candidate_dtype != candidate.provenance.output_dtype: + raise ContractResolveError( + f"candidate output dtype {candidate_dtype!r} disagrees with provenance " + f"output_dtype {candidate.provenance.output_dtype!r}" + ) + if gold_dtype != candidate.provenance.reference_dtype: + raise ContractResolveError( + f"gold output dtype {gold_dtype!r} disagrees with provenance " + f"reference_dtype {candidate.provenance.reference_dtype!r}" + ) output_checks = [ _compare_output( candidate_output, @@ -273,6 +350,13 @@ def _compare_case_outputs( output_index=index, atol=atol, rtol=rtol, + judgment="forward_accuracy", + comparison_lhs_role=( + forward_spec.comparison_lhs_role if forward_spec is not None else "bf16_candidate" + ), + comparison_rhs_role=( + forward_spec.comparison_rhs_role if forward_spec is not None else "fp32_reference" + ), ) for index, (candidate_output, gold_output) in enumerate( zip(candidate_outputs, gold_outputs, strict=True) @@ -407,7 +491,27 @@ def _resolve_tolerance( op_class: str, dtype: torch.dtype, arch_key: str | None = None, + backend_profile: str | None = None, + judgment: str = "forward_accuracy", ) -> tuple[float, float]: + """Resolve thresholds via the shared four-judgment contract (#267). + + Falls back to the legacy ``accuracy`` mirror only when the four-judgment + block is absent (older fixture contracts in unit tests). + """ + + if "judgments" in contract: + spec = resolve_tolerance( + contract, + judgment=judgment, + op_class=op_class, + dtype=dtype, + arch_key=arch_key, + backend_profile=backend_profile, + ) + return float(spec.atol), float(spec.rtol) + + # Legacy fixtures used by some unit tests that inject a minimal contract. dtype_name = _dtype_name(dtype) if arch_key is not None: arch_values = ( @@ -441,6 +545,9 @@ def _compare_output( output_index: int, atol: float, rtol: float, + judgment: str = "forward_accuracy", + comparison_lhs_role: str = "bf16_candidate", + comparison_rhs_role: str = "fp32_reference", message: str = "", ) -> OutputCheck: if candidate.shape != gold.shape: @@ -455,6 +562,9 @@ def _compare_output( mean_abs_error=float("inf"), max_rel_error=float("inf"), passed=False, + judgment=judgment, + comparison_lhs_role=comparison_lhs_role, + comparison_rhs_role=comparison_rhs_role, message=f"shape mismatch: candidate={tuple(candidate.shape)} gold={tuple(gold.shape)}", ) @@ -482,6 +592,9 @@ def _compare_output( mean_abs_error=mean_abs_error, max_rel_error=max_rel_error, passed=bool(torch.allclose(candidate_fp32, gold_fp32, atol=atol, rtol=rtol)), + judgment=judgment, + comparison_lhs_role=comparison_lhs_role, + comparison_rhs_role=comparison_rhs_role, message=message, ) diff --git a/rl_engine/kernels/gtest/tolerance.py b/rl_engine/kernels/gtest/tolerance.py index d0481e83..230dd95d 100644 --- a/rl_engine/kernels/gtest/tolerance.py +++ b/rl_engine/kernels/gtest/tolerance.py @@ -1,20 +1,977 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026 RL-Kernel Contributors +"""WS1 numerical contract loader and resolver (#267 / C1 of #266). + +This module is the sole authority for: +- dtype policy (BF16 execution, FP32 reference/accumulation, FP8 out) +- four-judgment tolerances +- comparison roles +- chain-level logprob aggregates (max_abs_dlogp / approx_kl0 / clipfrac0) + +Gates must obtain thresholds only through the resolvers defined here. +""" + from __future__ import annotations import json +import math +from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any +from typing import Any, Mapping, Sequence _CONTRACT_PATH = Path(__file__).with_name("tolerance_contract.json") +JUDGMENTS = ( + "forward_accuracy", + "forward_invariance", + "gradient_accuracy", + "gradient_invariance", +) +OP_CLASSES = ("elementwise", "reduction", "logprob", "attention") +MANDATORY_DTYPES = ("float32", "bfloat16") +OPTIONAL_DTYPES = ("float16",) +OUT_OF_SCOPE_DTYPES = ("float8",) +ALL_DTYPES = MANDATORY_DTYPES + OPTIONAL_DTYPES + OUT_OF_SCOPE_DTYPES +CHAIN_AGGREGATE_METRICS = ("max_abs_dlogp", "approx_kl0", "clipfrac0") +INVARIANCE_JUDGMENTS = ("forward_invariance", "gradient_invariance") +REPORT_KINDS = ( + "forward_accuracy", + "forward_invariance", + "train_infer_logprob_parity", + "gradient_accuracy", + "gradient_invariance", +) + + +class ContractError(ValueError): + """Base error for contract load / resolve failures.""" + + +class ContractSchemaError(ContractError): + """Contract JSON failed schema validation.""" + + +class ContractResolveError(ContractError): + """A resolve request cannot be satisfied under the contract.""" + + +@dataclass(frozen=True) +class DtypePolicy: + """Resolved WS1 dtype / TF32 / FP8 policy.""" + + execution_dtype: str + accumulation_dtype: str + reference_dtype: str + output_dtype_default: str + logprob_aggregates_dtype: str + fp8: str + fp16_status: str + tf32_reference: str + tf32_candidate_execution: str + backend_profiles: tuple[str, ...] + backend_private_tolerance_relaxation: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class BackendProvenance: + """Actual backend and dtype facts persisted by a WS1 report.""" + + backend_profile: str + requested_backend: str + actual_backend: str + execution_dtype: str + accumulation_dtype: str + output_dtype: str + reference_dtype: str + candidate_tf32_enabled: bool + reference_tf32_enabled: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class ToleranceSupport: + """Schema-level support result, including explicit N/A and out-of-scope cells.""" + + judgment: str + op_class: str + dtype_name: str + status: str + reason: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class ComparisonRoles: + """lhs/rhs roles for a report kind.""" + + report_kind: str + comparison_lhs_role: str + comparison_rhs_role: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class ToleranceSpec: + """Resolved tolerance for one (judgment, op_class, dtype) request.""" + + judgment: str + op_class: str + dtype_name: str + status: str + mode: str + atol: float + rtol: float + comparison_lhs_role: str + comparison_rhs_role: str + backend_profile: str | None = None + arch_key: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) -def load_contract(path: str | Path = _CONTRACT_PATH) -> dict[str, Any]: - """Load the dtype/operator-class tolerance contract.""" + +@dataclass(frozen=True) +class LogprobAggregates: + """Three chain-level logprob aggregates (FP32).""" + + max_abs_dlogp: float + approx_kl0: float + clipfrac0: float + active_token_count: int + clip_interval: tuple[float, float] + report_kind: str + comparison_lhs_role: str + comparison_rhs_role: str + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["clip_interval"] = list(self.clip_interval) + return data + + +@dataclass(frozen=True) +class AggregateMetricVerdict: + metric: str + value: float + threshold: float + passed: bool + + +@dataclass(frozen=True) +class LogprobAggregateVerdict: + aggregates: LogprobAggregates + metrics: tuple[AggregateMetricVerdict, ...] + passed: bool + report_kind: str + comparison_lhs_role: str + comparison_rhs_role: str + + def to_dict(self) -> dict[str, Any]: + return { + "aggregates": self.aggregates.to_dict(), + "metrics": [asdict(m) for m in self.metrics], + "passed": self.passed, + "report_kind": self.report_kind, + "comparison_lhs_role": self.comparison_lhs_role, + "comparison_rhs_role": self.comparison_rhs_role, + } + + +def load_contract( + path: str | Path = _CONTRACT_PATH, + *, + validate: bool = True, +) -> dict[str, Any]: + """Load the WS1 dtype/operator-class tolerance contract.""" with Path(path).open("r", encoding="utf-8") as handle: - return json.load(handle) + contract = json.load(handle) + if validate: + validate_contract_schema(contract) + return contract + + +def validate_contract_schema(contract: Mapping[str, Any]) -> None: + """Validate four-judgment schema, dtype policy, roles, and aggregates.""" + + if not isinstance(contract, Mapping): + raise ContractSchemaError("contract must be a mapping") + + for key in ( + "version", + "policy", + "comparison_roles", + "judgments", + "chain_logprob_aggregates", + ): + if key not in contract: + raise ContractSchemaError(f"contract missing required key {key!r}") + + _validate_policy(contract["policy"]) + _validate_comparison_roles(contract["comparison_roles"]) + _validate_judgments(contract["judgments"]) + _validate_chain_aggregates(contract["chain_logprob_aggregates"]) + _validate_compat_views(contract) + + +def resolve_dtype_policy(contract: Mapping[str, Any]) -> DtypePolicy: + """Resolve independent execution / accumulation / output / reference dtypes.""" + + policy = contract["policy"] + output = policy["output_dtype"] + tf32 = policy["tf32"] + fp16 = policy["fp16"] + return DtypePolicy( + execution_dtype=str(policy["execution_dtype"]), + accumulation_dtype=str(policy["accumulation_dtype"]), + reference_dtype=str(policy["reference_dtype"]), + output_dtype_default=( + str(policy["execution_dtype"]) + if output["default"] == "execution" + else str(output["default"]) + ), + logprob_aggregates_dtype=str(output["logprob_aggregates"]), + fp8=str(policy["fp8"]), + fp16_status=str(fp16["status"]), + tf32_reference=str(tf32["reference"]), + tf32_candidate_execution=str(tf32["candidate_execution"]), + backend_profiles=tuple(str(p) for p in policy["backend_profiles"]), + backend_private_tolerance_relaxation=bool(policy["backend_private_tolerance_relaxation"]), + ) + + +def validate_backend_provenance( + contract: Mapping[str, Any], + provenance: BackendProvenance, +) -> BackendProvenance: + """Fail closed when reported backend or dtype facts violate the WS1 profile.""" + + policy = resolve_dtype_policy(contract) + if provenance.backend_profile not in policy.backend_profiles: + raise ContractResolveError(f"unknown backend_profile {provenance.backend_profile!r}") + profile_contract = contract["policy"]["backend_profile_contracts"][provenance.backend_profile] + expected_backend = str(profile_contract["backend_family"]) + for field_name, actual in ( + ("requested_backend", provenance.requested_backend), + ("actual_backend", provenance.actual_backend), + ): + if actual != expected_backend: + raise ContractResolveError( + f"backend provenance mismatch for {field_name}: expected " + f"{expected_backend!r}, got {actual!r}" + ) + + expected_dtypes = { + "execution_dtype": policy.execution_dtype, + "accumulation_dtype": policy.accumulation_dtype, + "output_dtype": policy.output_dtype_default, + "reference_dtype": policy.reference_dtype, + } + for field_name, expected in expected_dtypes.items(): + actual = _dtype_name(getattr(provenance, field_name)) + if actual != expected: + raise ContractResolveError( + f"backend provenance mismatch for {field_name}: expected " + f"{expected!r}, got {actual!r}" + ) + for field_name in ("candidate_tf32_enabled", "reference_tf32_enabled"): + if getattr(provenance, field_name): + raise ContractResolveError( + f"backend provenance reports {field_name}=true; WS1 requires disabled" + ) + return provenance + + +def resolve_comparison_roles( + contract: Mapping[str, Any], + report_kind: str, +) -> ComparisonRoles: + """Return lhs/rhs roles for a report kind.""" + + roles_root = contract["comparison_roles"] + forbidden = set(roles_root.get("forbidden", ())) + by_kind = roles_root["by_report_kind"] + if report_kind not in by_kind: + raise ContractResolveError(f"unknown report_kind {report_kind!r}") + entry = by_kind[report_kind] + lhs = str(entry["comparison_lhs_role"]) + rhs = str(entry["comparison_rhs_role"]) + for role in (lhs, rhs): + if role in forbidden: + raise ContractResolveError( + f"forbidden comparison role {role!r} for report_kind {report_kind!r}" + ) + if role not in roles_root["allowed"]: + raise ContractResolveError( + f"unknown comparison role {role!r} for report_kind {report_kind!r}" + ) + return ComparisonRoles( + report_kind=report_kind, + comparison_lhs_role=lhs, + comparison_rhs_role=rhs, + ) + + +def assert_comparison_roles( + contract: Mapping[str, Any], + report_kind: str, + comparison_lhs_role: str, + comparison_rhs_role: str, +) -> ComparisonRoles: + """Hard-fail if report roles are reversed, unknown, or forbidden.""" + + expected = resolve_comparison_roles(contract, report_kind) + if comparison_lhs_role in contract["comparison_roles"].get("forbidden", ()): + raise ContractResolveError(f"forbidden comparison_lhs_role {comparison_lhs_role!r}") + if comparison_rhs_role in contract["comparison_roles"].get("forbidden", ()): + raise ContractResolveError(f"forbidden comparison_rhs_role {comparison_rhs_role!r}") + if ( + comparison_lhs_role != expected.comparison_lhs_role + or comparison_rhs_role != expected.comparison_rhs_role + ): + raise ContractResolveError( + f"role mismatch for {report_kind!r}: expected " + f"lhs={expected.comparison_lhs_role!r}, rhs={expected.comparison_rhs_role!r}; " + f"got lhs={comparison_lhs_role!r}, rhs={comparison_rhs_role!r}" + ) + return expected + + +def resolve_tolerance( + contract: Mapping[str, Any], + *, + judgment: str, + op_class: str, + dtype: str | Any, + arch_key: str | None = None, + backend_profile: str | None = None, +) -> ToleranceSpec: + """Resolve one four-judgment tolerance cell. + + ``cuda_bf16`` and ``triton_cuda_bf16`` share the same rows. Backend-private + threshold relaxation is forbidden. + """ + + if judgment not in JUDGMENTS: + raise ContractResolveError(f"unknown judgment {judgment!r}") + if op_class not in OP_CLASSES: + raise ContractResolveError(f"unknown op_class {op_class!r}") + + dtype_name = _dtype_name(dtype) + policy = resolve_dtype_policy(contract) + + if backend_profile is not None: + if backend_profile not in policy.backend_profiles: + raise ContractResolveError( + f"unknown backend_profile {backend_profile!r}; " + f"allowed={list(policy.backend_profiles)}" + ) + if policy.backend_private_tolerance_relaxation: + raise ContractResolveError( + "backend_private_tolerance_relaxation must remain false under WS1 C1" + ) + + if dtype_name in OUT_OF_SCOPE_DTYPES or policy.fp8 == "out_of_scope" and dtype_name == "float8": + raise ContractResolveError( + f"dtype {dtype_name!r} is out of scope for WS1 (FP8 requests hard-fail)" + ) + + support = resolve_tolerance_support( + contract, + judgment=judgment, + op_class=op_class, + dtype=dtype_name, + arch_key=arch_key, + ) + judgment_root = contract["judgments"][judgment] + cell = _lookup_cell(judgment_root, op_class=op_class, dtype_name=dtype_name, arch_key=arch_key) + if cell is None: + raise ContractResolveError( + f"missing declared cell for judgment={judgment!r}, " + f"op_class={op_class!r}, dtype={dtype_name!r}" + ) + + status = support.status + if status == "out_of_scope": + raise ContractResolveError( + f"cell out_of_scope for judgment={judgment!r}, " + f"op_class={op_class!r}, dtype={dtype_name!r}" + ) + if status == "not_applicable": + raise ContractResolveError( + f"cell not_applicable for judgment={judgment!r}, " + f"op_class={op_class!r}, dtype={dtype_name!r}; " + "callers must not request non-applicable judgments without an explicit N/A path" + ) + if status not in {"applicable", "optional"}: + raise ContractResolveError( + f"invalid status {status!r} for judgment={judgment!r}, " + f"op_class={op_class!r}, dtype={dtype_name!r}" + ) + + mode = str(cell.get("mode", judgment_root.get("default_mode", "tolerance"))) + if "atol" not in cell or "rtol" not in cell: + raise ContractResolveError( + f"cell missing atol/rtol for judgment={judgment!r}, " + f"op_class={op_class!r}, dtype={dtype_name!r}" + ) + atol = float(cell["atol"]) + rtol = float(cell["rtol"]) + + if judgment in INVARIANCE_JUDGMENTS and status == "applicable": + if mode != "bitwise" or atol != 0.0 or rtol != 0.0: + raise ContractResolveError( + f"Batch/Chunk invariance requires bitwise atol=0 rtol=0; got " + f"mode={mode!r}, atol={atol}, rtol={rtol} for {judgment}/{op_class}/{dtype_name}" + ) + + roles = resolve_comparison_roles(contract, judgment) + return ToleranceSpec( + judgment=judgment, + op_class=op_class, + dtype_name=dtype_name, + status=status, + mode=mode, + atol=atol, + rtol=rtol, + comparison_lhs_role=roles.comparison_lhs_role, + comparison_rhs_role=roles.comparison_rhs_role, + backend_profile=backend_profile, + arch_key=arch_key, + ) + + +def resolve_tolerance_support( + contract: Mapping[str, Any], + *, + judgment: str, + op_class: str, + dtype: str | Any, + arch_key: str | None = None, +) -> ToleranceSupport: + """Resolve schema support without pretending N/A cells have thresholds.""" + + if judgment not in JUDGMENTS: + raise ContractResolveError(f"unknown judgment {judgment!r}") + if op_class not in OP_CLASSES: + raise ContractResolveError(f"unknown op_class {op_class!r}") + dtype_name = _dtype_name(dtype) + cell = _lookup_cell( + contract["judgments"][judgment], + op_class=op_class, + dtype_name=dtype_name, + arch_key=arch_key, + ) + if cell is None: + raise ContractResolveError( + f"missing declared cell for judgment={judgment!r}, " + f"op_class={op_class!r}, dtype={dtype_name!r}" + ) + status = str(cell.get("status", "")) + if status not in {"applicable", "optional", "not_applicable", "out_of_scope"}: + raise ContractResolveError( + f"invalid support status {status!r} for {judgment}/{op_class}/{dtype_name}" + ) + reason = cell.get("reason") + return ToleranceSupport( + judgment=judgment, + op_class=op_class, + dtype_name=dtype_name, + status=status, + reason=str(reason) if reason is not None else None, + ) + + +def resolve_chain_aggregate_thresholds( + contract: Mapping[str, Any], + metric_name: str, + execution_dtype: str | Any, +) -> float: + """Named resolve for max_abs_dlogp / approx_kl0 / clipfrac0 thresholds.""" + + if metric_name not in CHAIN_AGGREGATE_METRICS: + raise ContractResolveError( + f"unknown chain aggregate metric {metric_name!r}; " + f"only {list(CHAIN_AGGREGATE_METRICS)} are allowed" + ) + dtype_name = _dtype_name(execution_dtype) + metrics = contract["chain_logprob_aggregates"]["metrics"] + by_dtype = metrics[metric_name]["by_execution_dtype"] + if dtype_name not in by_dtype: + raise ContractResolveError( + f"missing chain aggregate threshold for metric={metric_name!r}, " + f"execution_dtype={dtype_name!r}" + ) + return float(by_dtype[dtype_name]["threshold"]) + + +def compute_logprob_aggregates( + lhs_logp: Any, + rhs_logp: Any, + active_mask: Any, + *, + contract: Mapping[str, Any], + report_kind: str, + clip_interval: Sequence[float] | tuple[float, float], + comparison_lhs_role: str, + comparison_rhs_role: str, +) -> LogprobAggregates: + """Compute the three chain-level logprob aggregates in FP32. + + ``dlogp = lhs_logp - rhs_logp`` on active selected tokens only. + Empty active set / NaN / Inf → hard fail. + """ + + assert_comparison_roles(contract, report_kind, comparison_lhs_role, comparison_rhs_role) + + try: + import torch + except ImportError as exc: # pragma: no cover + raise ContractResolveError("torch is required for aggregate computation") from exc + + if len(clip_interval) != 2: + raise ContractResolveError("clip_interval must be a length-2 [lo, hi] pair") + lo, hi = float(clip_interval[0]), float(clip_interval[1]) + if not (lo < hi): + raise ContractResolveError(f"clip_interval requires lo < hi, got [{lo}, {hi}]") + + lhs = torch.as_tensor(lhs_logp).detach().float().reshape(-1) + rhs = torch.as_tensor(rhs_logp).detach().float().reshape(-1) + mask = torch.as_tensor(active_mask).detach().reshape(-1).bool() + if lhs.shape != rhs.shape or lhs.shape != mask.shape: + raise ContractResolveError( + f"lhs/rhs/mask shape mismatch: {tuple(lhs.shape)} vs " + f"{tuple(rhs.shape)} vs {tuple(mask.shape)}" + ) + active = int(mask.sum().item()) + if active == 0: + raise ContractResolveError("empty active-token set is a hard fail for logprob aggregates") + + dlogp = lhs[mask] - rhs[mask] + if not torch.isfinite(dlogp).all(): + raise ContractResolveError("NaN/Inf in dlogp is a hard fail for logprob aggregates") + + ratio0 = torch.exp(dlogp) + if not torch.isfinite(ratio0).all(): + raise ContractResolveError("NaN/Inf in ratio0 is a hard fail for logprob aggregates") + + max_abs_dlogp = float(dlogp.abs().max().item()) + approx_kl0 = float((ratio0 - 1.0 - dlogp).mean().item()) + outside = (ratio0 < lo) | (ratio0 > hi) + clipfrac0 = float(outside.float().mean().item()) + + for name, value in ( + ("max_abs_dlogp", max_abs_dlogp), + ("approx_kl0", approx_kl0), + ("clipfrac0", clipfrac0), + ): + if not math.isfinite(value): + raise ContractResolveError(f"NaN/Inf in aggregate {name} is a hard fail") + + return LogprobAggregates( + max_abs_dlogp=max_abs_dlogp, + approx_kl0=approx_kl0, + clipfrac0=clipfrac0, + active_token_count=active, + clip_interval=(lo, hi), + report_kind=report_kind, + comparison_lhs_role=comparison_lhs_role, + comparison_rhs_role=comparison_rhs_role, + ) + + +def judge_logprob_aggregates( + aggregates: LogprobAggregates, + contract: Mapping[str, Any], + *, + execution_dtype: str | Any, + clip_interval: Sequence[float] | tuple[float, float] | None = None, +) -> LogprobAggregateVerdict: + """Judge all three chain logprob aggregates; all must pass.""" + + assert_comparison_roles( + contract, + aggregates.report_kind, + aggregates.comparison_lhs_role, + aggregates.comparison_rhs_role, + ) + + if clip_interval is not None: + lo, hi = float(clip_interval[0]), float(clip_interval[1]) + if (lo, hi) != aggregates.clip_interval: + raise ContractResolveError( + "clip_interval mismatch between compute and judge " + f"(computed={aggregates.clip_interval}, judge=({lo}, {hi}))" + ) + + metrics: list[AggregateMetricVerdict] = [] + for name in CHAIN_AGGREGATE_METRICS: + threshold = resolve_chain_aggregate_thresholds(contract, name, execution_dtype) + value = float(getattr(aggregates, name)) + if not math.isfinite(value): + raise ContractResolveError(f"NaN/Inf in aggregate {name} is a hard fail") + metrics.append( + AggregateMetricVerdict( + metric=name, + value=value, + threshold=threshold, + passed=value <= threshold, + ) + ) + require_all = bool(contract["chain_logprob_aggregates"].get("require_all", True)) + passed = all(m.passed for m in metrics) if require_all else any(m.passed for m in metrics) + return LogprobAggregateVerdict( + aggregates=aggregates, + metrics=tuple(metrics), + passed=passed, + report_kind=aggregates.report_kind, + comparison_lhs_role=aggregates.comparison_lhs_role, + comparison_rhs_role=aggregates.comparison_rhs_role, + ) + + +def default_clip_interval(contract: Mapping[str, Any]) -> tuple[float, float]: + """Return the contract default clip interval for clipfrac0.""" + + interval = contract["chain_logprob_aggregates"]["default_clip_interval"] + return float(interval[0]), float(interval[1]) + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _validate_policy(policy: Mapping[str, Any]) -> None: + required = ( + "execution_dtype", + "accumulation_dtype", + "reference_dtype", + "output_dtype", + "fp8", + "fp16", + "tf32", + "backend_profiles", + "backend_profile_contracts", + "backend_private_tolerance_relaxation", + ) + for key in required: + if key not in policy: + raise ContractSchemaError(f"policy missing {key!r}") + if policy["execution_dtype"] != "bfloat16": + raise ContractSchemaError("policy.execution_dtype must be bfloat16 for WS1") + if policy["accumulation_dtype"] != "float32": + raise ContractSchemaError("policy.accumulation_dtype must be float32 for WS1") + if policy["reference_dtype"] != "float32": + raise ContractSchemaError("policy.reference_dtype must be float32 for WS1") + if policy["fp8"] != "out_of_scope": + raise ContractSchemaError("policy.fp8 must be out_of_scope for WS1") + output = policy["output_dtype"] + for key in ("default", "logprob_aggregates"): + if key not in output: + raise ContractSchemaError(f"policy.output_dtype missing {key!r}") + if output["logprob_aggregates"] != "float32": + raise ContractSchemaError("logprob aggregates must be computed in float32") + if output["default"] != "execution": + raise ContractSchemaError("policy.output_dtype.default must follow execution") + if policy["fp16"].get("status") != "optional": + raise ContractSchemaError("policy.fp16.status must be optional for WS1") + tf32 = policy["tf32"] + for key in ("reference", "candidate_execution"): + if key not in tf32: + raise ContractSchemaError(f"policy.tf32 missing {key!r}") + if tf32[key] != "disabled": + raise ContractSchemaError( + f"policy.tf32.{key} must be 'disabled' under the WS1 single policy" + ) + profiles = list(policy["backend_profiles"]) + profile_contracts = policy["backend_profile_contracts"] + required_profile_families = { + "cuda_bf16": "cuda", + "triton_cuda_bf16": "triton", + } + for required_profile, expected_family in required_profile_families.items(): + if required_profile not in profiles: + raise ContractSchemaError(f"policy.backend_profiles must include {required_profile!r}") + if required_profile not in profile_contracts: + raise ContractSchemaError( + f"policy.backend_profile_contracts missing {required_profile!r}" + ) + actual_family = profile_contracts[required_profile].get("backend_family") + if actual_family != expected_family: + raise ContractSchemaError( + f"profile {required_profile!r} requires backend_family " + f"{expected_family!r}, got {actual_family!r}" + ) + if policy["backend_private_tolerance_relaxation"] is not False: + raise ContractSchemaError("backend_private_tolerance_relaxation must be false") + + +def _validate_comparison_roles(roles_root: Mapping[str, Any]) -> None: + for key in ("allowed", "forbidden", "by_report_kind"): + if key not in roles_root: + raise ContractSchemaError(f"comparison_roles missing {key!r}") + forbidden = set(roles_root["forbidden"]) + for name in ("baseline", "singleton_aggregate"): + if name not in forbidden: + raise ContractSchemaError(f"comparison_roles.forbidden must include {name!r}") + by_kind = roles_root["by_report_kind"] + for kind in REPORT_KINDS: + if kind not in by_kind: + raise ContractSchemaError(f"comparison_roles.by_report_kind missing {kind!r}") + entry = by_kind[kind] + for role_key in ("comparison_lhs_role", "comparison_rhs_role"): + if role_key not in entry: + raise ContractSchemaError( + f"comparison_roles.by_report_kind[{kind!r}] missing {role_key!r}" + ) + role = entry[role_key] + if role in forbidden: + raise ContractSchemaError(f"report_kind {kind!r} uses forbidden role {role!r}") + if role not in roles_root["allowed"]: + raise ContractSchemaError(f"report_kind {kind!r} uses unknown role {role!r}") + + +def _validate_judgments(judgments: Mapping[str, Any]) -> None: + for judgment in JUDGMENTS: + if judgment not in judgments: + raise ContractSchemaError(f"judgments missing {judgment!r}") + root = judgments[judgment] + if "by_op_class" not in root: + raise ContractSchemaError(f"judgments[{judgment!r}] missing by_op_class") + by_op = root["by_op_class"] + for op_class in OP_CLASSES: + if op_class not in by_op: + raise ContractSchemaError( + f"judgments[{judgment!r}].by_op_class missing {op_class!r}" + ) + dtype_map = by_op[op_class] + for dtype_name in ALL_DTYPES: + if dtype_name not in dtype_map: + raise ContractSchemaError( + f"missing cell judgments[{judgment!r}][{op_class!r}][{dtype_name!r}]" + ) + cell = dtype_map[dtype_name] + status = cell.get("status") + if status is None: + raise ContractSchemaError( + f"cell missing status: {judgment}/{op_class}/{dtype_name}" + ) + if dtype_name in OUT_OF_SCOPE_DTYPES: + if status != "out_of_scope": + raise ContractSchemaError( + f"FP8 cell must be out_of_scope: {judgment}/{op_class}/{dtype_name}" + ) + continue + if status == "not_applicable" and not cell.get("reason"): + raise ContractSchemaError( + f"not_applicable cell requires reason: {judgment}/{op_class}/{dtype_name}" + ) + if dtype_name in MANDATORY_DTYPES and status not in { + "applicable", + "not_applicable", + }: + # BF16/FP32 must be explicitly applicable (or explicit N/A). + if status != "applicable": + raise ContractSchemaError( + f"mandatory dtype cell must be applicable: " + f"{judgment}/{op_class}/{dtype_name} status={status!r}" + ) + if status in {"applicable", "optional"}: + for thr in ("atol", "rtol", "mode"): + if thr not in cell: + raise ContractSchemaError( + f"cell missing {thr}: {judgment}/{op_class}/{dtype_name}" + ) + if judgment in INVARIANCE_JUDGMENTS and status == "applicable": + mode = cell.get("mode") + atol = float(cell.get("atol", 1.0)) + rtol = float(cell.get("rtol", 1.0)) + if mode != "bitwise" or atol != 0.0 or rtol != 0.0: + raise ContractSchemaError( + f"invariance applicable cells must be bitwise 0/0: " + f"{judgment}/{op_class}/{dtype_name}" + ) + + +def _validate_chain_aggregates(root: Mapping[str, Any]) -> None: + for key in ( + "compute_dtype", + "require_all", + "nan_inf_policy", + "empty_active_token_set", + "dlogp_definition", + "default_clip_interval", + "sole_chain_level_logprob_metrics", + "metrics", + ): + if key not in root: + raise ContractSchemaError(f"chain_logprob_aggregates missing {key!r}") + if root["compute_dtype"] != "float32": + raise ContractSchemaError("chain aggregates must use compute_dtype=float32") + if root["nan_inf_policy"] != "hard_fail": + raise ContractSchemaError("nan_inf_policy must be hard_fail") + if root["empty_active_token_set"] != "hard_fail": + raise ContractSchemaError("empty_active_token_set must be hard_fail") + if not root["require_all"]: + raise ContractSchemaError("require_all must be true for chain logprob aggregates") + sole = list(root["sole_chain_level_logprob_metrics"]) + if set(sole) != set(CHAIN_AGGREGATE_METRICS) or len(sole) != 3: + raise ContractSchemaError( + "sole_chain_level_logprob_metrics must be exactly " f"{list(CHAIN_AGGREGATE_METRICS)}" + ) + interval = root["default_clip_interval"] + if len(interval) != 2 or float(interval[0]) >= float(interval[1]): + raise ContractSchemaError("default_clip_interval must be [lo, hi] with lo < hi") + metrics = root["metrics"] + for name in CHAIN_AGGREGATE_METRICS: + if name not in metrics: + raise ContractSchemaError(f"chain metrics missing {name!r}") + by_dtype = metrics[name].get("by_execution_dtype") + if not isinstance(by_dtype, Mapping): + raise ContractSchemaError(f"metric {name!r} missing by_execution_dtype") + for dtype_name in ("bfloat16", "float32"): + if dtype_name not in by_dtype or "threshold" not in by_dtype[dtype_name]: + raise ContractSchemaError(f"metric {name!r} missing threshold for {dtype_name}") + + +def _validate_compat_views(contract: Mapping[str, Any]) -> None: + """Legacy accuracy / batch_invariance must mirror the four-judgment SSOT.""" + + if "batch_invariance" not in contract: + raise ContractSchemaError("compat key batch_invariance is required") + bi = contract["batch_invariance"] + if float(bi.get("atol", 1.0)) != 0.0 or float(bi.get("rtol", 1.0)) != 0.0: + raise ContractSchemaError("batch_invariance must remain bitwise 0/0") + + if "accuracy" not in contract: + raise ContractSchemaError("compat key accuracy is required") + accuracy = contract["accuracy"]["default"] + fwd = contract["judgments"]["forward_accuracy"]["by_op_class"] + for op_class in OP_CLASSES: + if op_class not in accuracy: + raise ContractSchemaError(f"compat accuracy missing op_class {op_class!r}") + for dtype_name in MANDATORY_DTYPES + OPTIONAL_DTYPES: + if dtype_name not in accuracy[op_class]: + raise ContractSchemaError(f"compat accuracy missing {op_class}/{dtype_name}") + cell = fwd[op_class][dtype_name] + if cell.get("status") not in {"applicable", "optional"}: + continue + acc = accuracy[op_class][dtype_name] + if float(acc["atol"]) != float(cell["atol"]) or float(acc["rtol"]) != float( + cell["rtol"] + ): + raise ContractSchemaError( + f"compat accuracy mismatch vs forward_accuracy for " f"{op_class}/{dtype_name}" + ) + + +def _lookup_cell( + judgment_root: Mapping[str, Any], + *, + op_class: str, + dtype_name: str, + arch_key: str | None, +) -> Mapping[str, Any] | None: + if arch_key is not None: + arch_cell = ( + judgment_root.get("arch_overrides", {}) + .get(arch_key, {}) + .get(op_class, {}) + .get(dtype_name) + ) + if arch_cell is not None: + return arch_cell + return judgment_root.get("by_op_class", {}).get(op_class, {}).get(dtype_name) + + +def _dtype_name(dtype: str | Any) -> str: + if isinstance(dtype, str): + name = dtype + # Accept torch-style aliases. + aliases = { + "torch.float32": "float32", + "torch.bfloat16": "bfloat16", + "torch.float16": "float16", + "torch.float8": "float8", + "fp32": "float32", + "bf16": "bfloat16", + "fp16": "float16", + "fp8": "float8", + } + name = aliases.get(name, name) + if name not in ALL_DTYPES: + raise ContractResolveError(f"unsupported dtype name {dtype!r}") + return name + + # torch.dtype without importing torch at module import time for non-torch tests. + module = getattr(type(dtype), "__module__", "") + qual = getattr(dtype, "name", None) or str(dtype) + if module.startswith("torch") or "torch" in str(type(dtype)): + mapping = { + "torch.float32": "float32", + "torch.bfloat16": "bfloat16", + "torch.float16": "float16", + "float32": "float32", + "bfloat16": "bfloat16", + "float16": "float16", + } + # torch.dtype str is like "torch.float32" + as_str = str(dtype) + if as_str in mapping: + return mapping[as_str] + if qual in mapping: + return mapping[qual] + try: + import torch + + if dtype is torch.float32: + return "float32" + if dtype is torch.bfloat16: + return "bfloat16" + if dtype is torch.float16: + return "float16" + except ImportError: # pragma: no cover + pass + raise ContractResolveError(f"unsupported dtype: {dtype!r}") -__all__ = ["load_contract"] +__all__ = [ + "ALL_DTYPES", + "CHAIN_AGGREGATE_METRICS", + "JUDGMENTS", + "OP_CLASSES", + "AggregateMetricVerdict", + "BackendProvenance", + "ComparisonRoles", + "ContractError", + "ContractResolveError", + "ContractSchemaError", + "DtypePolicy", + "LogprobAggregateVerdict", + "LogprobAggregates", + "ToleranceSpec", + "ToleranceSupport", + "assert_comparison_roles", + "compute_logprob_aggregates", + "default_clip_interval", + "judge_logprob_aggregates", + "load_contract", + "resolve_chain_aggregate_thresholds", + "resolve_comparison_roles", + "resolve_dtype_policy", + "resolve_tolerance", + "resolve_tolerance_support", + "validate_backend_provenance", + "validate_contract_schema", +] diff --git a/rl_engine/kernels/gtest/tolerance_contract.json b/rl_engine/kernels/gtest/tolerance_contract.json index 975ae450..e7645b75 100644 --- a/rl_engine/kernels/gtest/tolerance_contract.json +++ b/rl_engine/kernels/gtest/tolerance_contract.json @@ -1,5 +1,239 @@ { - "batch_invariance": {"atol": 0.0, "rtol": 0.0}, + "version": "ws1-c1-v1", + "policy": { + "execution_dtype": "bfloat16", + "accumulation_dtype": "float32", + "reference_dtype": "float32", + "output_dtype": { + "default": "execution", + "logprob_aggregates": "float32" + }, + "fp8": "out_of_scope", + "fp16": { + "status": "optional", + "note": "FP16 rows are complete when declared; not mandatory for WS1 EXIT." + }, + "tf32": { + "reference": "disabled", + "candidate_execution": "disabled", + "policy": "Repo-wide single policy: TF32 is disabled for FP32 reference and for candidate execution under this contract." + }, + "backend_profiles": ["cuda_bf16", "triton_cuda_bf16"], + "backend_profile_contracts": { + "cuda_bf16": {"backend_family": "cuda"}, + "triton_cuda_bf16": {"backend_family": "triton"} + }, + "backend_private_tolerance_relaxation": false + }, + "comparison_roles": { + "allowed": [ + "bf16_candidate", + "fp32_reference", + "canonical_config", + "transformed_config", + "training_style_teacher_forcing", + "inference_style_rollout_decode" + ], + "forbidden": ["baseline", "singleton_aggregate"], + "by_report_kind": { + "forward_accuracy": { + "comparison_lhs_role": "bf16_candidate", + "comparison_rhs_role": "fp32_reference" + }, + "forward_invariance": { + "comparison_lhs_role": "transformed_config", + "comparison_rhs_role": "canonical_config" + }, + "train_infer_logprob_parity": { + "comparison_lhs_role": "training_style_teacher_forcing", + "comparison_rhs_role": "inference_style_rollout_decode" + }, + "gradient_accuracy": { + "comparison_lhs_role": "bf16_candidate", + "comparison_rhs_role": "fp32_reference" + }, + "gradient_invariance": { + "comparison_lhs_role": "transformed_config", + "comparison_rhs_role": "canonical_config" + } + } + }, + "judgments": { + "forward_accuracy": { + "default_mode": "tolerance", + "by_op_class": { + "elementwise": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-5, "rtol": 1.0e-5}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 2.0e-2, "rtol": 1.6e-2}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 1.0e-3, "rtol": 1.0e-3}, + "float8": {"status": "out_of_scope"} + }, + "reduction": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-4, "rtol": 1.0e-4}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 5.0e-2, "rtol": 2.0e-2}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 1.0e-3, "rtol": 1.0e-3}, + "float8": {"status": "out_of_scope"} + }, + "logprob": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-5, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 5.0e-2, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 5.0e-3, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "attention": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-4, "rtol": 1.0e-4}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 5.0e-2, "rtol": 2.0e-2}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 1.0e-3, "rtol": 1.0e-3}, + "float8": {"status": "out_of_scope"} + } + }, + "arch_overrides": { + "sm90": {} + } + }, + "forward_invariance": { + "default_mode": "bitwise", + "scope": "batch_chunk_padding_layout", + "by_op_class": { + "elementwise": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "reduction": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "logprob": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "attention": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + } + }, + "arch_overrides": { + "sm90": {} + } + }, + "gradient_accuracy": { + "default_mode": "tolerance", + "by_op_class": { + "elementwise": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-5, "rtol": 1.0e-5}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 2.0e-2, "rtol": 1.6e-2}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 1.0e-3, "rtol": 1.0e-3}, + "float8": {"status": "out_of_scope"} + }, + "reduction": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-4, "rtol": 1.0e-4}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 5.0e-2, "rtol": 2.0e-2}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 1.0e-3, "rtol": 1.0e-3}, + "float8": {"status": "out_of_scope"} + }, + "logprob": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-5, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 5.0e-2, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 5.0e-3, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "attention": { + "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-4, "rtol": 1.0e-4}, + "bfloat16": {"status": "applicable", "mode": "tolerance", "atol": 5.0e-2, "rtol": 2.0e-2}, + "float16": {"status": "optional", "mode": "tolerance", "atol": 1.0e-3, "rtol": 1.0e-3}, + "float8": {"status": "out_of_scope"} + } + }, + "arch_overrides": { + "sm90": {} + } + }, + "gradient_invariance": { + "default_mode": "bitwise", + "scope": "batch_chunk_padding_layout", + "by_op_class": { + "elementwise": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "reduction": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "logprob": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + }, + "attention": { + "float32": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "bfloat16": {"status": "applicable", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float16": {"status": "optional", "mode": "bitwise", "atol": 0.0, "rtol": 0.0}, + "float8": {"status": "out_of_scope"} + } + }, + "arch_overrides": { + "sm90": {} + } + } + }, + "chain_logprob_aggregates": { + "compute_dtype": "float32", + "require_all": true, + "nan_inf_policy": "hard_fail", + "empty_active_token_set": "hard_fail", + "active_token_policy": "active selected tokens only", + "dlogp_definition": "comparison_lhs_logp - comparison_rhs_logp", + "clip_interval_field": "clip_interval", + "default_clip_interval": [0.8, 1.2], + "sole_chain_level_logprob_metrics": [ + "max_abs_dlogp", + "approx_kl0", + "clipfrac0" + ], + "metrics": { + "max_abs_dlogp": { + "formula": "max(abs(dlogp))", + "pass_rule": "value <= threshold", + "by_execution_dtype": { + "bfloat16": {"threshold": 5.0e-2}, + "float32": {"threshold": 1.0e-5}, + "float16": {"threshold": 5.0e-3} + } + }, + "approx_kl0": { + "formula": "mean(exp(dlogp) - 1 - dlogp)", + "pass_rule": "value <= threshold", + "by_execution_dtype": { + "bfloat16": {"threshold": 5.0e-2}, + "float32": {"threshold": 1.0e-5}, + "float16": {"threshold": 5.0e-3} + } + }, + "clipfrac0": { + "formula": "mean(1[exp(dlogp) outside clip_interval])", + "pass_rule": "value <= threshold", + "by_execution_dtype": { + "bfloat16": {"threshold": 0.0}, + "float32": {"threshold": 0.0}, + "float16": {"threshold": 0.0} + } + } + } + }, "accuracy": { "default": { "elementwise": { @@ -26,5 +260,6 @@ "arch_overrides": { "sm90": {} } - } + }, + "batch_invariance": {"atol": 0.0, "rtol": 0.0} } diff --git a/tests/test_op_checks.py b/tests/test_op_checks.py index e076e106..35de05a5 100644 --- a/tests/test_op_checks.py +++ b/tests/test_op_checks.py @@ -5,6 +5,7 @@ import argparse +import pytest import torch from rl_engine.kernels.gtest.op_checks import CandidateSpec, OperatorCase, run_operator_suite @@ -13,6 +14,7 @@ make_operator_case, operator_names, ) +from rl_engine.kernels.gtest.tolerance import BackendProvenance, ContractResolveError from rl_engine.kernels.ops.pytorch.linear.embedding import NativeEmbeddingOp from rl_engine.kernels.ops.pytorch.linear.lm_head import NativeLMHeadOp from rl_engine.kernels.ops.pytorch.loss.logp import NativeLogpOp @@ -119,7 +121,11 @@ def test_embedding_native_candidate_suite_passes_issue_108_helper(): ) assert report.passed - assert report.candidates[0].cases[0].outputs[1].message == "gradient:weight" + gradient = report.candidates[0].cases[0].outputs[1] + assert gradient.message == "gradient:weight" + assert gradient.judgment == "gradient_accuracy" + assert gradient.comparison_lhs_role == "bf16_candidate" + assert gradient.comparison_rhs_role == "fp32_reference" def test_lm_head_native_candidate_suite_passes_issue_108_helper(): @@ -182,6 +188,97 @@ def test_suite_report_to_dict_contains_error_metrics(): assert "passed" in output +def test_ws1_report_persists_roles_and_backend_provenance(): + provenance = BackendProvenance( + backend_profile="cuda_bf16", + requested_backend="cuda", + actual_backend="cuda", + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, + ) + report = run_operator_suite( + "logp", + candidates=[ + CandidateSpec( + name="cuda-logp", + backend="cuda", + fn=NativeLogpOp(), + provenance=provenance, + ) + ], + cases=[_logp_case("bf16", torch.bfloat16, seed=12)], + ) + output = report.candidates[0].cases[0].outputs[0] + assert output.judgment == "forward_accuracy" + assert output.comparison_lhs_role == "bf16_candidate" + assert output.comparison_rhs_role == "fp32_reference" + data = report.to_dict()["candidates"][0] + assert data["backend_provenance"]["actual_backend"] == "cuda" + assert "baseline" not in data["cases"][0]["outputs"][0] + + +def test_ws1_report_rejects_backend_provenance_mismatch(): + provenance = BackendProvenance( + backend_profile="cuda_bf16", + requested_backend="cuda", + actual_backend="triton", + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, + ) + with pytest.raises(ContractResolveError, match="actual_backend"): + run_operator_suite( + "logp", + candidates=[ + CandidateSpec( + name="bad", + backend="triton", + fn=NativeLogpOp(), + provenance=provenance, + ) + ], + cases=[_logp_case("bf16", torch.bfloat16, seed=13)], + ) + + +def test_ws1_report_checks_observed_output_dtype_against_provenance(): + provenance = BackendProvenance( + backend_profile="cuda_bf16", + requested_backend="cuda", + actual_backend="cuda", + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, + ) + + def wrong_output_dtype(logits, token_ids): + return NativeLogpOp().forward(logits, token_ids).float() + + with pytest.raises(ContractResolveError, match="candidate output dtype"): + run_operator_suite( + "logp", + candidates=[ + CandidateSpec( + name="wrong-output", + backend="cuda", + fn=wrong_output_dtype, + provenance=provenance, + ) + ], + cases=[_logp_case("bf16", torch.bfloat16, seed=14)], + ) + + def test_candidate_arch_key_uses_tolerance_override(): def slightly_shifted_logp(logits, token_ids): return NativeLogpOp().forward_fp32(logits, token_ids) + 0.02 diff --git a/tests/test_tolerance_contract.py b/tests/test_tolerance_contract.py index 5eb75cbd..5f711061 100644 --- a/tests/test_tolerance_contract.py +++ b/tests/test_tolerance_contract.py @@ -1,9 +1,36 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026 RL-Kernel Contributors +"""Schema and resolver tests for WS1 C1 four-judgment contract (#267).""" + from __future__ import annotations -from rl_engine.kernels.gtest.tolerance import load_contract +import copy +import math + +import pytest +import torch + +from rl_engine.kernels.gtest.tolerance import ( + CHAIN_AGGREGATE_METRICS, + JUDGMENTS, + OP_CLASSES, + BackendProvenance, + ContractResolveError, + ContractSchemaError, + assert_comparison_roles, + compute_logprob_aggregates, + default_clip_interval, + judge_logprob_aggregates, + load_contract, + resolve_chain_aggregate_thresholds, + resolve_comparison_roles, + resolve_dtype_policy, + resolve_tolerance, + resolve_tolerance_support, + validate_backend_provenance, + validate_contract_schema, +) def test_load_contract_contains_expected_operator_classes(): @@ -34,3 +61,420 @@ def test_attention_bfloat16_tolerance_matches_contract(): tolerance = contract["accuracy"]["default"]["attention"]["bfloat16"] assert tolerance["atol"] >= 5.0e-2 assert tolerance["rtol"] >= 2.0e-2 + + +def test_contract_schema_validates_on_load(): + contract = load_contract(validate=True) + validate_contract_schema(contract) + + +def test_dtype_policy_locks_bf16_fp32_fp8_tf32(): + policy = resolve_dtype_policy(load_contract()) + assert policy.execution_dtype == "bfloat16" + assert policy.accumulation_dtype == "float32" + assert policy.reference_dtype == "float32" + assert policy.output_dtype_default == "bfloat16" + assert policy.logprob_aggregates_dtype == "float32" + assert policy.fp8 == "out_of_scope" + assert policy.fp16_status == "optional" + assert policy.tf32_reference == "disabled" + assert policy.tf32_candidate_execution == "disabled" + assert "cuda_bf16" in policy.backend_profiles + assert "triton_cuda_bf16" in policy.backend_profiles + assert policy.backend_private_tolerance_relaxation is False + + +def test_four_judgments_present_and_complete(): + contract = load_contract() + assert set(contract["judgments"]) == set(JUDGMENTS) + for judgment in JUDGMENTS: + by_op = contract["judgments"][judgment]["by_op_class"] + assert set(by_op) == set(OP_CLASSES) + for op_class in OP_CLASSES: + for dtype_name in ("float32", "bfloat16", "float16", "float8"): + assert dtype_name in by_op[op_class] + + +def test_invariance_rows_are_bitwise_zero(): + contract = load_contract() + for judgment in ("forward_invariance", "gradient_invariance"): + for op_class in OP_CLASSES: + for dtype_name in ("float32", "bfloat16"): + spec = resolve_tolerance( + contract, + judgment=judgment, + op_class=op_class, + dtype=dtype_name, + ) + assert spec.mode == "bitwise" + assert spec.atol == 0.0 + assert spec.rtol == 0.0 + + +def test_cuda_and_triton_profiles_share_thresholds(): + contract = load_contract() + for profile in ("cuda_bf16", "triton_cuda_bf16"): + a = resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="logprob", + dtype="bfloat16", + backend_profile=profile, + ) + b = resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="logprob", + dtype="bfloat16", + backend_profile="cuda_bf16" if profile != "cuda_bf16" else "triton_cuda_bf16", + ) + assert a.atol == b.atol and a.rtol == b.rtol and a.mode == b.mode + + +def test_unknown_backend_profile_hard_fails(): + contract = load_contract() + with pytest.raises(ContractResolveError, match="backend_profile"): + resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="reduction", + dtype="bfloat16", + backend_profile="private_backend", + ) + + +def test_backend_provenance_checks_profile_backend_and_all_dtypes(): + contract = load_contract() + provenance = BackendProvenance( + backend_profile="cuda_bf16", + requested_backend="cuda", + actual_backend="cuda", + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, + ) + assert validate_backend_provenance(contract, provenance) == provenance + with pytest.raises(ContractResolveError, match="actual_backend"): + validate_backend_provenance( + contract, + BackendProvenance(**{**provenance.to_dict(), "actual_backend": "triton"}), + ) + with pytest.raises(ContractResolveError, match="output_dtype"): + validate_backend_provenance( + contract, + BackendProvenance(**{**provenance.to_dict(), "output_dtype": "float32"}), + ) + with pytest.raises(ContractResolveError, match="candidate_tf32_enabled"): + validate_backend_provenance( + contract, + BackendProvenance(**{**provenance.to_dict(), "candidate_tf32_enabled": True}), + ) + + +def test_not_applicable_has_explicit_support_result_but_no_threshold(): + contract = copy.deepcopy(load_contract()) + cell = contract["judgments"]["forward_accuracy"]["by_op_class"]["elementwise"]["float16"] + cell["status"] = "not_applicable" + cell["reason"] = "profile does not declare FP16" + validate_contract_schema(contract) + support = resolve_tolerance_support( + contract, judgment="forward_accuracy", op_class="elementwise", dtype="float16" + ) + assert support.status == "not_applicable" + with pytest.raises(ContractResolveError, match="not_applicable"): + resolve_tolerance( + contract, judgment="forward_accuracy", op_class="elementwise", dtype="float16" + ) + + +def test_fp8_request_hard_fails(): + contract = load_contract() + with pytest.raises(ContractResolveError, match="out of scope"): + resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="reduction", + dtype="float8", + ) + + +def test_missing_applicable_cell_hard_fails(): + contract = copy.deepcopy(load_contract()) + del contract["judgments"]["forward_accuracy"]["by_op_class"]["attention"]["bfloat16"] + with pytest.raises(ContractSchemaError): + validate_contract_schema(contract) + # Resolver path: re-insert schema-invalid by skipping validate, then resolve. + with pytest.raises(ContractResolveError, match="missing declared cell"): + resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="attention", + dtype="bfloat16", + ) + + +def test_gradient_thresholds_do_not_inherit_forward(): + contract = copy.deepcopy(load_contract()) + # Mutate only forward_accuracy BF16 reduction. + contract["judgments"]["forward_accuracy"]["by_op_class"]["reduction"]["bfloat16"]["atol"] = 9.9 + # Keep compat mirror in sync is not required for this unit test of independence. + fwd = resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="reduction", + dtype="bfloat16", + ) + grad = resolve_tolerance( + contract, + judgment="gradient_accuracy", + op_class="reduction", + dtype="bfloat16", + ) + assert fwd.atol == 9.9 + assert grad.atol == 5.0e-2 + assert grad.atol != fwd.atol + + +def test_comparison_roles_by_report_kind(): + contract = load_contract() + expected = { + "forward_accuracy": ("bf16_candidate", "fp32_reference"), + "forward_invariance": ("transformed_config", "canonical_config"), + "train_infer_logprob_parity": ( + "training_style_teacher_forcing", + "inference_style_rollout_decode", + ), + "gradient_accuracy": ("bf16_candidate", "fp32_reference"), + "gradient_invariance": ("transformed_config", "canonical_config"), + } + for kind, (lhs, rhs) in expected.items(): + roles = resolve_comparison_roles(contract, kind) + assert roles.comparison_lhs_role == lhs + assert roles.comparison_rhs_role == rhs + assert_comparison_roles(contract, kind, lhs, rhs) + + +def test_forbidden_and_reversed_roles_hard_fail(): + contract = load_contract() + with pytest.raises(ContractResolveError, match="role mismatch"): + assert_comparison_roles( + contract, + "train_infer_logprob_parity", + "inference_style_rollout_decode", + "training_style_teacher_forcing", + ) + + +def test_aggregate_requires_declared_roles_and_direction(): + contract = load_contract() + values = torch.zeros(2) + with pytest.raises(ContractResolveError, match="role mismatch"): + compute_logprob_aggregates( + values, + values, + torch.ones(2, dtype=torch.bool), + contract=contract, + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="inference_style_rollout_decode", + comparison_rhs_role="training_style_teacher_forcing", + ) + with pytest.raises(ContractResolveError, match="forbidden"): + assert_comparison_roles( + contract, + "forward_accuracy", + "baseline", + "fp32_reference", + ) + with pytest.raises(ContractResolveError, match="forbidden"): + assert_comparison_roles( + contract, + "forward_invariance", + "singleton_aggregate", + "canonical_config", + ) + + +def test_resolve_tolerance_attaches_roles(): + contract = load_contract() + spec = resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="logprob", + dtype=torch.bfloat16, + ) + assert spec.comparison_lhs_role == "bf16_candidate" + assert spec.comparison_rhs_role == "fp32_reference" + assert "baseline" not in (spec.comparison_lhs_role, spec.comparison_rhs_role) + + +def test_chain_aggregate_named_resolve(): + contract = load_contract() + for metric in CHAIN_AGGREGATE_METRICS: + thr = resolve_chain_aggregate_thresholds(contract, metric, "bfloat16") + assert thr >= 0.0 + with pytest.raises(ContractResolveError, match="unknown chain aggregate"): + resolve_chain_aggregate_thresholds(contract, "mean_abs_dlogp", "bfloat16") + + +def test_compute_logprob_aggregates_formulas(): + # lhs - rhs = [0.0, 0.1, -0.2] + lhs = torch.tensor([1.0, 2.1, 0.8], dtype=torch.float32) + rhs = torch.tensor([1.0, 2.0, 1.0], dtype=torch.float32) + mask = torch.tensor([True, True, True]) + clip = (0.8, 1.2) + agg = compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=clip, + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + dlogp = torch.tensor([0.0, 0.1, -0.2]) + expected_max = float(dlogp.abs().max()) + expected_kl = float((torch.exp(dlogp) - 1.0 - dlogp).mean()) + ratio = torch.exp(dlogp) + expected_clip = float(((ratio < clip[0]) | (ratio > clip[1])).float().mean()) + assert math.isclose(agg.max_abs_dlogp, expected_max, rel_tol=0.0, abs_tol=1e-6) + assert math.isclose(agg.approx_kl0, expected_kl, rel_tol=0.0, abs_tol=1e-6) + assert math.isclose(agg.clipfrac0, expected_clip, rel_tol=0.0, abs_tol=1e-6) + assert agg.active_token_count == 3 + + +def test_active_mask_filters_tokens(): + lhs = torch.tensor([0.0, 10.0], dtype=torch.float32) + rhs = torch.tensor([0.0, 0.0], dtype=torch.float32) + mask = torch.tensor([True, False]) + agg = compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=default_clip_interval(load_contract()), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + assert agg.max_abs_dlogp == 0.0 + assert agg.active_token_count == 1 + + +def test_empty_active_set_hard_fails(): + lhs = torch.zeros(2) + rhs = torch.zeros(2) + mask = torch.tensor([False, False]) + with pytest.raises(ContractResolveError, match="empty active-token"): + compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + + +def test_nan_inf_hard_fail(): + lhs = torch.tensor([float("nan"), 0.0]) + rhs = torch.tensor([0.0, 0.0]) + mask = torch.tensor([True, True]) + with pytest.raises(ContractResolveError, match="NaN/Inf"): + compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + + lhs = torch.tensor([float("inf"), 0.0]) + with pytest.raises(ContractResolveError, match="NaN/Inf"): + compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + + +def test_judge_requires_all_three_aggregates(): + contract = load_contract() + clip = default_clip_interval(contract) + # Perfect match → all pass. + lhs = torch.zeros(4) + rhs = torch.zeros(4) + mask = torch.ones(4, dtype=torch.bool) + agg = compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=contract, + report_kind="train_infer_logprob_parity", + clip_interval=clip, + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + verdict = judge_logprob_aggregates(agg, contract, execution_dtype="bfloat16") + assert verdict.passed + assert {m.metric for m in verdict.metrics} == set(CHAIN_AGGREGATE_METRICS) + assert all(m.passed for m in verdict.metrics) + + # Large drift fails max_abs_dlogp / approx_kl0 / possibly clipfrac. + lhs = torch.tensor([0.0, 1.0]) + rhs = torch.zeros(2) + mask = torch.ones(2, dtype=torch.bool) + agg = compute_logprob_aggregates( + lhs, + rhs, + mask, + contract=contract, + report_kind="train_infer_logprob_parity", + clip_interval=clip, + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + verdict = judge_logprob_aggregates(agg, contract, execution_dtype="bfloat16") + assert not verdict.passed + assert any(not m.passed for m in verdict.metrics) + + +def test_compat_accuracy_mirrors_forward_accuracy(): + contract = load_contract() + for op_class in OP_CLASSES: + for dtype_name in ("float32", "bfloat16", "float16"): + acc = contract["accuracy"]["default"][op_class][dtype_name] + cell = contract["judgments"]["forward_accuracy"]["by_op_class"][op_class][dtype_name] + assert acc["atol"] == cell["atol"] + assert acc["rtol"] == cell["rtol"] + assert contract["batch_invariance"] == {"atol": 0.0, "rtol": 0.0} + + +def test_schema_rejects_nonzero_invariance_tolerance(): + contract = copy.deepcopy(load_contract()) + contract["judgments"]["forward_invariance"]["by_op_class"]["logprob"]["bfloat16"]["atol"] = 1e-3 + with pytest.raises(ContractSchemaError, match="bitwise"): + validate_contract_schema(contract) + + +def test_schema_rejects_baseline_role(): + contract = copy.deepcopy(load_contract()) + contract["comparison_roles"]["by_report_kind"]["forward_accuracy"][ + "comparison_lhs_role" + ] = "baseline" + with pytest.raises(ContractSchemaError, match="forbidden role"): + validate_contract_schema(contract) From 70286fb9f18247d197a700433a1dade50a0f2503 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 00:27:38 +0800 Subject: [PATCH 02/10] docs(ws1): add #267 C1 closeout evidence map Record acceptance-criteria mapping, verification commands, and residual scope so issue #267 can close without implying full #266 exit. --- docs/design/ws1-c1-267-closeout-evidence.md | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/design/ws1-c1-267-closeout-evidence.md diff --git a/docs/design/ws1-c1-267-closeout-evidence.md b/docs/design/ws1-c1-267-closeout-evidence.md new file mode 100644 index 00000000..4005b02d --- /dev/null +++ b/docs/design/ws1-c1-267-closeout-evidence.md @@ -0,0 +1,72 @@ +# #267 (C1) closeout evidence + +**Issue:** [#267](https://github.com/RL-Align/RL-Kernel/issues/267) +**Parent:** [#266](https://github.com/RL-Align/RL-Kernel/issues/266) (C1 only; does **not** close #266) +**Branch:** `feat/ws1-c1-tolerance-contract-267` +**Commit:** `af4d9c2` (and follow-ups on the same branch) + +## Acceptance criteria map + +| AC | Status | Where | +| --- | --- | --- | +| BF16 exec, FP32 ref/accum, FP8 out, TF32 policy documented + tested | Pass | `tolerance_contract.json` `policy`; `test_dtype_policy_*`; `docs/design/ws1-numerical-contract.md` | +| Independent execution/accum/output/reference + backend provenance | Pass | `resolve_dtype_policy`, `validate_backend_provenance`; tests | +| Missing applicable four-judgment cell → hard fail; explicit N/A only when declared | Pass | `resolve_tolerance` / `resolve_tolerance_support`; schema + unit tests | +| BF16+FP32 mandatory; FP16 optional complete; FP8 hard-fail | Pass | schema validation + resolve tests | +| Gradient tolerances independent of forward | Pass | separate `gradient_accuracy` rows; `op_checks` uses `gradient_accuracy`; independence test | +| Batch/Chunk inv rows bitwise `atol=0,rtol=0` | Pass | `forward_invariance` / `gradient_invariance`; schema rejects nonzero | +| Aggregate formulas, roles, active mask, clip, empty/NaN rules + boundary tests | Pass | `compute_logprob_aggregates` / `judge_logprob_aggregates`; tests | +| Reports persist `comparison_lhs_role` / `comparison_rhs_role`; reversed roles hard-fail | Pass | `OutputCheck` fields; `assert_comparison_roles` | +| No bare `baseline`; `singleton_aggregate` not a comparison role | Pass | `comparison_roles.forbidden` + schema tests | +| Named resolve for three aggregates; all three in logprob pass/fail | Pass | `resolve_chain_aggregate_thresholds`; `require_all` | +| Docs: three aggregates sole chain logprob metrics; grads independent | Pass | numerical contract + gtest usage guide | +| New gates obtain thresholds only via shared resolver | Pass for gtest path | `op_checks` / `check_operator`; residual private-atol inventory tracked in migration checklist (C3/C4/C8) | +| CUDA + Triton same contract rows; no backend-private relaxation | Pass | shared thresholds; `backend_private_tolerance_relaxation=false` | + +## Docking paths + +- `rl_engine/kernels/gtest/tolerance_contract.json` +- `rl_engine/kernels/gtest/tolerance.py` +- `rl_engine/kernels/gtest/op_checks.py` +- `tests/test_tolerance_contract.py` +- `tests/test_op_checks.py` +- `docs/design/ws1-numerical-contract.md` +- `docs/contributing/gtest-usage.md` +- `docs/design/ws1-gtest-migration-checklist.md` + +## Local verification + +```bash +python -m pytest tests/test_tolerance_contract.py tests/test_op_checks.py -q +# 41 passed +``` + +Sample resolve (logprob BF16): + +```text +forward_accuracy: mode=tolerance atol=0.05 rtol=0.0 lhs=bf16_candidate rhs=fp32_reference +forward_invariance: mode=bitwise atol=0.0 rtol=0.0 lhs=transformed_config rhs=canonical_config +gradient_accuracy: mode=tolerance (independent keys; does not read forward) +gradient_invariance: mode=bitwise atol=0.0 rtol=0.0 +``` + +## Explicitly out of this issue (still open under #266) + +- C2 full-model workload / manifest pin of clip interval (#268) +- C3/C4 shared invariance harnesses (#269/#270) +- Migrating every historical private-atol pytest (checklist; C8 evidence) +- Full-model train/infer gate and CI (#276/#277) +- Closing parent #266 + +## Suggested issue comment when PR is green + +```text +C1 complete on . + +- Contract + resolver + schema tests (41 passed locally) +- op_checks: forward_accuracy vs gradient_accuracy +- Docs: ws1-numerical-contract.md, gtest-usage.md, migration checklist +- Residual private-atol in legacy op tests tracked for C3/C4/C8; not a C1 dock gap + +Closing #267. Parent #266 remains open (C2–C11). +``` From 087156e5486be02e6104707000fad7cdfcc808c2 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 00:55:33 +0800 Subject: [PATCH 03/10] docs(ws1): streamline C1 gtest documentation --- docs/contributing/gtest-usage.md | 12 +- docs/contributing/testing.md | 3 +- docs/design/ws1-c1-267-closeout-evidence.md | 72 ------ docs/design/ws1-gtest-migration-checklist.md | 248 ------------------- docs/design/ws1-numerical-contract.md | 187 -------------- 5 files changed, 8 insertions(+), 514 deletions(-) delete mode 100644 docs/design/ws1-c1-267-closeout-evidence.md delete mode 100644 docs/design/ws1-gtest-migration-checklist.md delete mode 100644 docs/design/ws1-numerical-contract.md diff --git a/docs/contributing/gtest-usage.md b/docs/contributing/gtest-usage.md index 036c1b65..40438c77 100644 --- a/docs/contributing/gtest-usage.md +++ b/docs/contributing/gtest-usage.md @@ -3,7 +3,6 @@ > **Audience:** contributors implementing train–inference / batch-invariant operators > **Entry point:** `scripts/check_operator.py` + `rl_engine/kernels/gtest/*` > **Numerical SSOT:** [#267](https://github.com/RL-Align/RL-Kernel/issues/267) four-judgment contract -> **Related:** [WS1 numerical contract](../design/ws1-numerical-contract.md) · [migration checklist](../design/ws1-gtest-migration-checklist.md) This is the official how-to for the gtest harness: register an op, build inputs, run the CLI for forward/backward checks, and obtain tolerances from the shared contract (not private `atol`/`rtol`). @@ -270,6 +269,7 @@ verdict = judge_logprob_aggregates(agg, contract, execution_dtype="bfloat16") | Reference / accumulation | FP32 | | FP8 | Out of scope (resolve hard-fails) | | TF32 | Disabled | +| Profiles | `cuda_bf16` and `triton_cuda_bf16` share **the same** thresholds | WS1 evidence must attach checked provenance to its candidate report: @@ -297,9 +297,13 @@ candidate = CandidateSpec( The suite rejects backend fallback, dtype drift, TF32 enablement, and observed output dtypes that disagree with this provenance before producing a passing report. -| Profiles | `cuda_bf16` and `triton_cuda_bf16` share **the same** thresholds | -**Do not** use private `atol=1e-5` (etc.) as WS1 gate evidence. Inventory of legacy private thresholds: [migration checklist](../design/ws1-gtest-migration-checklist.md). +`check_operator.py` is a local debugging CLI and does not construct provenance on its +own. A WS1 gate must create `CandidateSpec(..., provenance=provenance)` in its harness; +use `--json` to retain the resolved judgment and comparison-role fields in CLI reports. + +**Do not** use private `atol=1e-5` (etc.) as WS1 gate evidence. Migrate a gate to +the shared resolver before using it as WS1 evidence. ### 6.4 Report `tol=(atol=..., rtol=...)` @@ -375,8 +379,6 @@ New pytest code should call `resolve_tolerance` instead of copying magic numbers | Doc | Content | |-----|---------| -| [ws1-numerical-contract.md](../design/ws1-numerical-contract.md) | Four judgments, roles, aggregate formulas | -| [ws1-gtest-migration-checklist.md](../design/ws1-gtest-migration-checklist.md) | Which tests still use private thresholds and when to migrate | | [testing.md](testing.md) | Short testing entry points | | Issues [#266](https://github.com/RL-Align/RL-Kernel/issues/266) / [#267](https://github.com/RL-Align/RL-Kernel/issues/267) | WS1 closeout and C1 contract | diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index b0924ba0..7d8aff90 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -14,8 +14,7 @@ Full usage (register `OP_SPECS`, build inputs, CLI flags, and the WS1 four-judgm tolerance contract after #267): - **[gtest usage guide](gtest-usage.md)** (operator CLI + `OP_SPECS` + contract; English) -- [WS1 numerical contract](../design/ws1-numerical-contract.md) -- [gtest private-threshold migration checklist](../design/ws1-gtest-migration-checklist.md) +- **[gtest 使用指南](gtest-usage.zh-CN.md)**（算子 CLI、`OP_SPECS` 与数值合同；中文） ## Dispatch Tests diff --git a/docs/design/ws1-c1-267-closeout-evidence.md b/docs/design/ws1-c1-267-closeout-evidence.md deleted file mode 100644 index 4005b02d..00000000 --- a/docs/design/ws1-c1-267-closeout-evidence.md +++ /dev/null @@ -1,72 +0,0 @@ -# #267 (C1) closeout evidence - -**Issue:** [#267](https://github.com/RL-Align/RL-Kernel/issues/267) -**Parent:** [#266](https://github.com/RL-Align/RL-Kernel/issues/266) (C1 only; does **not** close #266) -**Branch:** `feat/ws1-c1-tolerance-contract-267` -**Commit:** `af4d9c2` (and follow-ups on the same branch) - -## Acceptance criteria map - -| AC | Status | Where | -| --- | --- | --- | -| BF16 exec, FP32 ref/accum, FP8 out, TF32 policy documented + tested | Pass | `tolerance_contract.json` `policy`; `test_dtype_policy_*`; `docs/design/ws1-numerical-contract.md` | -| Independent execution/accum/output/reference + backend provenance | Pass | `resolve_dtype_policy`, `validate_backend_provenance`; tests | -| Missing applicable four-judgment cell → hard fail; explicit N/A only when declared | Pass | `resolve_tolerance` / `resolve_tolerance_support`; schema + unit tests | -| BF16+FP32 mandatory; FP16 optional complete; FP8 hard-fail | Pass | schema validation + resolve tests | -| Gradient tolerances independent of forward | Pass | separate `gradient_accuracy` rows; `op_checks` uses `gradient_accuracy`; independence test | -| Batch/Chunk inv rows bitwise `atol=0,rtol=0` | Pass | `forward_invariance` / `gradient_invariance`; schema rejects nonzero | -| Aggregate formulas, roles, active mask, clip, empty/NaN rules + boundary tests | Pass | `compute_logprob_aggregates` / `judge_logprob_aggregates`; tests | -| Reports persist `comparison_lhs_role` / `comparison_rhs_role`; reversed roles hard-fail | Pass | `OutputCheck` fields; `assert_comparison_roles` | -| No bare `baseline`; `singleton_aggregate` not a comparison role | Pass | `comparison_roles.forbidden` + schema tests | -| Named resolve for three aggregates; all three in logprob pass/fail | Pass | `resolve_chain_aggregate_thresholds`; `require_all` | -| Docs: three aggregates sole chain logprob metrics; grads independent | Pass | numerical contract + gtest usage guide | -| New gates obtain thresholds only via shared resolver | Pass for gtest path | `op_checks` / `check_operator`; residual private-atol inventory tracked in migration checklist (C3/C4/C8) | -| CUDA + Triton same contract rows; no backend-private relaxation | Pass | shared thresholds; `backend_private_tolerance_relaxation=false` | - -## Docking paths - -- `rl_engine/kernels/gtest/tolerance_contract.json` -- `rl_engine/kernels/gtest/tolerance.py` -- `rl_engine/kernels/gtest/op_checks.py` -- `tests/test_tolerance_contract.py` -- `tests/test_op_checks.py` -- `docs/design/ws1-numerical-contract.md` -- `docs/contributing/gtest-usage.md` -- `docs/design/ws1-gtest-migration-checklist.md` - -## Local verification - -```bash -python -m pytest tests/test_tolerance_contract.py tests/test_op_checks.py -q -# 41 passed -``` - -Sample resolve (logprob BF16): - -```text -forward_accuracy: mode=tolerance atol=0.05 rtol=0.0 lhs=bf16_candidate rhs=fp32_reference -forward_invariance: mode=bitwise atol=0.0 rtol=0.0 lhs=transformed_config rhs=canonical_config -gradient_accuracy: mode=tolerance (independent keys; does not read forward) -gradient_invariance: mode=bitwise atol=0.0 rtol=0.0 -``` - -## Explicitly out of this issue (still open under #266) - -- C2 full-model workload / manifest pin of clip interval (#268) -- C3/C4 shared invariance harnesses (#269/#270) -- Migrating every historical private-atol pytest (checklist; C8 evidence) -- Full-model train/infer gate and CI (#276/#277) -- Closing parent #266 - -## Suggested issue comment when PR is green - -```text -C1 complete on . - -- Contract + resolver + schema tests (41 passed locally) -- op_checks: forward_accuracy vs gradient_accuracy -- Docs: ws1-numerical-contract.md, gtest-usage.md, migration checklist -- Residual private-atol in legacy op tests tracked for C3/C4/C8; not a C1 dock gap - -Closing #267. Parent #266 remains open (C2–C11). -``` diff --git a/docs/design/ws1-gtest-migration-checklist.md b/docs/design/ws1-gtest-migration-checklist.md deleted file mode 100644 index 3b0a9346..00000000 --- a/docs/design/ws1-gtest-migration-checklist.md +++ /dev/null @@ -1,248 +0,0 @@ -# WS1 gtest 阈值迁移清单 - -> **关联：** [#266](https://github.com/RL-Align/RL-Kernel/issues/266) 父收尾 · [#267](https://github.com/RL-Align/RL-Kernel/issues/267) C1 契约 · [数值契约说明](ws1-numerical-contract.md) -> **目的：** 盘点「哪些测试仍用私有 `atol`/`rtol`、哪些已走 SSOT、何时必须迁到 `resolve_tolerance`」。 -> **快照：** 基于 `feat/ws1-c1-tolerance-contract-267` 落地 C1 后的仓库状态；文件增减时请更新本表。 - ---- - -## 0. 迁移总原则 - -### 0.1 SSOT 入口（改后唯一推荐） - -```python -from rl_engine.kernels.gtest.tolerance import ( - load_contract, - resolve_tolerance, - compute_logprob_aggregates, - judge_logprob_aggregates, - default_clip_interval, -) - -contract = load_contract() -spec = resolve_tolerance( - contract, - judgment="forward_accuracy", # 或 forward_invariance / gradient_* - op_class="logprob", # elementwise | reduction | logprob | attention - dtype="bfloat16", - backend_profile="cuda_bf16", # 与 triton_cuda_bf16 同阈值 -) -# assert_close(..., atol=spec.atol, rtol=spec.rtol) -# 不变性：spec.mode == "bitwise" 且 atol=rtol=0 → 优先 torch.equal -``` - -| Judgment | 用于 | -|----------|------| -| `forward_accuracy` | BF16 candidate vs FP32 reference | -| `forward_invariance` | 同逻辑 workload 跨 batch/chunk/layout（**bitwise**） | -| `gradient_accuracy` | 梯度 vs FP32 参考（**不得**读 forward 行） | -| `gradient_invariance` | 梯度跨 config（**bitwise**） | -| 三聚合 API | 链级 / 训推 selected-logprob（`max_abs_dlogp` / `approx_kl0` / `clipfrac0`） | - -### 0.2 什么叫「私有阈值」（禁止作为 WS1 gate 证据） - -- 测试文件内字面量：`atol=1e-5`、`atol=5e-2`、模块常量 `_DECODE_ATOL` 等 -- 文档里写死但未从 `tolerance_contract.json` resolve 的数 -- 从 `contract["accuracy"]...` 手抄数值后本地再改（应用 resolve，不要复制常量） -- 用非零 `atol` 充当 Batch/Chunk **invariance** 通过条件 - -### 0.3 什么可以保留（不必硬迁） - -| 场景 | 处理 | -|------|------| -| **bitwise 身份断言**（`torch.equal`） | 合法；对应 invariance judgment 的 `mode=bitwise` | -| **非数值语义**（mask 形状、版本单调、manifest 字段） | 不迁 | -| **框架/集成单测**（bridge、vLLM mock、DeepSpeed worker 编排） | 非 WS1 op gate；可保留宽松 `allclose`，但**不能**当作 #266 EXIT 证据 | -| **生产 FA / SDPA 对齐**（`test_attention_correctness`） | 非 BI 候选路径；阈值可独立，**不得**写进 WS1 EXIT claim | -| **legacy `accuracy` 键** | 仅兼容；新代码禁止新增依赖，应改 `resolve_tolerance` | - -### 0.4 建议迁移句式 - -```python -# BAD — 私有阈值 -torch.testing.assert_close(out, ref, atol=1e-5, rtol=1e-5) - -# GOOD — accuracy -spec = resolve_tolerance(contract, judgment="forward_accuracy", op_class="reduction", dtype=dtype) -torch.testing.assert_close(out, ref, atol=spec.atol, rtol=spec.rtol) - -# GOOD — invariance -ispec = resolve_tolerance(contract, judgment="forward_invariance", op_class="attention", dtype=dtype) -assert ispec.mode == "bitwise" and ispec.atol == 0.0 -assert torch.equal(a, b) # 或 assert_close(..., atol=0, rtol=0) - -# GOOD — gradient accuracy（独立 judgment） -gspec = resolve_tolerance(contract, judgment="gradient_accuracy", op_class="logprob", dtype=dtype) -``` - ---- - -## 1. 状态总表（`tests/`） - -图例： - -| 标记 | 含义 | -|------|------| -| **A** | 已走 resolver / gtest suite（目标态） | -| **B** | 走 `load_contract` 旧键或 gtest 间接路径（过渡） | -| **C** | WS1 相关但 **私有 atol**（应迁） | -| **D** | 多为 `torch.equal` / 结构断言（ok 或仅需声明 judgment） | -| **E** | 非 WS1 门禁（框架/产品路径，低优先级） | - -### 1.1 已对齐或接近 SSOT - -| 文件 | 状态 | 说明 | 下一步 | -|------|------|------|--------| -| `test_tolerance_contract.py` | **A** | C1 schema + resolve + 聚合 | 保持；契约变更必跑 | -| `test_op_checks.py` | **A/B** | suite 已按 judgment 解析；部分用例注入最小 contract | 新 fixture 尽量带 `judgments` | -| `test_operator_inputs.py` | **B** | 输入/规格，无数值阈值主责 | 无需迁阈值 | -| `test_swiglu.py` | **B/D** | issue-108 harness + 大量 `torch.equal` | accuracy 路径确认走 suite；字面量 atol 清零 | -| `test_det_gemm.py` | **B** | `load_contract()["accuracy"]...` | **优先迁**：改为 `resolve_tolerance(..., forward/gradient_accuracy)` | -| `test_deterministic_attention_cuda.py` | **B/C** | 部分用 suite；仍见 `5e-2/2e-2` 字面量 | 字面量改为 resolve；invariance 保持 equal | - -### 1.2 WS1 算子测试 — 私有阈值（应迁，按优先级） - -| 优先级 | 文件 | op_class 建议 | 现状摘要 | 何时必须迁 | -|--------|------|---------------|----------|------------| -| **P0** | `test_batch_invariant_logp.py` | `logprob` | 大量 `1e-6`…`1e-2` 私有；含 bwd | 接 C3/C4/C8 证据前 | -| **P0** | `test_linear_logp.py` | `logprob` | `1e-5`…`1.5e-1` 混用；bf16 松阈值 | 同上；链级改用三聚合 API | -| **P0** | `test_logp.py` / `test_deterministic_logp.py` | `logprob` | 私有 atol | 关 #148 residual / C8 前 | -| **P0** | `test_rms_norm.py` | `reduction` | `1e-5`…`8e-2`；bwd 混用 | C8 RMSNorm 证据前 | -| **P0** | `test_triton_batch_invariant_attention.py` | `attention` | 混 `1e-5` 与 `5e-2/2e-2` | C8 Attention 证据前 | -| **P0** | `test_attention.py` | `attention` | native GT；`1e-4`/`2e-6` 等 | 与 contract `attention` 行对齐 | -| **P1** | `test_kv_cache_attention.py` | `attention` | 含 `2e-6` 等；#152 相关 | **C6/C7 前必须**消私有 decode 阈值 | -| **P1** | `test_issue151_embedding_lm_head_invariance.py` | emb + lm_head + logp | bf16 `5e-2` 手写 | C8 emb/lm_head 证据前 | -| **P1** | `test_lm_head.py` | `reduction` | 多 equal；grad `1e-5` 私有 | 迁 grad → `gradient_accuracy` | -| **P1** | `test_embedding.py` | `elementwise` | 多为 equal | 若有 tolerance 路径再 resolve | -| **P1** | `test_rope.py` | `elementwise` | `1e-3`…`2e-2` | C5 RoPE 证据前 | -| **P1** | `test_matmul.py` | `reduction` | 私有 `1e-4/1e-5` | 与 det_gemm 统一 | -| **P2** | `test_pack.py` | `elementwise` | 几乎 equal；gradcheck `1e-6` | packing 纳入 #150 时 | -| **P2** | `test_grpo_loss.py` / `test_ratio_kl.py` | （loss，契约暂无独立 class） | `1e-4` 等 | 若进 chain 则扩展 op_class 或显式 N/A | -| **P3** | `test_attention_correctness.py` | 非 BI EXIT | FA/SDPA 私有表 | **不迁入 WS1 SSOT**；文档标明 out of WS1 claim | -| **P3** | `test_op_accuracy.py` | 杂项 harness | `1e-3` | 废弃或改走 `check_operator` + contract | - -### 1.3 非 WS1 门禁（低优先级 / 不阻塞 #267） - -| 文件 | 状态 | 说明 | -|------|------|------| -| `test_deepspeed_training_worker.py` | **E** | 训练 worker；`atol=1e-5` 编排级 | -| `test_stateless_training_contract.py` | **E** | 契约字段/数值 smoke | -| `test_rl_kernel_loss_step.py` | **E** | 端到端 loss 步 | -| `test_sampler_temperature.py` | **E** | 采样 | -| `test_weight_sync_bridge.py` 等 | **D/E** | bridge / IPC | -| `test_vllm_rollout_sampler.py` | **D/E** | vLLM mock | -| `test_alignment_model_wrappers.py` | **D/E** | wrapper 行为 | -| `test_rl_batch_fixture.py` | **D** | fixture 身份 | -| `test_stateless_executor.py` / `*_hf_integration*` | **D/E** | 执行器集成 | - -这些**不**作为 #266 Full WS1 EXIT 的数值证据来源；C10/C11 不得引用其私有阈值刷绿。 - ---- - -## 2. 按 #266 子 issue 的「何时必须迁」 - -| 子 issue | 阻塞迁移范围 | 完成信号 | -|----------|--------------|----------| -| **C1 #267** | 契约 + resolver + schema/报告测试 | **实现完成；待 CI / issue evidence** | -| **C3 #269** forward harness | 所有 **forward_accuracy / forward_invariance** 的 op 单测证据路径 | 无 private forward atol 作为 gate | -| **C4 #270** grad harness | 所有 **gradient_*** 证据路径 | 无「grad 抄 forward 字面量」 | -| **C5 #271** RoPE/elementwise | `test_rope.py`、activation/swiglu residual | audit 报告阈值均来自 resolve | -| **C6/C7 #272/#273** KV | `test_kv_cache_attention.py` 及后续 kv harness | **禁止** `_DECODE_ATOL` 类私有常量 | -| **C8 #274** closed-op 矩阵 | rmsnorm / gemm / attn / logp / emb / lm_head 测试 | 每格 `requested/actual backend` + resolve 阈值 | -| **C10 #276** 全模型 gate | 仅用 resolver + 三聚合；禁止任何测试内字面量阈值 | gate 报告无 private tol 字段 | -| **C11 #277** CI | CI 只跑 resolve 路径 | fail-closed | - -**规则：** 某 op 的 PR 若声称「满足 #266/C8」，则该 PR 触达的 assert **必须**来自 `resolve_tolerance` / 聚合 API，而不是文件顶部的魔法数。 - ---- - -## 3. 文件级迁移清单（可勾选） - -### 3.1 P0 — 直接挡 C3/C4/C8 - -- [ ] `tests/test_batch_invariant_logp.py` — fwd/bwd accuracy + invariance 拆 judgment -- [ ] `tests/test_linear_logp.py` — 同上；训推/链级改用三聚合 -- [ ] `tests/test_logp.py` -- [ ] `tests/test_deterministic_logp.py` -- [ ] `tests/test_rms_norm.py` -- [ ] `tests/test_triton_batch_invariant_attention.py` -- [ ] `tests/test_attention.py` -- [ ] `tests/test_det_gemm.py` — 去掉 `contract["accuracy"]` 直读 - -### 3.2 P1 — C5/C6/C7/C8 residual - -- [ ] `tests/test_kv_cache_attention.py` -- [ ] `tests/test_issue151_embedding_lm_head_invariance.py` -- [ ] `tests/test_lm_head.py` -- [ ] `tests/test_embedding.py`（若有 non-bitwise 路径） -- [ ] `tests/test_rope.py` -- [ ] `tests/test_matmul.py` -- [ ] `tests/test_deterministic_attention_cuda.py` 中剩余字面量 -- [ ] `tests/test_swiglu.py` 中任何 residual 字面量 - -### 3.3 P2 — 进 chain 时 - -- [ ] `tests/test_pack.py` -- [ ] `tests/test_grpo_loss.py` / `tests/test_ratio_kl.py`（先扩 contract op_class 或标 N/A） -- [ ] `scripts/check_operator.py` 报告字段确认只回传 resolve 结果（已间接） - -### 3.4 明确不迁入 WS1 SSOT - -- [x] `tests/test_attention_correctness.py` — 生产 FA；文档标注非 EXIT -- [x] bridge / vLLM / DeepSpeed / sampler 类 **E** 组 - ---- - -## 4. 推荐落地动作（每个测试文件） - -1. **分类每条 assert** - - identity / batch-invariance → `forward_invariance` 或 `gradient_invariance` + `torch.equal` - - vs fp32 gold → `*_accuracy` - - train vs infer logp → 三聚合，不用单点 atol 冒充 -2. **删除模块级 `_ATOL` / `_RTOL`** -3. **dtype 参数化** 时用 `resolve_tolerance(..., dtype=dtype)`，禁止 bf16 写死 `5e-2` -4. **报告**（若有）写上 `comparison_lhs_role` / `comparison_rhs_role`（从 spec 取） -5. **禁止** 为让 invariance 通过而调大 atol - -### 4.1 与契约行不一致时怎么办 - -| 情况 | 动作 | -|------|------| -| 测试私有更松，契约更紧 → 测试红 | **修 kernel** 或开 Blocker；**禁止**在测试放宽 | -| 测试私有更紧，契约更松 | 迁到契约后可能变绿；可保留额外严格 assert 但须标注 *non-gate* | -| 需要新 op_class（如 `grpo_loss`） | 先改 `tolerance_contract.json` + schema 测试，再迁测试 | -| decode vs prefill 无法 bitwise | 用 contract 已声明的 semantic 行 / 三聚合；**不要**私设 `_DECODE_ATOL` | - ---- - -## 5. 工具与 CI 建议（后续，非 C1 范围） - -| 建议 | 作用 | -|------|------| -| 简单 lint：`tests/**/*.py` 禁止 `atol=\d`（allowlist 契约测试与 FA 测试） | 防回流 | -| `pytest` marker：`ws1_gate` 仅收集 resolve 路径 | C11 门禁清晰 | -| 在 `check_operator.py` 输出中强制打印 `judgment` + roles | 证据可检索 | - -C1 **不**强制上 lint；C8/C10 前建议至少做 allowlist 扫描。 - ---- - -## 6. 现状一句话 - -| 层 | 状态 | -|----|------| -| **契约 + resolver（gtest 核心）** | 已就绪；待 CI / issue evidence（#267） | -| **op_checks 接入** | 已按 judgment 分叉，并持久化 roles / provenance | -| **存量 op 单测** | **多数仍私有 atol**（上表 P0/P1） | -| **#266 EXIT** | 依赖后续把 P0/P1 迁完，而不是只合 C1 | - -**C1 的价值是「唯一入口已存在」；清单的价值是「知道还欠哪些文件」。** -未完成 P0/P1 迁移前，**不得**声称「全仓测试已统一走 WS1 数值契约」。 - ---- - -## 7. 修订记录 - -| 日期 | 说明 | -|------|------| -| 2026-08-11 | 初版：C1 落地后基于 `tests/` 扫描的迁移清单与优先级 | diff --git a/docs/design/ws1-numerical-contract.md b/docs/design/ws1-numerical-contract.md deleted file mode 100644 index 76410fcb..00000000 --- a/docs/design/ws1-numerical-contract.md +++ /dev/null @@ -1,187 +0,0 @@ -# WS1 Numerical Contract (C1 / #267) - -> **Parent:** [#266](https://github.com/RL-Align/RL-Kernel/issues/266) -> **Issue:** [#267](https://github.com/RL-Align/RL-Kernel/issues/267) -> **SSOT files:** `rl_engine/kernels/gtest/tolerance_contract.json`, `rl_engine/kernels/gtest/tolerance.py` - -This document freezes the **sole** numerical judgment source for WS1 ablations and -gates. New gates must obtain thresholds only through the shared resolver APIs; private -`atol` / `rtol` constants are forbidden. - -## Scope boundary - -**Allowed WS1 claim after full exit (#266):** single-GPU model-level train–inference -consistency for full Qwen3-8B Dense under required CUDA BF16 and Triton-on-CUDA BF16 -profiles (in-repo BI stack). - -**Not claimed here:** multi-GPU (WS2), real vLLM vs Megatron / vime product alignment -(WS3), or kernel bug fixes (open Blockers). - -## Dtype policy - -| Field | Lock | -| --- | --- | -| `execution_dtype` | **BF16** (mandatory) | -| `accumulation_dtype` | **FP32** | -| `reference_dtype` | **FP32** | -| `output_dtype.default` | follows execution | -| logprob aggregates compute dtype | **FP32** | -| FP8 | **out of scope** (request → hard fail) | -| FP16 | optional; rows complete when declared | -| TF32 (reference + candidate) | **disabled** (repo-wide single policy) | -| Backend profiles | `cuda_bf16`, `triton_cuda_bf16` (same thresholds) | -| Backend-private tolerance relaxation | **forbidden** | - -Execution, accumulation, output, and reference dtypes resolve **independently** via -`resolve_dtype_policy()`. - -## Four judgments - -| Judgment | What it compares | Default mode | -| --- | --- | --- | -| `forward_accuracy` | BF16 candidate vs FP32 reference outputs | tolerance | -| `forward_invariance` | transformed vs canonical config, same backend/dtype/logical workload | **bitwise** (`atol=0`, `rtol=0`) | -| `gradient_accuracy` | candidate gradient/VJP vs FP32 reference gradient/VJP | tolerance (independent of forward) | -| `gradient_invariance` | transformed vs canonical gradients, same logical workload | **bitwise** (`atol=0`, `rtol=0`) | - -Every declared-applicable `(judgment, op_class, dtype)` tuple must resolve. Missing -applicable cells hard-fail. Explicit `not_applicable` / `out_of_scope` is allowed only -when present in the schema. Use `resolve_tolerance_support()` to persist the explicit -support status; requesting thresholds for an N/A or out-of-scope cell still hard-fails. - -**Batch/Chunk invariance** (issue #150 / C10 matrix) **must** use the invariance -judgments in bitwise mode. Nonzero tolerance cannot satisfy that gate. - -Op classes: `elementwise`, `reduction`, `logprob`, `attention`. - -## Comparison roles - -Reports must record `comparison_lhs_role` / `comparison_rhs_role`. A bare `baseline` -field is forbidden. C2 `singleton_aggregate` is an **execution/aggregation mode**, not -a comparison role. - -| Report kind | `comparison_lhs_role` | `comparison_rhs_role` | -| --- | --- | --- | -| `forward_accuracy` | `bf16_candidate` | `fp32_reference` | -| `forward_invariance` | `transformed_config` | `canonical_config` | -| `train_infer_logprob_parity` | `training_style_teacher_forcing` | `inference_style_rollout_decode` | -| `gradient_accuracy` | `bf16_candidate` | `fp32_reference` | -| `gradient_invariance` | `transformed_config` | `canonical_config` | - -Direction is locked so train/infer preserves: - -```text -dlogp = train_logp - rollout_logp -ratio0 = exp(dlogp) -``` - -Swapping lhs/rhs without a different declared contract row hard-fails. - -API: `resolve_comparison_roles()`, `assert_comparison_roles()`. - -Aggregate callers must also provide `contract`, `report_kind`, and both roles; the -compute API validates the direction before calculating any metric. gtest reports -persist these roles on every output verdict. Backend reports must include -`BackendProvenance` (requested/actual backend, all four dtypes, and TF32 state), -which `validate_backend_provenance()` checks against the selected profile. - -## Chain-level logprob aggregates - -These three metrics are the **only** chain-level logprob / ablation aggregates for WS1 -pass/fail. Gradients use independent `gradient_*` tensor verdicts and **do not** use -these aggregates. - -Computed in FP32 on **active selected tokens only**: - -```text -dlogp = comparison_lhs_logp - comparison_rhs_logp -max_abs_dlogp = max(abs(dlogp)) -approx_kl0 = mean(exp(dlogp) - 1 - dlogp) -clipfrac0 = mean(1[exp(dlogp) outside clip_interval]) -``` - -Rules: - -- **All three** must pass (`require_all=true`). -- Empty active-token set → hard fail. -- NaN / Inf in `dlogp`, `ratio0`, or aggregates → hard fail. -- Clip interval is pinned by the workload manifest (C2); the contract stores a default - and the field name `clip_interval`. - -API: `compute_logprob_aggregates()`, `judge_logprob_aggregates()`, -`resolve_chain_aggregate_thresholds()`. - -## Resolver usage - -```python -from rl_engine.kernels.gtest.tolerance import ( - load_contract, - resolve_tolerance, - resolve_dtype_policy, - compute_logprob_aggregates, - judge_logprob_aggregates, - default_clip_interval, -) - -contract = load_contract() -policy = resolve_dtype_policy(contract) - -fwd = resolve_tolerance( - contract, - judgment="forward_accuracy", - op_class="logprob", - dtype="bfloat16", - backend_profile="cuda_bf16", -) -bwd = resolve_tolerance( - contract, - judgment="gradient_accuracy", - op_class="logprob", - dtype="bfloat16", - backend_profile="triton_cuda_bf16", # same thresholds as cuda_bf16 -) -inv = resolve_tolerance( - contract, - judgment="forward_invariance", - op_class="attention", - dtype="bfloat16", -) -# inv.mode == "bitwise", inv.atol == 0.0, inv.rtol == 0.0 -``` - -`op_checks.run_operator_suite` resolves **forward_accuracy** for outputs and -**gradient_accuracy** for gradients. - -## Compatibility keys - -For older tests that still dig into: - -- `contract["accuracy"]["default"][op_class][dtype]` — mirror of `forward_accuracy` -- `contract["batch_invariance"]` — `{atol: 0, rtol: 0}` - -New code should call the resolvers above. Schema validation fails if the compatibility -mirror drifts from `forward_accuracy` or if invariance rows leave bitwise mode. - -## Related issues - -| ID | Role | -| --- | --- | -| #266 | WS1 closeout parent | -| #267 | This contract (C1) | -| #268 | Full-model workload / clip interval pin in manifest | -| #269 / #270 | Forward / gradient invariance harnesses | -| #276 | Full-model train/infer gate consuming this contract | -| #154 / #108 | Historical contract owners (superseded remaining work → C1) | - -## Migration of existing tests - -Most operator tests still use **private** `atol` / `rtol` literals. That is expected -after C1: the SSOT exists, but call sites have not all moved. - -See the full inventory, priority, and “when it must migrate” map: - -- [WS1 gtest 阈值迁移清单](ws1-gtest-migration-checklist.md) - -How to register ops and run the CLI (post-#267): - -- [gtest usage guide](../contributing/gtest-usage.md) From 81ddd652aa9d16d4c9b52925fed7ace36f3a4606 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 01:43:15 +0800 Subject: [PATCH 04/10] fix(ws1): address tolerance contract review --- docs/contributing/testing.md | 1 - rl_engine/kernels/gtest/__init__.py | 8 ++ rl_engine/kernels/gtest/op_checks.py | 2 +- rl_engine/kernels/gtest/tolerance.py | 36 ++++++-- tests/test_op_checks.py | 2 +- tests/test_tolerance_contract.py | 123 ++++++++++++++++++++++----- 6 files changed, 137 insertions(+), 35 deletions(-) diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 7d8aff90..749b203e 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -14,7 +14,6 @@ Full usage (register `OP_SPECS`, build inputs, CLI flags, and the WS1 four-judgm tolerance contract after #267): - **[gtest usage guide](gtest-usage.md)** (operator CLI + `OP_SPECS` + contract; English) -- **[gtest 使用指南](gtest-usage.zh-CN.md)**（算子 CLI、`OP_SPECS` 与数值合同；中文） ## Dispatch Tests diff --git a/rl_engine/kernels/gtest/__init__.py b/rl_engine/kernels/gtest/__init__.py index 61b43218..a12db99e 100644 --- a/rl_engine/kernels/gtest/__init__.py +++ b/rl_engine/kernels/gtest/__init__.py @@ -4,8 +4,12 @@ from .op_checks import CandidateSpec, OperatorCase, run_operator_suite from .tolerance import ( BackendProvenance, + ContractError, ContractResolveError, ContractSchemaError, + load_contract, + resolve_dtype_policy, + resolve_tolerance, resolve_tolerance_support, validate_backend_provenance, ) @@ -15,8 +19,12 @@ "OperatorCase", "run_operator_suite", "BackendProvenance", + "ContractError", "ContractResolveError", "ContractSchemaError", + "load_contract", + "resolve_tolerance", + "resolve_dtype_policy", "resolve_tolerance_support", "validate_backend_provenance", ] diff --git a/rl_engine/kernels/gtest/op_checks.py b/rl_engine/kernels/gtest/op_checks.py index 085b5f89..1bada7b3 100644 --- a/rl_engine/kernels/gtest/op_checks.py +++ b/rl_engine/kernels/gtest/op_checks.py @@ -155,7 +155,7 @@ def _run_candidate( validate_backend_provenance(contract, candidate.provenance) if candidate.backend != candidate.provenance.actual_backend: raise ContractResolveError( - f"candidate backend {candidate.backend!r} disagrees with reported actual backend " + f"candidate backend {candidate.backend!r} disagrees with reported actual_backend " f"{candidate.provenance.actual_backend!r}" ) for case in cases: diff --git a/rl_engine/kernels/gtest/tolerance.py b/rl_engine/kernels/gtest/tolerance.py index 230dd95d..0d2ae2e7 100644 --- a/rl_engine/kernels/gtest/tolerance.py +++ b/rl_engine/kernels/gtest/tolerance.py @@ -381,7 +381,7 @@ def resolve_tolerance( "backend_private_tolerance_relaxation must remain false under WS1 C1" ) - if dtype_name in OUT_OF_SCOPE_DTYPES or policy.fp8 == "out_of_scope" and dtype_name == "float8": + if dtype_name in OUT_OF_SCOPE_DTYPES: raise ContractResolveError( f"dtype {dtype_name!r} is out of scope for WS1 (FP8 requests hard-fail)" ) @@ -781,12 +781,10 @@ def _validate_judgments(judgments: Mapping[str, Any]) -> None: "applicable", "not_applicable", }: - # BF16/FP32 must be explicitly applicable (or explicit N/A). - if status != "applicable": - raise ContractSchemaError( - f"mandatory dtype cell must be applicable: " - f"{judgment}/{op_class}/{dtype_name} status={status!r}" - ) + raise ContractSchemaError( + f"mandatory dtype cell must be applicable: " + f"{judgment}/{op_class}/{dtype_name} status={status!r}" + ) if status in {"applicable", "optional"}: for thr in ("atol", "rtol", "mode"): if thr not in cell: @@ -810,6 +808,8 @@ def _validate_chain_aggregates(root: Mapping[str, Any]) -> None: "require_all", "nan_inf_policy", "empty_active_token_set", + "active_token_policy", + "clip_interval_field", "dlogp_definition", "default_clip_interval", "sole_chain_level_logprob_metrics", @@ -823,6 +823,12 @@ def _validate_chain_aggregates(root: Mapping[str, Any]) -> None: raise ContractSchemaError("nan_inf_policy must be hard_fail") if root["empty_active_token_set"] != "hard_fail": raise ContractSchemaError("empty_active_token_set must be hard_fail") + if root["active_token_policy"] != "active selected tokens only": + raise ContractSchemaError("active_token_policy must be 'active selected tokens only'") + if root["clip_interval_field"] != "clip_interval": + raise ContractSchemaError("clip_interval_field must be 'clip_interval'") + if root["dlogp_definition"] != "comparison_lhs_logp - comparison_rhs_logp": + raise ContractSchemaError("dlogp_definition does not match implementation") if not root["require_all"]: raise ContractSchemaError("require_all must be true for chain logprob aggregates") sole = list(root["sole_chain_level_logprob_metrics"]) @@ -843,6 +849,15 @@ def _validate_chain_aggregates(root: Mapping[str, Any]) -> None: for dtype_name in ("bfloat16", "float32"): if dtype_name not in by_dtype or "threshold" not in by_dtype[dtype_name]: raise ContractSchemaError(f"metric {name!r} missing threshold for {dtype_name}") + expected_formula = { + "max_abs_dlogp": "max(abs(dlogp))", + "approx_kl0": "mean(exp(dlogp) - 1 - dlogp)", + "clipfrac0": "mean(1[exp(dlogp) outside clip_interval])", + }[name] + if metrics[name].get("formula") != expected_formula: + raise ContractSchemaError(f"metric {name!r} formula does not match implementation") + if metrics[name].get("pass_rule") != "value <= threshold": + raise ContractSchemaError(f"metric {name!r} pass_rule must be 'value <= threshold'") def _validate_compat_views(contract: Mapping[str, Any]) -> None: @@ -883,6 +898,7 @@ def _lookup_cell( dtype_name: str, arch_key: str | None, ) -> Mapping[str, Any] | None: + base = judgment_root.get("by_op_class", {}).get(op_class, {}).get(dtype_name) if arch_key is not None: arch_cell = ( judgment_root.get("arch_overrides", {}) @@ -891,8 +907,10 @@ def _lookup_cell( .get(dtype_name) ) if arch_cell is not None: - return arch_cell - return judgment_root.get("by_op_class", {}).get(op_class, {}).get(dtype_name) + if base is None: + return arch_cell + return {**base, **arch_cell} + return base def _dtype_name(dtype: str | Any) -> str: diff --git a/tests/test_op_checks.py b/tests/test_op_checks.py index 35de05a5..de2ceb22 100644 --- a/tests/test_op_checks.py +++ b/tests/test_op_checks.py @@ -225,7 +225,7 @@ def test_ws1_report_rejects_backend_provenance_mismatch(): provenance = BackendProvenance( backend_profile="cuda_bf16", requested_backend="cuda", - actual_backend="triton", + actual_backend="cuda", execution_dtype="bfloat16", accumulation_dtype="float32", output_dtype="bfloat16", diff --git a/tests/test_tolerance_contract.py b/tests/test_tolerance_contract.py index 5f711061..4fcbe23f 100644 --- a/tests/test_tolerance_contract.py +++ b/tests/test_tolerance_contract.py @@ -113,22 +113,28 @@ def test_invariance_rows_are_bitwise_zero(): def test_cuda_and_triton_profiles_share_thresholds(): contract = load_contract() - for profile in ("cuda_bf16", "triton_cuda_bf16"): - a = resolve_tolerance( - contract, - judgment="forward_accuracy", - op_class="logprob", - dtype="bfloat16", - backend_profile=profile, - ) - b = resolve_tolerance( - contract, - judgment="forward_accuracy", - op_class="logprob", - dtype="bfloat16", - backend_profile="cuda_bf16" if profile != "cuda_bf16" else "triton_cuda_bf16", - ) - assert a.atol == b.atol and a.rtol == b.rtol and a.mode == b.mode + for judgment in JUDGMENTS: + for op_class in OP_CLASSES: + for dtype_name in ("float32", "bfloat16"): + cuda = resolve_tolerance( + contract, + judgment=judgment, + op_class=op_class, + dtype=dtype_name, + backend_profile="cuda_bf16", + ) + triton = resolve_tolerance( + contract, + judgment=judgment, + op_class=op_class, + dtype=dtype_name, + backend_profile="triton_cuda_bf16", + ) + assert (cuda.atol, cuda.rtol, cuda.mode) == ( + triton.atol, + triton.rtol, + triton.mode, + ) def test_unknown_backend_profile_hard_fails(): @@ -313,9 +319,15 @@ def test_resolve_tolerance_attaches_roles(): def test_chain_aggregate_named_resolve(): contract = load_contract() + expected = { + "max_abs_dlogp": {"bfloat16": 5.0e-2, "float32": 1.0e-5}, + "approx_kl0": {"bfloat16": 5.0e-2, "float32": 1.0e-5}, + "clipfrac0": {"bfloat16": 0.0, "float32": 0.0}, + } + assert set(expected) == set(CHAIN_AGGREGATE_METRICS) for metric in CHAIN_AGGREGATE_METRICS: - thr = resolve_chain_aggregate_thresholds(contract, metric, "bfloat16") - assert thr >= 0.0 + for dtype, value in expected[metric].items(): + assert resolve_chain_aggregate_thresholds(contract, metric, dtype) == value with pytest.raises(ContractResolveError, match="unknown chain aggregate"): resolve_chain_aggregate_thresholds(contract, "mean_abs_dlogp", "bfloat16") @@ -398,6 +410,21 @@ def test_nan_inf_hard_fail(): comparison_rhs_role="inference_style_rollout_decode", ) + # Finite dlogp can still overflow exp(dlogp), which is a separate hard-fail. + lhs = torch.tensor([200.0, 0.0], dtype=torch.float32) + rhs = torch.zeros(2) + with pytest.raises(ContractResolveError, match="ratio0"): + compute_logprob_aggregates( + lhs, + rhs, + torch.ones(2, dtype=torch.bool), + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + lhs = torch.tensor([float("inf"), 0.0]) with pytest.raises(ContractResolveError, match="NaN/Inf"): compute_logprob_aggregates( @@ -412,6 +439,50 @@ def test_nan_inf_hard_fail(): ) +def test_inactive_nan_is_ignored(): + agg = compute_logprob_aggregates( + torch.tensor([0.0, float("nan")]), + torch.zeros(2), + torch.tensor([True, False]), + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + assert agg.active_token_count == 1 + assert agg.max_abs_dlogp == 0.0 + + +def test_clipfrac0_counts_ratios_outside_the_interval(): + agg = compute_logprob_aggregates( + torch.tensor([0.0, 1.0, -1.0]), + torch.zeros(3), + torch.ones(3, dtype=torch.bool), + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(0.8, 1.2), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + assert math.isclose(agg.clipfrac0, 2.0 / 3.0, rel_tol=0.0, abs_tol=1e-6) + + +def test_clip_interval_endpoints_count_as_inside(): + lo, hi = 0.5, 2.0 + agg = compute_logprob_aggregates( + torch.tensor([math.log(lo), math.log(hi)]), + torch.zeros(2), + torch.ones(2, dtype=torch.bool), + contract=load_contract(), + report_kind="train_infer_logprob_parity", + clip_interval=(lo, hi), + comparison_lhs_role="training_style_teacher_forcing", + comparison_rhs_role="inference_style_rollout_decode", + ) + assert agg.clipfrac0 == 0.0 + + def test_judge_requires_all_three_aggregates(): contract = load_contract() clip = default_clip_interval(contract) @@ -434,10 +505,11 @@ def test_judge_requires_all_three_aggregates(): assert {m.metric for m in verdict.metrics} == set(CHAIN_AGGREGATE_METRICS) assert all(m.passed for m in verdict.metrics) - # Large drift fails max_abs_dlogp / approx_kl0 / possibly clipfrac. - lhs = torch.tensor([0.0, 1.0]) - rhs = torch.zeros(2) - mask = torch.ones(2, dtype=torch.bool) + # A small in-interval drift fails only max_abs_dlogp. This proves the + # overall verdict requires all three metrics, rather than any one metric. + lhs = torch.tensor([0.1]) + rhs = torch.zeros(1) + mask = torch.ones(1, dtype=torch.bool) agg = compute_logprob_aggregates( lhs, rhs, @@ -450,7 +522,12 @@ def test_judge_requires_all_three_aggregates(): ) verdict = judge_logprob_aggregates(agg, contract, execution_dtype="bfloat16") assert not verdict.passed - assert any(not m.passed for m in verdict.metrics) + by_metric = {metric.metric: metric.passed for metric in verdict.metrics} + assert by_metric == { + "max_abs_dlogp": False, + "approx_kl0": True, + "clipfrac0": True, + } def test_compat_accuracy_mirrors_forward_accuracy(): From e857084ba0d68729ff383e07b28d958e9e2e887d Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 13:44:16 +0800 Subject: [PATCH 05/10] feat(ws1): land C2 canonical workload identity (#268) Freeze the full Qwen3-8B Dense logical workload SSOT for WS1 closeout C2: manifest pins (config fingerprint, weight content hash, 2x2 Batch/Chunk matrix, varlen fixtures, packing, dual backend profiles, representative case_ids), logical identity restore after pad/pack/chunk, singleton_aggregate vs BN multiset plan, registry-resolved candidate binding, and a single reference command. Document registry-vs-runtime actual boundary and Triton missing_required reds without silent fallback. Closes #268 --- docs/design/ws1-c2-268-closeout-evidence.md | 60 ++ docs/design/ws1-c2-268-workload-plan.md | 265 +++++ rl_engine/testing/__init__.py | 24 + rl_engine/testing/ws1_manifest.json | 1032 ++++++++++++++++++ rl_engine/testing/ws1_workload.py | 1078 +++++++++++++++++++ scripts/ws1_reference.py | 140 +++ tests/test_ws1_workload.py | 494 +++++++++ 7 files changed, 3093 insertions(+) create mode 100644 docs/design/ws1-c2-268-closeout-evidence.md create mode 100644 docs/design/ws1-c2-268-workload-plan.md create mode 100644 rl_engine/testing/ws1_manifest.json create mode 100644 rl_engine/testing/ws1_workload.py create mode 100755 scripts/ws1_reference.py create mode 100644 tests/test_ws1_workload.py diff --git a/docs/design/ws1-c2-268-closeout-evidence.md b/docs/design/ws1-c2-268-closeout-evidence.md new file mode 100644 index 00000000..72bcd5b4 --- /dev/null +++ b/docs/design/ws1-c2-268-closeout-evidence.md @@ -0,0 +1,60 @@ +# WS1 C2 (#268) Closeout Evidence + +**Issue:** #268 · **Parent:** #266 · **Workload:** `ws1-qwen3-8b-dense-primary-v3` +**Branch:** `feat/ws1-c2-canonical-workload-268` + +## Deliverables + +| Path | Role | +| --- | --- | +| `rl_engine/testing/ws1_manifest.json` | SSOT workload identity / matrix / profiles / cases | +| `rl_engine/testing/ws1_workload.py` | Load, validate, logical identity, pad/pack/chunk restore | +| `scripts/ws1_reference.py` | One-command reference emission | +| `tests/test_ws1_workload.py` | CPU acceptance tests | +| `docs/design/ws1-c2-268-workload-plan.md` | Landing plan | +| `docs/design/ws1-c2-268-closeout-evidence.md` | This map | + +## Acceptance criteria map + +| #268 AC | Status | Evidence | +| --- | --- | --- | +| Manifest pins numerics-affecting fields | **Pass** | model, seed, tokens, prompt/completion lenses, masks, positions, dtypes, clip, aggregates, RNG, TF32 ref | +| Full Qwen3-8B Dense identity + weight hash | **Pass** | config fingerprint + shard SHA-256 `content_hash` | +| Same workload ID → same fixture/reference identity | **Pass** | `fixture_identity_sha256` + `fixture_hash` tests | +| pad/pack/chunk restore logical identity | **Pass** | `apply_padding` / `apply_packing` / `apply_chunking` + restore tests | +| B1 singleton_aggregate vs BN same multiset | **Pass** | `singleton_aggregate_plan` test | +| Naming: singleton_aggregate ≠ C1 roles; no bare baseline | **Pass** | `forbidden_comparison_roles` + `report_naming` | +| 2×2 + perm + multi-chunk non-divisible + pad/varlen | **Pass** | primary matrix + varlen samples `[11,16,13,19]` | +| clip_interval for clipfrac0 | **Pass** | `[0.8, 1.2]` aligned with C1 | +| Dropout/sampling/RNG policy; undeclared hard-fail | **Pass** | `stochastic_policy` + helper test | +| Short + representative fixtures hit declared candidates | **Pass\*** | fixture `candidate_case_ids` + registry resolution tests | +| Stable case_id for C8/C10/C11 reference | **Pass** | `representative_cases[].case_id` | +| expected + actual backend/kernel + algorithm property | **Pass\*** | registry-resolved actual; runtime observation owned by C8+ (declared in manifest) | +| One command emits reference (workload ID, seed, dtype) | **Pass** | `scripts/ws1_reference.py` | +| Packing / QK-Norm / required ops status | **Pass** | packing supported + packed fixture; qk_norm required | +| Both profiles enumerate required nodes; no untracked missing | **Pass** | Triton gaps are `missing_required` (red, tracked) | + +\*C2 binds **registry-resolved** candidate paths. Live GPU dispatch observation is explicitly out of C2 (`provenance_boundary`) and owned by C3/C8/C10/C11. + +## Verification commands + +```bash +# From repo root with PYTHONPATH=repo root (or editable install) +python -m pytest tests/test_ws1_workload.py -q +python scripts/ws1_reference.py --dtype bf16 --cell-id BN/full --emit-json - +``` + +Expected: all C2 tests green; CLI prints `workload_id`, `seed`, `dtype`, `fixture_hash`, and `reference_outputs` digests. + +## Residual (explicitly not #268) + +| Item | Owner | +| --- | --- | +| Runtime observed actual backend on GPU | C3 / C8 / C10 / C11 | +| Triton `missing_required`: embedding, lm_head, logprob | later candidate work / Blocker; tracked red in C2 | +| #150 numerical asserts / full-model e2e | C9 / C10 | +| Full WS1 EXIT | #266 after C1–C11 | + +## Close recommendation + +Close **#268** once this branch is merged. Do **not** claim #266 WS1 EXIT from C2 alone. diff --git a/docs/design/ws1-c2-268-workload-plan.md b/docs/design/ws1-c2-268-workload-plan.md new file mode 100644 index 00000000..e05a6f4a --- /dev/null +++ b/docs/design/ws1-c2-268-workload-plan.md @@ -0,0 +1,265 @@ +# WS1 C2 (#268) Landing Plan — Canonical Workload & Logical Identity + +**Parent:** #266 · **Issue:** #268 · **Depends on:** C1 (#267) contract roles only +**Branch:** `feat/ws1-c2-canonical-workload-268` (from `feat/ws1-c1-tolerance-contract-267@81ddd65`) +**Does not modify:** C1 branch tip + +--- + +## 1. Goal (one sentence) + +Freeze a **reproducible full-Qwen3-8B-Dense logical workload** (identity + fixtures + 2×2 Batch/Chunk matrix + backend profile map + representative `case_id`s) so later C3–C11 gates compare the **same** sample/token multiset after pad/pack/chunk transforms. + +C2 does **not** implement #150 numerical asserts, full model forward, or multi-GPU. + +--- + +## 2. Context from parent tree + +| Item | Lock from #266 / #268 | +| --- | --- | +| Model | Full official **Qwen3-8B Dense** (no layer/hidden/head/vocab shrink) | +| Architecture pin | Config fingerprint + weight snapshot identity | +| Fixture scaling allowed | Seq length / padding / batch layout only | +| Primary matrix | `B1-singleton_aggregate/full`, `BN/full`, `B1-singleton_aggregate/chunked`, `BN/chunked` | +| Logical identity | `(sample_id, token_position)` recoverable after pad/pack/chunk | +| Gradient B1 vs BN | `singleton_aggregate` = N× B=1 of **same** N samples, fixed order + active-token denom | +| Naming | `singleton_aggregate` is **execution mode only** — never a C1 `comparison_*_role` | +| Backends | `cuda_bf16` + `triton_cuda_bf16`; every required chain node has expected candidate/path | +| Clip | `clip_interval` for `clipfrac0` co-located with aggregate pins (align C1 default `[0.8, 1.2]`) | +| Stochastic | Gate uses `dropout=0`; sampling out of logprob parity; undeclared RNG hard-fails | + +### Official Qwen3-8B Dense fingerprint (source: HF `Qwen/Qwen3-8B` config) + +| Field | Value | +| --- | --- | +| `model_id` | `Qwen/Qwen3-8B` | +| `num_hidden_layers` | 36 | +| `hidden_size` | 4096 | +| `intermediate_size` | 12288 | +| `num_attention_heads` | 32 | +| `num_key_value_heads` | 8 (GQA) | +| `head_dim` | 128 | +| `vocab_size` | 151936 | +| `rope_theta` | 1e6 | +| `rms_norm_eps` | 1e-6 | +| `hidden_act` | silu (SwiGLU MLP) | +| `tie_word_embeddings` | **false** | +| `attention_dropout` | 0.0 | +| QK-Norm | **enabled** in Qwen3 architecture (per-head q/k RMSNorm; not a config flag) | +| Config revision (pinned) | HF `x-repo-commit` at plan time: `b968826d9c46dd6066d109eabc6255188de91218` | + +Weight snapshot: pin revision + SHA-256 of `model.safetensors.index.json` + all five +official LFS shard content SHA-256/size records. The manifest also stores a reproducible +`sha256-of-sorted-shard-records-v1` aggregate, tensor payload bytes, and physical shard +bytes. This is a full content-addressed weight identity without downloading 16 GB locally. + +--- + +## 3. Deliverables (issue docking) + +| Path | Role | +| --- | --- | +| `rl_engine/testing/ws1_manifest.json` | SSOT: model identity, matrix, fixtures, backends, cases, clip/RNG policy | +| `rl_engine/testing/ws1_workload.py` | Load/validate manifest; build logical samples; pad/pack/chunk; restore identity; fixture hash | +| `scripts/ws1_reference.py` | One command: emit workload ID + seed + dtype + fixture/reference identity payload | +| `tests/test_ws1_workload.py` | Schema + identity + matrix + naming + backend completeness | +| `docs/design/ws1-c2-268-workload-plan.md` | This plan (closeout evidence pointer) | + +Reuse, do not fork: + +- C1 roles: `comparison_lhs_role` / `comparison_rhs_role` from `tolerance_contract.json` — **never** put `singleton_aggregate` or bare `baseline` there. +- Op defaults: `operator_inputs.py` dims must match manifest fingerprint. +- Candidate paths: `operator_specs.py` `candidate_paths` as the path vocabulary for profile maps. + +--- + +## 4. Manifest schema (normative sections) + +```text +version / workload_id / seed +model_identity + model_id, revision, config_fingerprint{}, weight_snapshot{}, architecture_notes +chain_semantics + execution_dtype, reference_dtype, temperature, loss_reduction, + logprob_selection, clip_interval, aggregates[] +stochastic_policy + dropout, sampling_in_logprob_parity, rng_source, undeclared_randomness +primary_matrix + N, cells[{cell_id, batch_mode, prefill_mode, ...}] + batch_permutation, chunk{size, require_ge_2_chunks, non_divisible_case} +fixtures + samples[], short/long/varlen, left/right pad, packing status +logical_identity + key=(sample_id, token_position), restore_after[] +capabilities + packing, qk_norm, required_chain_ops[{op, status}] +backend_profiles + cuda_bf16 / triton_cuda_bf16 → required_nodes[{node, expected_backend_id, expected_kernel_config_id, algorithm_property}] +representative_cases[] + case_id, family(gemm|attention|logprob), shape pins, backend pins, algorithm property +``` + +### Primary matrix cells (fixed IDs) + +| cell_id | batch_mode | prefill_mode | +| --- | --- | --- | +| `B1-singleton_aggregate/full` | B=1 × N runs → aggregate | full prefill | +| `BN/full` | B=N single run | full prefill | +| `B1-singleton_aggregate/chunked` | B=1 × N → aggregate | chunked prefill | +| `BN/chunked` | B=N | chunked prefill | + +Fixed: `N=4` ( >1 ), target sample order fixed, at least one chunk size that yields ≥2 chunks and a non-divisible remainder case. + +### Backend profiles + +Enumerate every on-chain required node for full-model topology (#266 §5): + +`embedding`, `rms_norm`, `det_gemm` (Q/K/V/O/gate/up/down), `qk_norm` (elementwise/RMS), `rope`, `attention`, `swiglu`/`silu`, `lm_head`, `logprob`/`batch_invariant_logp`/`linear_logp` as declared. + +For each profile: + +- Expected `backend_id` + `kernel_config_id` (or path id from `operator_specs`). +- Missing Triton candidate for a **required** node → status `red` / `missing_required` (not N/A, not silent fallback). + +### Representative cases (stable `case_id`) + +1–3 per family, full-model graph/weights identity, seq may be short: + +| Family | Property exercised | +| --- | --- | +| GEMM | Multiple flattened-token `M`, incl. non-tile-aligned; no-Split-K path | +| Attention | Prefill/decode, GQA 32/8/128, multi KV len + non-tile-aligned; no-Split-KV | +| Logprob | Vocab/reduction crossing at least one declared block boundary | + +Changing any pinned field → new `case_id` / revision. + +--- + +## 5. Workload API (Python) + +```text +load_manifest() / validate_manifest() +build_logical_batch(workload_id) -> LogicalBatch + samples: list[LogicalSample] # sample_id, token_ids, positions, loss_mask, ... +apply_padding(batch) / apply_chunking(batch) / apply_packing(batch) -> physical layout +restore_logical_order(physical, values) -> aligned values keyed by (sample_id, token_position) +singleton_aggregate_plan(N samples) -> execution schedule for B1×N vs BN +fixture_hash(batch|manifest) -> stable hex +matrix_cells() / get_cell(cell_id) +profile_required_nodes(profile_id) +get_case(case_id) +``` + +Rules: + +- After pad/pack/chunk, compare **only** after `restore_logical_order`. +- B1 `singleton_aggregate` and BN share the **same** logical sample/token multiset and fixed aggregation order + active-token denominator. +- Hard-fail on undeclared stochastic sources when building gate fixtures. + +--- + +## 6. Reference command + +```bash +python scripts/ws1_reference.py \ + --workload-id \ + --seed \ + --dtype bf16 \ + [--cell-id BN/full] \ + [--emit-json path|-] +``` + +Emits: workload_id, seed, dtype, fixture_hash, model identity pins, cell descriptor, +clip_interval, profile ids, and deterministic logical/pad/chunk/pack/short/long tensor digests. +Does **not** run full 8B forward (owned by C9/C10); may emit tensor fixture digests for token/mask tensors only. + +--- + +## 7. Test plan (`tests/test_ws1_workload.py`) + +| Test group | Asserts | +| --- | --- | +| Schema | Every numerics-affecting field present; no forbidden comparison roles | +| Model identity | Full fingerprint; no shrink fields; weight pin present | +| Repro | Same workload_id → same fixture_hash / sample multiset | +| Logical identity | pad / chunk / (pack if supported) restore `(sample_id, token_position)` | +| Aggregate | B1 singleton plan multiset == BN multiset; fixed order | +| Naming | `singleton_aggregate` not in C1 role sets; no bare `baseline` in report fields | +| Matrix | 2×2 cells fixed; N>1; perm; multi-chunk non-divisible | +| Clip / RNG | clip_interval pinned; dropout=0; undeclared RNG rejected | +| Profiles | Both profiles list all required nodes; missing Triton required → red | +| Cases | Stable case_id; expected+actual path fields schema; algorithm property present | +| CLI | `ws1_reference.py` exits 0 and prints workload_id/seed/dtype/hash | + +CPU-only; no GPU / no weight download required for C2 unit tests. + +--- + +## 8. Implementation order + +1. **Manifest JSON** with full pins (model, matrix, fixtures, profiles, cases). +2. **`ws1_workload.py`** loader + validators + logical batch + pad/chunk restore + hash. +3. **`scripts/ws1_reference.py`** thin CLI. +4. **Tests** green on CPU. +5. Wire exports in `rl_engine/testing/__init__.py` (minimal public surface). +6. Short evidence comment map on PR / issue #268 (acceptance checklist). + +--- + +## 9. Explicit non-goals (stay out) + +| Out | Owner | +| --- | --- | +| Four-judgment numerical asserts / #150 matrix green | C10 | +| Forward harness + backend provenance runtime | C3 | +| Gradient harness | C4 | +| Full model assembly / real 8B run | C9 | +| Stateful KV / generate-rescore | C6/C7 | +| CI gate jobs | C11 | +| Multi-GPU | WS2 | + +--- + +## 10. Acceptance ↔ evidence map + +| #268 AC | Evidence | +| --- | --- | +| Manifest pins numerics fields | `ws1_manifest.json` + schema tests | +| Full Qwen3-8B Dense identity | `model_identity` section | +| Same workload_id → same identity | `fixture_hash` tests | +| Transforms restore logical identity | pad/chunk restore tests | +| B1 singleton vs BN same multiset | aggregate plan tests | +| Naming boundary vs C1 roles | forbidden-role tests + contract cross-check | +| 2×2 + perm + multi-chunk | matrix section + tests | +| clip_interval pinned | manifest + tests | +| Dropout/RNG policy | stochastic_policy + hard-fail test | +| Short + rep fixtures hit candidates | representative_cases + profile map | +| Stable case_id | cases + tests | +| expected backend/kernel pins | cases + profiles | +| One reference command | `scripts/ws1_reference.py` | +| Packing / QK-Norm / ops status | capabilities | +| Both profiles enumerate nodes | backend_profiles tests | + +--- + +## 11. Risk notes + +1. **Weight identity without multi-GB download:** pin HF revision, index SHA-256, every + shard's official LFS content SHA-256/size, and a reproducible aggregate digest. +2. **Triton gaps:** declare `missing_required` honestly for nodes without Triton candidates (e.g. some embedding/lm_head paths) — C2 records red status; does not invent fallbacks. +3. **Packing:** because `NativePackOp` exists, C2 marks packing supported, freezes the + variable-length packed fixture, and round-trips its logical identity even though packing + is outside the primary 2×2 matrix. +4. **C1 alignment:** re-export clip_interval default from C1; dual-write in manifest so C2 is self-contained for gates. + +--- + +## 12. Done definition for this branch + +- [x] Docking files land on `feat/ws1-c2-canonical-workload-268` without rewriting C1 tip history. +- [x] `pytest tests/test_ws1_workload.py -q` green — 33 passed (CPU). +- [x] `python scripts/ws1_reference.py` emits workload_id / seed / dtype / fixture digests. +- [x] Closeout evidence: `docs/design/ws1-c2-268-closeout-evidence.md`. +- [x] #268 residual closeout: per-sample `completion_lens`, TF32 ref, report naming, registry-vs-runtime actual boundary. +- [x] Explicit non-claim: does not close #266 or turn Triton missing_required green. diff --git a/rl_engine/testing/__init__.py b/rl_engine/testing/__init__.py index 42be8c1b..eff1be01 100644 --- a/rl_engine/testing/__init__.py +++ b/rl_engine/testing/__init__.py @@ -13,15 +13,39 @@ summarize_kernel_drift, ) from .rl_batch import SyntheticRLKernelBatch, make_synthetic_rl_kernel_batch +from .ws1_workload import ( + LogicalBatch, + PhysicalLayout, + WorkloadError, + WS1Manifest, + apply_chunking, + apply_packing, + build_logical_batch, + fixture_hash, + load_manifest, + reference_payload, + restore_logical_order, +) __all__ = [ + "LogicalBatch", + "PhysicalLayout", "SyntheticRLKernelBatch", + "WS1Manifest", + "WorkloadError", "active_token_count", + "apply_chunking", + "apply_packing", + "build_logical_batch", "compute_policy_ratio", "compute_reference_kl", + "fixture_hash", + "load_manifest", "make_synthetic_rl_kernel_batch", "masked_mean", "masked_sum", + "reference_payload", + "restore_logical_order", "selected_logprobs_reference", "summarize_kernel_drift", ] diff --git a/rl_engine/testing/ws1_manifest.json b/rl_engine/testing/ws1_manifest.json new file mode 100644 index 00000000..b3e69cb9 --- /dev/null +++ b/rl_engine/testing/ws1_manifest.json @@ -0,0 +1,1032 @@ +{ + "version": "ws1-c2-v3", + "workload_id": "ws1-qwen3-8b-dense-primary-v3", + "seed": 20260812, + "model_identity": { + "model_id": "Qwen/Qwen3-8B", + "hf_repo": "Qwen/Qwen3-8B", + "revision": "b968826d9c46dd6066d109eabc6255188de91218", + "architecture": "Qwen3ForCausalLM", + "model_type": "qwen3", + "density": "dense", + "exit_forbids_architecture_shrink": true, + "config_fingerprint": { + "num_hidden_layers": 36, + "hidden_size": 4096, + "intermediate_size": 12288, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 128, + "vocab_size": 151936, + "max_position_embeddings": 40960, + "rope_theta": 1000000.0, + "rope_scaling": null, + "rms_norm_eps": 1e-06, + "hidden_act": "silu", + "swiglu": true, + "tie_word_embeddings": false, + "attention_bias": false, + "attention_dropout": 0.0, + "use_sliding_window": false, + "sliding_window": null, + "qk_norm": true, + "qk_norm_note": "Qwen3 applies per-head RMSNorm on Q and K before RoPE; not a separate HF config flag." + }, + "weight_snapshot": { + "pin_method": "hf_revision_plus_index_sha256_plus_all_lfs_shard_sha256", + "index_file": "model.safetensors.index.json", + "index_sha256": "f9fdbcb91c23971c13ec5d5f2573d2349e8f61f2f049371ec699281748fdb1bc", + "tensor_total_size_bytes": 16381470720, + "weight_files_total_size_bytes": 16381516776, + "total_size_bytes": 16381470720, + "content_hash_algorithm": "sha256-of-sorted-shard-records-v1", + "content_hash": "fc664a19c52c82b6f5ddb33d4fe2723181daeb93a344b16fee6369963e5a13a5", + "shards": [ + { + "filename": "model-00001-of-00005.safetensors", + "sha256": "31d6a825ae35f11fb85b195b4c42c146c051e446433125a215336abdf95cbf5f", + "size_bytes": 3996250744 + }, + { + "filename": "model-00002-of-00005.safetensors", + "sha256": "5991236cea6fe21f3d43cab0f0e84448734fbbe0789816202989f2ddc9d18282", + "size_bytes": 3993160032 + }, + { + "filename": "model-00003-of-00005.safetensors", + "sha256": "c5185c4794be2d8a9784d5753c9922db38df478ce11f9ed0b415b7304d896836", + "size_bytes": 3959604768 + }, + { + "filename": "model-00004-of-00005.safetensors", + "sha256": "b5ee7de71fbf17db3d5704e0c8f2bc7d005ca9e1d7ca2aeb19827b0cfcaa917a", + "size_bytes": 3187841392 + }, + { + "filename": "model-00005-of-00005.safetensors", + "sha256": "20c2d6366ab85c90786ccdd829cd2b9e7d30ef3b2ebbb998280e7e4014b542ff", + "size_bytes": 1244659840 + } + ], + "source": "HF LFS x-linked-etag at the pinned revision; each value is the shard content SHA-256" + } + }, + "chain_semantics": { + "execution_dtype": "bfloat16", + "reference_dtype": "float32", + "accumulation_dtype": "float32", + "temperature": 1.0, + "loss_reduction": "sum_over_active_tokens_then_optional_mean_by_active_count", + "logprob_selection": "selected_token_logprob_on_active_mask", + "active_token_policy": "active selected tokens only", + "aggregates": [ + "max_abs_dlogp", + "approx_kl0", + "clipfrac0" + ], + "clip_interval": [ + 0.8, + 1.2 + ], + "clip_interval_note": "Pinned for clipfrac0; must match C1 chain_logprob_aggregates.default_clip_interval unless an explicit contract revision changes both.", + "comparison_roles_source": "rl_engine/kernels/gtest/tolerance_contract.json", + "forbidden_comparison_roles": [ + "baseline", + "singleton_aggregate" + ], + "singleton_aggregate_note": "singleton_aggregate is a C2 execution/aggregation mode only. It must never populate comparison_lhs_role or comparison_rhs_role.", + "tf32_policy_ref": "rl_engine/kernels/gtest/tolerance_contract.json#/policy/tf32", + "tf32_note": "WS1 TF32 enable/disable is owned by the C1 contract; C2 gates must not introduce a private TF32 policy.", + "report_naming": { + "comparison_lhs_role": "from_c1_by_report_kind", + "comparison_rhs_role": "from_c1_by_report_kind", + "forbidden_in_reports": [ + "baseline", + "singleton_aggregate" + ], + "singleton_aggregate_is": "c2_execution_aggregation_mode_only", + "note": "C2 freezes naming rules; C3+ emit reports that must obey these roles." + }, + "backend_actual_semantics": { + "c2_actual_backend_id": "registry_resolved_expected_candidate", + "c2_actual_kernel_config_id": "operator_specs_candidate_path", + "runtime_observed_actual_owner": [ + "C3", + "C8", + "C10", + "C11" + ], + "note": "For C2, actual_* equals expected_* after operator_specs resolution. GPU runtime provenance that proves a live kernel hit is owned by later closeout children; missing required Triton nodes stay status=missing_required (red)." + } + }, + "stochastic_policy": { + "dropout": 0.0, + "attention_dropout": 0.0, + "sampling_in_logprob_parity": false, + "canonical_gate_uses_dropout_zero": true, + "rng_source": "manifest_seed_plus_logical_sample_token_identity", + "undeclared_randomness": "hard_fail", + "retained_stochastic_ops": [] + }, + "primary_matrix": { + "description": "Fixed #150 Batch × Chunked-Prefill matrix prerequisite workload cells.", + "N": 4, + "batch_size_bn": 4, + "sample_ids": [ + "s0", + "s1", + "s2", + "s3" + ], + "sample_order_fixed": true, + "batch_permutation": { + "enabled": true, + "permutation": [ + 2, + 0, + 3, + 1 + ], + "target_sample_position_in_bn": 0, + "note": "Permutation exercises layout invariance; logical compare restores sample_id order." + }, + "chunk": { + "chunk_size_tokens": 7, + "require_ge_2_chunks": true, + "non_divisible_case": true, + "note": "Longest primary seq_len=19 with chunk_size=7 yields chunks [7,7,5]." + }, + "cells": [ + { + "cell_id": "B1-singleton_aggregate/full", + "batch_mode": "singleton_aggregate", + "batch_size_per_run": 1, + "num_runs": 4, + "prefill_mode": "full", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + }, + { + "cell_id": "BN/full", + "batch_mode": "batched", + "batch_size_per_run": 4, + "num_runs": 1, + "prefill_mode": "full", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + }, + { + "cell_id": "B1-singleton_aggregate/chunked", + "batch_mode": "singleton_aggregate", + "batch_size_per_run": 1, + "num_runs": 4, + "prefill_mode": "chunked", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + }, + { + "cell_id": "BN/chunked", + "batch_mode": "batched", + "batch_size_per_run": 4, + "num_runs": 1, + "prefill_mode": "chunked", + "aggregation": { + "order": "sample_ids", + "denominator": "active_token_count_across_all_samples" + } + } + ] + }, + "fixtures": { + "prompt_template": "ws1_fixed_token_fixture", + "dtype_for_token_tensors": "int64", + "position_ids": { + "basis": "logical_zero_based_per_sample", + "reset_after_pack_boundary": true + }, + "attention_mask": { + "active_value": 1, + "padding_value": 0, + "causal": true + }, + "primary_seq_len": 19, + "primary_prompt_len": 8, + "short_seq_len": 8, + "long_seq_len": 32, + "varlen_seq_lens": [ + 11, + 16, + 13, + 19 + ], + "padding": { + "modes": [ + "right", + "left" + ], + "pad_token_id": 151643, + "primary_padded_len": 20 + }, + "packing": { + "status": "supported", + "implementation": "rl_engine.kernels.ops.pytorch.packing.pack.NativePackOp", + "packed_fixture": { + "sample_order": [ + "s0", + "s1", + "s2", + "s3" + ], + "segment_lengths": [ + 11, + 16, + 13, + 19 + ], + "total_tokens": 59, + "restore_key": [ + "sample_id", + "token_position" + ] + } + }, + "loss_mask": { + "prompt_tokens_active": false, + "completion_tokens_active": true + }, + "samples": [ + { + "sample_id": "s0", + "seq_len": 11, + "prompt_len": 8, + "token_ids": [ + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 200, + 201, + 202 + ] + }, + { + "sample_id": "s1", + "seq_len": 16, + "prompt_len": 8, + "token_ids": [ + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217 + ] + }, + { + "sample_id": "s2", + "seq_len": 13, + "prompt_len": 8, + "token_ids": [ + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 220, + 221, + 222, + 223, + 224 + ] + }, + { + "sample_id": "s3", + "seq_len": 19, + "prompt_len": 8, + "token_ids": [ + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240 + ] + } + ], + "short_full_model_fixture": { + "fixture_id": "short_full_model_seq8", + "seq_len": 8, + "prompt_len": 4, + "token_ids": [ + 310, + 311, + 312, + 313, + 410, + 411, + 412, + 413 + ], + "note": "Shorter sequence on full architecture+weights only; never shrinks layers/hidden/heads/vocab.", + "candidate_case_ids": [ + "gemm-m127-k4096-n4096-no-splitk-v1", + "logp-vocab151936-btok17-reduction-boundary-v1" + ] + }, + "long_full_model_fixture": { + "fixture_id": "long_full_model_seq32", + "seq_len": 32, + "prompt_len": 16, + "token_ids": [ + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + 512, + 513, + 514, + 515, + 600, + 601, + 602, + 603, + 604, + 605, + 606, + 607, + 608, + 609, + 610, + 611, + 612, + 613, + 614, + 615 + ], + "note": "Long fixed sequence on the same full architecture and pinned weight snapshot.", + "candidate_case_ids": [ + "attn-decode-gqa-b1-sq1-skv129-no-splitkv-v1", + "attn-triton-prefill-gqa-b2-sq33-skv33-no-splitkv-v1" + ] + }, + "representative_full_model_fixture": { + "fixture_id": "rep_full_model_seq16", + "seq_len": 16, + "prompt_len": 8, + "sample_ids": [ + "s0", + "s1", + "s2", + "s3" + ], + "note": "Primary variable-length matrix fixture; full architecture+weights.", + "candidate_case_ids": [ + "gemm-m256-k4096-n12288-no-splitk-v1", + "logp-bi-triton-vocab151936-btok15-v1" + ] + }, + "prompt_lens": [ + 8, + 8, + 8, + 8 + ], + "completion_lens": [ + 3, + 8, + 5, + 11 + ], + "max_completion_len": 11 + }, + "logical_identity": { + "key": [ + "sample_id", + "token_position" + ], + "token_position_basis": "logical_unpadded_index_in_sample", + "restore_before_compare_after": [ + "pad", + "pack", + "chunk", + "batch_permute" + ], + "gradient_singleton_aggregate": { + "definition": "N independent B=1 runs of the same N logical samples, aggregated with fixed sample order and active-token denominator", + "compare_to": "single B=N run of the same logical sample/token multiset", + "forbid_different_sample_sets": true + } + }, + "capabilities": { + "packing": { + "status": "supported", + "detail": "NativePackOp is present; C2 pins and round-trips the packed variable-length fixture even though packing is outside the primary 2x2 matrix." + }, + "qk_norm": { + "status": "required_on_chain", + "detail": "Qwen3-8B Dense applies QK-Norm before RoPE on every layer." + }, + "operator_spec_map": { + "embedding": "embedding", + "rms_norm": "rms_norm", + "det_gemm": "det_gemm", + "qk_norm": "rms_norm", + "rope": "rope", + "attention": "attention", + "swiglu": "swiglu", + "silu": "silu", + "lm_head": "lm_head", + "logprob": "logp", + "batch_invariant_logp": "batch_invariant_logp" + }, + "required_chain_ops": [ + { + "op": "embedding", + "status": "required" + }, + { + "op": "rms_norm", + "status": "required" + }, + { + "op": "det_gemm", + "status": "required" + }, + { + "op": "qk_norm", + "status": "required" + }, + { + "op": "rope", + "status": "required" + }, + { + "op": "attention", + "status": "required" + }, + { + "op": "swiglu", + "status": "required" + }, + { + "op": "silu", + "status": "required" + }, + { + "op": "lm_head", + "status": "required" + }, + { + "op": "logprob", + "status": "required" + }, + { + "op": "batch_invariant_logp", + "status": "required" + }, + { + "op": "linear_logp", + "status": "optional_fused_path" + } + ] + }, + "backend_profiles": { + "cuda_bf16": { + "backend_family": "cuda", + "execution_dtype": "bfloat16", + "required_nodes": [ + { + "node": "embedding", + "expected_backend_id": "cuda-sm90", + "expected_kernel_config_id": "sm90_embedding", + "algorithm_property": "deterministic_table_lookup", + "status": "declared" + }, + { + "node": "rms_norm", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_rmsnorm_bf16", + "algorithm_property": "fused_batch_invariant_rmsnorm", + "status": "declared" + }, + { + "node": "det_gemm", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_det_gemm_no_splitk", + "algorithm_property": "no_split_k_deterministic_gemm", + "status": "declared" + }, + { + "node": "qk_norm", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_rmsnorm_qk", + "algorithm_property": "per_head_rms_on_q_k", + "status": "declared" + }, + { + "node": "rope", + "expected_backend_id": "cuda-sm90", + "expected_kernel_config_id": "sm90_rope", + "algorithm_property": "rotate_half_theta_1e6", + "status": "declared" + }, + { + "node": "attention", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_deterministic_attn_no_splitkv", + "algorithm_property": "no_split_kv_batch_invariant_attention", + "status": "declared" + }, + { + "node": "swiglu", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_swiglu", + "algorithm_property": "elementwise_swiglu", + "status": "declared" + }, + { + "node": "silu", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_silu", + "algorithm_property": "elementwise_silu", + "status": "declared" + }, + { + "node": "lm_head", + "expected_backend_id": "cuda-sm90", + "expected_kernel_config_id": "sm90_lm_head", + "algorithm_property": "deterministic_untied_lm_head", + "status": "declared" + }, + { + "node": "logprob", + "expected_backend_id": "cuda", + "expected_kernel_config_id": "cuda_fused_logp_generic", + "algorithm_property": "deterministic_selected_logprob", + "status": "declared" + }, + { + "node": "batch_invariant_logp", + "expected_backend_id": "cuda-sm90", + "expected_kernel_config_id": "sm90_batch_invariant_logp", + "algorithm_property": "batch_invariant_logprob_reduction", + "status": "declared" + } + ] + }, + "triton_cuda_bf16": { + "backend_family": "triton", + "execution_dtype": "bfloat16", + "required_nodes": [ + { + "node": "embedding", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "deterministic_table_lookup", + "status": "missing_required", + "note": "No Triton embedding candidate in operator_specs; profile is red for this node until a candidate is declared — not N/A, not silent fallback." + }, + { + "node": "rms_norm", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_rmsnorm_bf16", + "algorithm_property": "fused_batch_invariant_rmsnorm", + "status": "declared" + }, + { + "node": "det_gemm", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_det_gemm_no_splitk", + "algorithm_property": "no_split_k_deterministic_gemm", + "status": "declared" + }, + { + "node": "qk_norm", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_rmsnorm_qk", + "algorithm_property": "per_head_rms_on_q_k", + "status": "declared" + }, + { + "node": "rope", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_rope", + "algorithm_property": "rotate_half_theta_1e6", + "status": "declared" + }, + { + "node": "attention", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_batch_invariant_attn_no_splitkv", + "algorithm_property": "no_split_kv_batch_invariant_attention", + "status": "declared" + }, + { + "node": "swiglu", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_swiglu", + "algorithm_property": "elementwise_swiglu", + "status": "declared" + }, + { + "node": "silu", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_silu", + "algorithm_property": "elementwise_silu", + "status": "declared" + }, + { + "node": "lm_head", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "deterministic_untied_lm_head", + "status": "missing_required", + "note": "No Triton lm_head candidate in operator_specs; profile is red for this node until declared." + }, + { + "node": "logprob", + "expected_backend_id": null, + "expected_kernel_config_id": null, + "algorithm_property": "deterministic_selected_logprob", + "status": "missing_required", + "note": "operator_specs logp has no triton candidate; use batch_invariant_logp/linear_logp where applicable. Node stays missing_required for plain logp." + }, + { + "node": "batch_invariant_logp", + "expected_backend_id": "triton", + "expected_kernel_config_id": "triton_batch_invariant_logp", + "algorithm_property": "batch_invariant_logprob_reduction", + "status": "declared" + } + ] + } + }, + "representative_cases": [ + { + "case_id": "gemm-m127-k4096-n4096-no-splitk-v1", + "family": "gemm", + "revision": 1, + "shape": { + "M": 127, + "K": 4096, + "N": 4096, + "note": "Non-tile-aligned flattened-token M on full-model projection K/N." + }, + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", + "actual_backend_id": "cuda", + "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_k_deterministic_gemm", + "profile_ids": [ + "cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "cuda", + "resolved_path": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", + "algorithm_source": "rl_engine/kernels/ops/cuda/matmul/det_gemm.py:3", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "gemm-m256-k4096-n12288-no-splitk-v1", + "family": "gemm", + "revision": 1, + "shape": { + "M": 256, + "K": 4096, + "N": 12288, + "note": "Gate/up projection width (intermediate_size)." + }, + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", + "actual_backend_id": "cuda", + "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_k_deterministic_gemm", + "profile_ids": [ + "cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "cuda", + "resolved_path": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", + "algorithm_source": "rl_engine/kernels/ops/cuda/matmul/det_gemm.py:3", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "gemm-triton-m63-k4096-n4096-no-splitk-v1", + "family": "gemm", + "revision": 1, + "shape": { + "M": 63, + "K": 4096, + "N": 4096, + "note": "Non-tile-aligned M on Triton det_gemm path." + }, + "expected_backend_id": "triton", + "expected_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", + "actual_backend_id": "triton", + "actual_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_k_deterministic_gemm", + "profile_ids": [ + "triton_cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "triton", + "resolved_path": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", + "algorithm_source": "rl_engine/kernels/ops/triton/matmul/det_gemm.py:3", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "gemm-triton-m256-k4096-n12288-no-splitk-v1", + "family": "gemm", + "revision": 1, + "shape": { + "M": 256, + "K": 4096, + "N": 12288, + "note": "Second Triton M and full gate/up projection width." + }, + "expected_backend_id": "triton", + "expected_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", + "actual_backend_id": "triton", + "actual_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_k_deterministic_gemm", + "profile_ids": [ + "triton_cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "triton", + "resolved_path": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", + "algorithm_source": "rl_engine/kernels/ops/triton/matmul/det_gemm.py:3", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "attn-prefill-gqa-b2-sq31-skv31-no-splitkv-v1", + "family": "attention", + "revision": 1, + "shape": { + "B": 2, + "Hq": 32, + "Hkv": 8, + "Sq": 31, + "Skv": 31, + "D": 128, + "mode": "prefill", + "note": "Non-tile-aligned sequence; GQA as official." + }, + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", + "actual_backend_id": "cuda", + "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_kv_batch_invariant_attention", + "profile_ids": [ + "cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "cuda", + "resolved_path": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", + "algorithm_source": "rl_engine/kernels/ops/cuda/attention/deterministic_attn.py:3", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "attn-decode-gqa-b1-sq1-skv129-no-splitkv-v1", + "family": "attention", + "revision": 1, + "shape": { + "B": 1, + "Hq": 32, + "Hkv": 8, + "Sq": 1, + "Skv": 129, + "D": 128, + "mode": "decode", + "note": "Decode step with non-tile-aligned KV length." + }, + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", + "actual_backend_id": "cuda", + "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_kv_batch_invariant_attention", + "profile_ids": [ + "cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "cuda", + "resolved_path": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", + "algorithm_source": "rl_engine/kernels/ops/cuda/attention/deterministic_attn.py:3", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "attn-triton-prefill-gqa-b2-sq33-skv33-no-splitkv-v1", + "family": "attention", + "revision": 1, + "shape": { + "B": 2, + "Hq": 32, + "Hkv": 8, + "Sq": 33, + "Skv": 33, + "D": 128, + "mode": "prefill", + "note": "Triton batch-invariant attention; non-power-of-two seq." + }, + "expected_backend_id": "triton", + "expected_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", + "actual_backend_id": "triton", + "actual_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_kv_batch_invariant_attention", + "profile_ids": [ + "triton_cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "triton", + "resolved_path": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", + "algorithm_source": "rl_engine/kernels/ops/triton/attention/standard_attn.py:1", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "attn-triton-decode-gqa-b1-sq1-skv129-no-splitkv-v1", + "family": "attention", + "revision": 1, + "shape": { + "B": 1, + "Hq": 32, + "Hkv": 8, + "Sq": 1, + "Skv": 129, + "D": 128, + "mode": "decode", + "note": "Triton decode with non-tile-aligned KV length." + }, + "expected_backend_id": "triton", + "expected_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", + "actual_backend_id": "triton", + "actual_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "no_split_kv_batch_invariant_attention", + "profile_ids": [ + "triton_cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "triton", + "resolved_path": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", + "algorithm_source": "rl_engine/kernels/ops/triton/attention/standard_attn.py:1", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "logp-vocab151936-btok17-reduction-boundary-v1", + "family": "logprob", + "revision": 1, + "shape": { + "B": 1, + "T": 17, + "vocab": 151936, + "note": "Full vocab; token count crosses common 16-aligned reduction boundary." + }, + "expected_backend_id": "cuda", + "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", + "actual_backend_id": "cuda", + "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "deterministic_selected_logprob", + "profile_ids": [ + "cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "cuda", + "resolved_path": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", + "algorithm_source": "rl_engine/kernels/ops/cuda/loss/logp.py:213", + "runtime_evidence_owner": "C8/C10/C11" + } + }, + { + "case_id": "logp-bi-triton-vocab151936-btok15-v1", + "family": "logprob", + "revision": 1, + "shape": { + "B": 1, + "T": 15, + "vocab": 151936, + "note": "Triton batch-invariant logp on full vocab; non-aligned T." + }, + "expected_backend_id": "triton", + "expected_kernel_config_id": "rl_engine.kernels.ops.triton.loss.batch_invariant_logp.TritonBatchInvariantLogpOp", + "actual_backend_id": "triton", + "actual_kernel_config_id": "rl_engine.kernels.ops.triton.loss.batch_invariant_logp.TritonBatchInvariantLogpOp", + "provenance_status": "registry_resolved_runtime_pending", + "algorithm_property": "batch_invariant_logprob_reduction", + "profile_ids": [ + "triton_cuda_bf16" + ], + "architecture_identity": "full_qwen3_8b_dense", + "provenance_evidence": { + "kind": "operator_specs_registry_resolution", + "registry": "rl_engine/kernels/gtest/operator_specs.py", + "candidate_name": "triton", + "resolved_path": "rl_engine.kernels.ops.triton.loss.batch_invariant_logp.TritonBatchInvariantLogpOp", + "algorithm_source": "rl_engine/kernels/ops/triton/loss/batch_invariant_logp.py:1", + "runtime_evidence_owner": "C8/C10/C11" + } + } + ], + "fixture_identity_sha256": "c2ec565a575aa3a02c3a27d89ffba3162d93455efd53a7fcb1de11f7e9db7f3d", + "provenance_boundary": { + "c2_scope": "logical_workload_identity_and_registry_path_binding", + "not_in_c2": [ + "full_model_forward", + "numerical_150_asserts", + "runtime_kernel_dispatch_observation", + "multi_gpu" + ], + "runtime_evidence_owner": [ + "C3", + "C8", + "C10", + "C11" + ] + } +} diff --git a/rl_engine/testing/ws1_workload.py b/rl_engine/testing/ws1_workload.py new file mode 100644 index 00000000..02d21964 --- /dev/null +++ b/rl_engine/testing/ws1_workload.py @@ -0,0 +1,1078 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""WS1 C2 (#268) canonical workload: logical identity, fixtures, and manifest API. + +This module freezes the full Qwen3-8B Dense logical sample workload used by later +gates (C3–C11). It does not run the full model or assert #150 numerical thresholds. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +_MANIFEST_PATH = Path(__file__).with_name("ws1_manifest.json") + +_REQUIRED_TOP_LEVEL = ( + "version", + "workload_id", + "seed", + "model_identity", + "chain_semantics", + "stochastic_policy", + "primary_matrix", + "fixtures", + "logical_identity", + "capabilities", + "backend_profiles", + "representative_cases", + "provenance_boundary", + "fixture_identity_sha256", +) + +_REQUIRED_MATRIX_CELLS = ( + "B1-singleton_aggregate/full", + "BN/full", + "B1-singleton_aggregate/chunked", + "BN/chunked", +) + +_REQUIRED_PROFILES = ("cuda_bf16", "triton_cuda_bf16") + +_REQUIRED_CHAIN_NODES = ( + "embedding", + "rms_norm", + "det_gemm", + "qk_norm", + "rope", + "attention", + "swiglu", + "silu", + "lm_head", + "logprob", + "batch_invariant_logp", +) + +_FORBIDDEN_COMPARISON_ROLES = frozenset({"baseline", "singleton_aggregate"}) + +_OFFICIAL_FINGERPRINT = { + "num_hidden_layers": 36, + "hidden_size": 4096, + "intermediate_size": 12288, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 128, + "vocab_size": 151936, +} + + +class WorkloadError(ValueError): + """Raised when the WS1 workload manifest or fixture is invalid.""" + + +@dataclass(frozen=True) +class LogicalToken: + """One active or inactive logical token position.""" + + sample_id: str + token_position: int + token_id: int + is_active: bool + + +@dataclass(frozen=True) +class LogicalSample: + """One logical sequence with identity recoverable after layout transforms.""" + + sample_id: str + token_ids: tuple[int, ...] + prompt_len: int + seq_len: int + + def tokens(self) -> tuple[LogicalToken, ...]: + out: list[LogicalToken] = [] + for pos, tid in enumerate(self.token_ids): + out.append( + LogicalToken( + sample_id=self.sample_id, + token_position=pos, + token_id=int(tid), + is_active=pos >= self.prompt_len, + ) + ) + return tuple(out) + + def active_tokens(self) -> tuple[LogicalToken, ...]: + return tuple(t for t in self.tokens() if t.is_active) + + +@dataclass(frozen=True) +class LogicalBatch: + """Ordered multiset of logical samples for one workload cell.""" + + workload_id: str + seed: int + samples: tuple[LogicalSample, ...] + cell_id: str | None = None + + @property + def sample_ids(self) -> tuple[str, ...]: + return tuple(s.sample_id for s in self.samples) + + def logical_keys(self, *, active_only: bool = False) -> tuple[tuple[str, int], ...]: + keys: list[tuple[str, int]] = [] + for sample in self.samples: + for tok in sample.tokens(): + if active_only and not tok.is_active: + continue + keys.append((tok.sample_id, tok.token_position)) + return tuple(keys) + + def active_token_count(self) -> int: + return sum(1 for s in self.samples for t in s.tokens() if t.is_active) + + def token_multiset(self, *, active_only: bool = True) -> tuple[tuple[str, int, int], ...]: + """Return (sample_id, token_position, token_id) multiset in fixed sample order.""" + items: list[tuple[str, int, int]] = [] + for sample in self.samples: + for tok in sample.tokens(): + if active_only and not tok.is_active: + continue + items.append((tok.sample_id, tok.token_position, tok.token_id)) + return tuple(items) + + +@dataclass(frozen=True) +class PaddedBatch: + """Right- or left-padded physical layout with restore indices.""" + + physical_token_ids: tuple[tuple[int, ...], ...] + physical_attention_mask: tuple[tuple[int, ...], ...] + physical_loss_mask: tuple[tuple[int, ...], ...] + physical_position_ids: tuple[tuple[int, ...], ...] + pad_side: str + pad_token_id: int + padded_len: int + # For each physical (batch_idx, phys_pos) -> (sample_id, token_position) or None if pad + restore_map: tuple[tuple[tuple[str, int] | None, ...], ...] + sample_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class PhysicalLayout: + """Flattened physical tokens plus an unambiguous logical restore map.""" + + layout_kind: str + physical_token_ids: tuple[int, ...] + physical_loss_mask: tuple[int, ...] + restore_map: tuple[tuple[str, int], ...] + segment_offsets: tuple[int, ...] + segment_lengths: tuple[int, ...] + + +@dataclass(frozen=True) +class ChunkPlan: + """Chunked-prefill plan for one logical sequence length.""" + + seq_len: int + chunk_size: int + chunk_spans: tuple[tuple[int, int], ...] # half-open [start, end) + + @property + def num_chunks(self) -> int: + return len(self.chunk_spans) + + +@dataclass(frozen=True) +class SingletonAggregatePlan: + """B=1 × N schedule that must match one B=N run of the same multiset.""" + + sample_ids: tuple[str, ...] + run_sample_ids: tuple[tuple[str, ...], ...] # each run is a 1-tuple + aggregation_order: tuple[str, ...] + denominator: str + token_multiset: tuple[tuple[str, int, int], ...] + + +@dataclass +class WS1Manifest: + """Validated in-memory view of ws1_manifest.json.""" + + raw: dict[str, Any] + path: Path = field(default=_MANIFEST_PATH) + + @property + def version(self) -> str: + return str(self.raw["version"]) + + @property + def workload_id(self) -> str: + return str(self.raw["workload_id"]) + + @property + def seed(self) -> int: + return int(self.raw["seed"]) + + @property + def model_identity(self) -> dict[str, Any]: + return dict(self.raw["model_identity"]) + + @property + def chain_semantics(self) -> dict[str, Any]: + return dict(self.raw["chain_semantics"]) + + @property + def clip_interval(self) -> tuple[float, float]: + interval = self.raw["chain_semantics"]["clip_interval"] + return (float(interval[0]), float(interval[1])) + + @property + def primary_matrix(self) -> dict[str, Any]: + return dict(self.raw["primary_matrix"]) + + @property + def fixtures(self) -> dict[str, Any]: + return dict(self.raw["fixtures"]) + + @property + def backend_profiles(self) -> dict[str, Any]: + return dict(self.raw["backend_profiles"]) + + @property + def representative_cases(self) -> list[dict[str, Any]]: + return list(self.raw["representative_cases"]) + + +def default_manifest_path() -> Path: + return _MANIFEST_PATH + + +def load_manifest(path: str | Path | None = None) -> WS1Manifest: + manifest_path = Path(path) if path is not None else _MANIFEST_PATH + with manifest_path.open("r", encoding="utf-8") as fh: + raw = json.load(fh) + if not isinstance(raw, dict): + raise WorkloadError("manifest root must be a JSON object") + validate_manifest(raw) + return WS1Manifest(raw=raw, path=manifest_path) + + +def validate_manifest(raw: Mapping[str, Any]) -> None: + """Hard-fail if any required C2 pin is missing or inconsistent.""" + missing = [k for k in _REQUIRED_TOP_LEVEL if k not in raw] + if missing: + raise WorkloadError(f"manifest missing top-level keys: {missing}") + + _validate_model_identity(raw["model_identity"]) + _validate_chain_semantics(raw["chain_semantics"]) + _validate_stochastic_policy(raw["stochastic_policy"]) + _validate_primary_matrix(raw["primary_matrix"], raw["fixtures"]) + _validate_fixtures(raw["fixtures"], raw["primary_matrix"]) + _validate_logical_identity(raw["logical_identity"]) + _validate_capabilities(raw["capabilities"]) + _validate_backend_profiles(raw["backend_profiles"], raw["capabilities"]) + _validate_representative_cases(raw["representative_cases"]) + expected_identity = manifest_identity_hash(raw) + if raw["fixture_identity_sha256"] != expected_identity: + raise WorkloadError( + "fixture_identity_sha256 does not match manifest; change workload_id/version " + "and regenerate the identity for any numerics-affecting edit" + ) + + +def _validate_model_identity(identity: Mapping[str, Any]) -> None: + for key in ("model_id", "revision", "config_fingerprint", "weight_snapshot"): + if key not in identity: + raise WorkloadError(f"model_identity missing {key!r}") + fp = identity["config_fingerprint"] + if not isinstance(fp, Mapping): + raise WorkloadError("config_fingerprint must be an object") + for key, expected in _OFFICIAL_FINGERPRINT.items(): + if key not in fp: + raise WorkloadError(f"config_fingerprint missing {key!r}") + if fp[key] != expected: + raise WorkloadError( + f"config_fingerprint {key}={fp[key]!r} does not match official " + f"Qwen3-8B Dense pin {expected!r}; architecture shrink is forbidden" + ) + if not identity.get("exit_forbids_architecture_shrink", False): + raise WorkloadError("exit_forbids_architecture_shrink must be true") + weight = identity["weight_snapshot"] + for key in ( + "pin_method", + "total_size_bytes", + "index_file", + "content_hash_algorithm", + "content_hash", + "shards", + ): + if key not in weight: + raise WorkloadError(f"weight_snapshot missing {key!r}") + shards = weight["shards"] + if not isinstance(shards, list) or not shards: + raise WorkloadError("weight_snapshot.shards must be a non-empty list") + if int(weight["weight_files_total_size_bytes"]) != sum( + int(s["size_bytes"]) for s in shards + ): + raise WorkloadError("weight_snapshot file total does not match shard sizes") + for shard in shards: + digest = str(shard.get("sha256", "")) + if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise WorkloadError("every weight shard must pin a lowercase SHA-256") + expected = weight_snapshot_hash(shards) + if weight["content_hash_algorithm"] != "sha256-of-sorted-shard-records-v1": + raise WorkloadError("unsupported weight_snapshot content_hash_algorithm") + if weight["content_hash"] != expected: + raise WorkloadError("weight_snapshot content_hash does not match shard records") + + +def _validate_chain_semantics(sem: Mapping[str, Any]) -> None: + for key in ( + "execution_dtype", + "reference_dtype", + "clip_interval", + "aggregates", + "forbidden_comparison_roles", + "tf32_policy_ref", + "report_naming", + "backend_actual_semantics", + ): + if key not in sem: + raise WorkloadError(f"chain_semantics missing {key!r}") + if sem["execution_dtype"] != "bfloat16": + raise WorkloadError("execution_dtype must be bfloat16 for WS1") + if sem["reference_dtype"] != "float32": + raise WorkloadError("reference_dtype must be float32 for WS1") + interval = sem["clip_interval"] + if not (isinstance(interval, (list, tuple)) and len(interval) == 2): + raise WorkloadError("clip_interval must be a length-2 list") + if float(interval[0]) >= float(interval[1]): + raise WorkloadError("clip_interval lower bound must be < upper bound") + aggregates = list(sem["aggregates"]) + for name in ("max_abs_dlogp", "approx_kl0", "clipfrac0"): + if name not in aggregates: + raise WorkloadError(f"aggregates must include {name}") + forbidden = set(sem["forbidden_comparison_roles"]) + if not _FORBIDDEN_COMPARISON_ROLES.issubset(forbidden): + raise WorkloadError( + f"forbidden_comparison_roles must include {_FORBIDDEN_COMPARISON_ROLES}" + ) + if "tolerance_contract.json" not in str(sem["tf32_policy_ref"]): + raise WorkloadError("tf32_policy_ref must point at the C1 tolerance contract") + report_naming = sem["report_naming"] + if not isinstance(report_naming, Mapping): + raise WorkloadError("report_naming must be an object") + report_forbidden = set(report_naming.get("forbidden_in_reports", [])) + if not _FORBIDDEN_COMPARISON_ROLES.issubset(report_forbidden): + raise WorkloadError( + "report_naming.forbidden_in_reports must include baseline and singleton_aggregate" + ) + if report_naming.get("singleton_aggregate_is") != "c2_execution_aggregation_mode_only": + raise WorkloadError( + "report_naming must declare singleton_aggregate as c2 execution mode only" + ) + actual_sem = sem["backend_actual_semantics"] + if not isinstance(actual_sem, Mapping): + raise WorkloadError("backend_actual_semantics must be an object") + if actual_sem.get("c2_actual_backend_id") != "registry_resolved_expected_candidate": + raise WorkloadError( + "C2 actual_backend_id semantics must be registry_resolved_expected_candidate" + ) + if "C8" not in actual_sem.get("runtime_observed_actual_owner", []): + raise WorkloadError( + "backend_actual_semantics must assign runtime observed actuals to C8+" + ) + + +def _validate_stochastic_policy(policy: Mapping[str, Any]) -> None: + for key in ("dropout", "sampling_in_logprob_parity", "undeclared_randomness"): + if key not in policy: + raise WorkloadError(f"stochastic_policy missing {key!r}") + if float(policy["dropout"]) != 0.0: + raise WorkloadError("canonical gate dropout must be 0.0") + if policy.get("sampling_in_logprob_parity", True): + raise WorkloadError("sampling_in_logprob_parity must be false") + if policy["undeclared_randomness"] != "hard_fail": + raise WorkloadError("undeclared_randomness must be hard_fail") + + +def _validate_primary_matrix(matrix: Mapping[str, Any], fixtures: Mapping[str, Any]) -> None: + n = int(matrix["N"]) + if n <= 1: + raise WorkloadError("primary_matrix.N must be > 1") + sample_ids = list(matrix["sample_ids"]) + if len(sample_ids) != n: + raise WorkloadError("sample_ids length must equal N") + if len(set(sample_ids)) != n: + raise WorkloadError("sample_ids must be unique") + perm = matrix.get("batch_permutation", {}) + if perm.get("enabled"): + p = list(perm["permutation"]) + if sorted(p) != list(range(n)): + raise WorkloadError("batch_permutation.permutation must be a permutation of [0..N)") + chunk = matrix["chunk"] + chunk_size = int(chunk["chunk_size_tokens"]) + seq_len = int(fixtures["primary_seq_len"]) + if chunk_size <= 0: + raise WorkloadError("chunk_size_tokens must be positive") + plan = build_chunk_plan(seq_len, chunk_size) + if chunk.get("require_ge_2_chunks") and plan.num_chunks < 2: + raise WorkloadError("chunk plan must create >= 2 chunks") + if chunk.get("non_divisible_case") and seq_len % chunk_size == 0: + raise WorkloadError("non_divisible_case requires seq_len % chunk_size != 0") + + cells = matrix["cells"] + if not isinstance(cells, list): + raise WorkloadError("primary_matrix.cells must be a list") + cell_ids = [c["cell_id"] for c in cells] + if set(cell_ids) != set(_REQUIRED_MATRIX_CELLS): + raise WorkloadError( + f"primary_matrix.cells must be exactly {_REQUIRED_MATRIX_CELLS}, got {cell_ids}" + ) + for cell in cells: + mode = cell["batch_mode"] + if mode not in ("singleton_aggregate", "batched"): + raise WorkloadError(f"unknown batch_mode {mode!r}") + if mode == "singleton_aggregate" and "singleton_aggregate" in str( + cell.get("comparison_lhs_role", "") + ): + raise WorkloadError("singleton_aggregate must not be used as a comparison role") + + +def _validate_fixtures(fixtures: Mapping[str, Any], matrix: Mapping[str, Any]) -> None: + samples = fixtures.get("samples") + if not isinstance(samples, list) or not samples: + raise WorkloadError("fixtures.samples must be a non-empty list") + expected_ids = list(matrix["sample_ids"]) + got_ids = [s["sample_id"] for s in samples] + if got_ids != expected_ids: + raise WorkloadError( + f"fixtures.samples order/ids must match primary_matrix.sample_ids " + f"{expected_ids}, got {got_ids}" + ) + primary_seq = int(fixtures["primary_seq_len"]) + declared_varlen = [int(x) for x in fixtures["varlen_seq_lens"]] + if declared_varlen != [int(s["seq_len"]) for s in samples]: + raise WorkloadError("varlen_seq_lens must match fixtures.samples seq_len values") + for sample in samples: + tids = sample["token_ids"] + if len(tids) != int(sample["seq_len"]): + raise WorkloadError( + f"sample {sample['sample_id']} token_ids length {len(tids)} " + f"!= sample seq_len {sample['seq_len']}" + ) + if not 0 < int(sample["prompt_len"]) < int(sample["seq_len"]): + raise WorkloadError(f"sample {sample['sample_id']} prompt_len is invalid") + if max(declared_varlen) != primary_seq: + raise WorkloadError("primary_seq_len must equal the maximum varlen sequence length") + # Per-sample prompt/completion lengths are authoritative (no stale scalar pin). + expected_prompt_lens = [int(s["prompt_len"]) for s in samples] + expected_completion_lens = [ + int(s["seq_len"]) - int(s["prompt_len"]) for s in samples + ] + if list(fixtures.get("prompt_lens", [])) != expected_prompt_lens: + raise WorkloadError("fixtures.prompt_lens must match per-sample prompt_len values") + if list(fixtures.get("completion_lens", [])) != expected_completion_lens: + raise WorkloadError( + "fixtures.completion_lens must match per-sample (seq_len - prompt_len)" + ) + if int(fixtures.get("max_completion_len", -1)) != max(expected_completion_lens): + raise WorkloadError("fixtures.max_completion_len must equal max(completion_lens)") + if "primary_completion_len" in fixtures: + raise WorkloadError( + "fixtures.primary_completion_len is forbidden under varlen primary samples; " + "use completion_lens / max_completion_len" + ) + padding = fixtures["padding"] + if "right" not in padding["modes"] or "left" not in padding["modes"]: + raise WorkloadError("padding.modes must include left and right") + packing = fixtures["packing"] + if packing["status"] not in { + "supported", + "n_a_with_capability_proof", + "unsupported", + "supported_op_not_in_exit_matrix", + }: + raise WorkloadError(f"unknown packing status {packing['status']!r}") + if packing["status"] != "supported": + raise WorkloadError("packing op is present, so C2 must pin a supported packed fixture") + if not packing.get("packed_fixture"): + raise WorkloadError("supported packing requires packed_fixture") + for name in ("short_full_model_fixture", "long_full_model_fixture"): + fixture = fixtures[name] + if len(fixture["token_ids"]) != int(fixture["seq_len"]): + raise WorkloadError(f"{name} token_ids length mismatch") + if not fixture.get("candidate_case_ids"): + raise WorkloadError(f"{name} must reference representative case IDs") + + +def _validate_logical_identity(logical: Mapping[str, Any]) -> None: + key = list(logical.get("key", [])) + if key != ["sample_id", "token_position"]: + raise WorkloadError("logical_identity.key must be [sample_id, token_position]") + grad = logical.get("gradient_singleton_aggregate", {}) + if not grad.get("forbid_different_sample_sets", False): + raise WorkloadError("gradient_singleton_aggregate must forbid different sample sets") + + +def _validate_capabilities(caps: Mapping[str, Any]) -> None: + for key in ("packing", "qk_norm", "required_chain_ops", "operator_spec_map"): + if key not in caps: + raise WorkloadError(f"capabilities missing {key!r}") + ops = {entry["op"]: entry["status"] for entry in caps["required_chain_ops"]} + for op in _REQUIRED_CHAIN_NODES: + if op not in ops: + raise WorkloadError(f"required_chain_ops missing {op!r}") + if op not in caps["operator_spec_map"]: + raise WorkloadError(f"operator_spec_map missing {op!r}") + + +def _validate_backend_profiles( + profiles: Mapping[str, Any], capabilities: Mapping[str, Any] +) -> None: + for name in _REQUIRED_PROFILES: + if name not in profiles: + raise WorkloadError(f"backend_profiles missing required profile {name!r}") + required_ops = [ + e["op"] + for e in capabilities["required_chain_ops"] + if e["status"] == "required" + ] + for name, profile in profiles.items(): + nodes = profile.get("required_nodes") + if not isinstance(nodes, list) or not nodes: + raise WorkloadError(f"profile {name} must declare required_nodes") + node_names = [n["node"] for n in nodes] + missing = [op for op in required_ops if op not in node_names] + if missing: + raise WorkloadError( + f"profile {name} missing required chain nodes {missing}; " + "undeclared missing nodes are forbidden (use status=missing_required)" + ) + for node in nodes: + status = node.get("status") + if status not in {"declared", "missing_required"}: + raise WorkloadError( + f"profile {name} node {node.get('node')}: status must be " + f"declared or missing_required, got {status!r}" + ) + if status == "missing_required": + if node.get("expected_backend_id") not in (None, ""): + raise WorkloadError( + f"profile {name} node {node['node']}: missing_required must not " + "claim an expected_backend_id" + ) + else: + for field_name in ( + "expected_backend_id", + "expected_kernel_config_id", + "algorithm_property", + ): + if not node.get(field_name): + raise WorkloadError( + f"profile {name} node {node['node']} missing {field_name}" + ) + + +def _validate_representative_cases(cases: Sequence[Mapping[str, Any]]) -> None: + if not cases: + raise WorkloadError("representative_cases must be non-empty") + ids = [c["case_id"] for c in cases] + if len(ids) != len(set(ids)): + raise WorkloadError("representative_cases case_id values must be unique") + families = {c["family"] for c in cases} + for family in ("gemm", "attention", "logprob"): + if family not in families: + raise WorkloadError(f"representative_cases must include family {family!r}") + for case in cases: + for key in ( + "case_id", + "family", + "shape", + "expected_backend_id", + "expected_kernel_config_id", + "actual_backend_id", + "actual_kernel_config_id", + "provenance_status", + "provenance_evidence", + "algorithm_property", + "architecture_identity", + ): + if key not in case: + raise WorkloadError(f"case {case.get('case_id')} missing {key!r}") + if case["architecture_identity"] != "full_qwen3_8b_dense": + raise WorkloadError( + f"case {case['case_id']} must pin architecture_identity=full_qwen3_8b_dense" + ) + if case["provenance_status"] != "registry_resolved_runtime_pending": + raise WorkloadError( + f"case {case['case_id']} must distinguish registry resolution from runtime" + ) + if case["actual_backend_id"] != case["expected_backend_id"]: + raise WorkloadError(f"case {case['case_id']} actual backend mismatch") + if case["actual_kernel_config_id"] != case["expected_kernel_config_id"]: + raise WorkloadError(f"case {case['case_id']} actual kernel mismatch") + evidence = case["provenance_evidence"] + if evidence.get("kind") != "operator_specs_registry_resolution": + raise WorkloadError(f"case {case['case_id']} lacks registry provenance") + if evidence.get("resolved_path") != case["actual_kernel_config_id"]: + raise WorkloadError(f"case {case['case_id']} evidence path mismatch") + if not evidence.get("algorithm_source"): + raise WorkloadError(f"case {case['case_id']} lacks algorithm source proof") + for profile in _REQUIRED_PROFILES: + profile_cases = [c for c in cases if profile in c.get("profile_ids", [])] + for family in ("gemm", "attention", "logprob"): + count = sum(c["family"] == family for c in profile_cases) + if not 1 <= count <= 3: + raise WorkloadError( + f"profile {profile} must have 1-3 {family} representative cases" + ) + gemm_m = {int(c["shape"]["M"]) for c in profile_cases if c["family"] == "gemm"} + if len(gemm_m) < 2: + raise WorkloadError(f"profile {profile} GEMM cases require multiple M values") + attn_modes = { + c["shape"]["mode"] for c in profile_cases if c["family"] == "attention" + } + if attn_modes != {"prefill", "decode"}: + raise WorkloadError(f"profile {profile} attention cases require prefill+decode") + + +def build_logical_batch( + manifest: WS1Manifest | None = None, + *, + cell_id: str | None = None, + sample_ids: Sequence[str] | None = None, +) -> LogicalBatch: + """Build the fixed logical sample multiset for the primary workload.""" + m = manifest if manifest is not None else load_manifest() + fixtures = m.fixtures + matrix = m.primary_matrix + by_id = {s["sample_id"]: s for s in fixtures["samples"]} + order = list(sample_ids) if sample_ids is not None else list(matrix["sample_ids"]) + samples: list[LogicalSample] = [] + for sid in order: + if sid not in by_id: + raise WorkloadError(f"unknown sample_id {sid!r}") + raw = by_id[sid] + token_ids = tuple(int(x) for x in raw["token_ids"]) + samples.append( + LogicalSample( + sample_id=sid, + token_ids=token_ids, + prompt_len=int(raw["prompt_len"]), + seq_len=int(raw["seq_len"]), + ) + ) + if cell_id is not None: + get_matrix_cell(m, cell_id) + return LogicalBatch( + workload_id=m.workload_id, + seed=m.seed, + samples=tuple(samples), + cell_id=cell_id, + ) + + +def get_matrix_cell(manifest: WS1Manifest, cell_id: str) -> dict[str, Any]: + for cell in manifest.primary_matrix["cells"]: + if cell["cell_id"] == cell_id: + return dict(cell) + raise WorkloadError(f"unknown cell_id {cell_id!r}") + + +def matrix_cell_ids(manifest: WS1Manifest | None = None) -> tuple[str, ...]: + m = manifest if manifest is not None else load_manifest() + return tuple(c["cell_id"] for c in m.primary_matrix["cells"]) + + +def build_chunk_plan(seq_len: int, chunk_size: int) -> ChunkPlan: + if chunk_size <= 0: + raise WorkloadError("chunk_size must be positive") + if seq_len <= 0: + raise WorkloadError("seq_len must be positive") + spans: list[tuple[int, int]] = [] + start = 0 + while start < seq_len: + end = min(start + chunk_size, seq_len) + spans.append((start, end)) + start = end + return ChunkPlan(seq_len=seq_len, chunk_size=chunk_size, chunk_spans=tuple(spans)) + + +def chunk_plan_from_manifest(manifest: WS1Manifest | None = None) -> ChunkPlan: + m = manifest if manifest is not None else load_manifest() + return build_chunk_plan( + int(m.fixtures["primary_seq_len"]), + int(m.primary_matrix["chunk"]["chunk_size_tokens"]), + ) + + +def apply_chunking(batch: LogicalBatch, *, chunk_size: int) -> PhysicalLayout: + """Materialize chunked-prefill order for every sample.""" + if chunk_size <= 0: + raise WorkloadError("chunk_size must be positive") + ids: list[int] = [] + masks: list[int] = [] + restore: list[tuple[str, int]] = [] + offsets: list[int] = [] + lengths: list[int] = [] + for sample in batch.samples: + plan = build_chunk_plan(sample.seq_len, chunk_size) + for start, end in plan.chunk_spans: + offsets.append(len(ids)) + lengths.append(end - start) + for pos in range(start, end): + ids.append(sample.token_ids[pos]) + masks.append(int(pos >= sample.prompt_len)) + restore.append((sample.sample_id, pos)) + return PhysicalLayout( + layout_kind="chunked", + physical_token_ids=tuple(ids), + physical_loss_mask=tuple(masks), + restore_map=tuple(restore), + segment_offsets=tuple(offsets), + segment_lengths=tuple(lengths), + ) + + +def apply_packing(batch: LogicalBatch) -> PhysicalLayout: + """Pack variable-length samples in fixed sample/token order.""" + ids: list[int] = [] + masks: list[int] = [] + restore: list[tuple[str, int]] = [] + offsets: list[int] = [] + lengths: list[int] = [] + for sample in batch.samples: + offsets.append(len(ids)) + lengths.append(sample.seq_len) + ids.extend(sample.token_ids) + masks.extend(int(pos >= sample.prompt_len) for pos in range(sample.seq_len)) + restore.extend((sample.sample_id, pos) for pos in range(sample.seq_len)) + return PhysicalLayout( + layout_kind="packed", + physical_token_ids=tuple(ids), + physical_loss_mask=tuple(masks), + restore_map=tuple(restore), + segment_offsets=tuple(offsets), + segment_lengths=tuple(lengths), + ) + + +def restore_logical_order( + layout: PhysicalLayout, physical_values: Sequence[Any] +) -> dict[tuple[str, int], Any]: + if len(physical_values) != len(layout.restore_map): + raise WorkloadError("physical_values length does not match restore map") + out: dict[tuple[str, int], Any] = {} + for key, value in zip(layout.restore_map, physical_values): + if key in out: + raise WorkloadError(f"duplicate logical key {key}") + out[key] = value + return out + + +def apply_padding( + batch: LogicalBatch, + *, + pad_side: str, + padded_len: int | None = None, + pad_token_id: int | None = None, + manifest: WS1Manifest | None = None, +) -> PaddedBatch: + """Pad logical sequences; restore_map recovers (sample_id, token_position).""" + if pad_side not in ("left", "right"): + raise WorkloadError(f"pad_side must be left or right, got {pad_side!r}") + m = manifest if manifest is not None else load_manifest() + pad_id = ( + int(pad_token_id) + if pad_token_id is not None + else int(m.fixtures["padding"]["pad_token_id"]) + ) + target_len = ( + int(padded_len) + if padded_len is not None + else int(m.fixtures["padding"]["primary_padded_len"]) + ) + max_seq = max(s.seq_len for s in batch.samples) + if target_len < max_seq: + raise WorkloadError(f"padded_len {target_len} < max logical seq_len {max_seq}") + + physical_ids: list[tuple[int, ...]] = [] + masks: list[tuple[int, ...]] = [] + loss_masks: list[tuple[int, ...]] = [] + positions: list[tuple[int, ...]] = [] + restore: list[tuple[tuple[str, int] | None, ...]] = [] + for sample in batch.samples: + pad_count = target_len - sample.seq_len + pad_tokens = (pad_id,) * pad_count + pad_restore: tuple[None, ...] = (None,) * pad_count + logical_restore = tuple( + (sample.sample_id, pos) for pos in range(sample.seq_len) + ) + if pad_side == "right": + ids = sample.token_ids + pad_tokens + mask = (1,) * sample.seq_len + (0,) * pad_count + rmap = logical_restore + pad_restore + loss_mask = tuple( + int(pos >= sample.prompt_len) for pos in range(sample.seq_len) + ) + (0,) * pad_count + position_ids = tuple(range(sample.seq_len)) + (0,) * pad_count + else: + ids = pad_tokens + sample.token_ids + mask = (0,) * pad_count + (1,) * sample.seq_len + rmap = pad_restore + logical_restore + loss_mask = (0,) * pad_count + tuple( + int(pos >= sample.prompt_len) for pos in range(sample.seq_len) + ) + position_ids = (0,) * pad_count + tuple(range(sample.seq_len)) + physical_ids.append(ids) + masks.append(mask) + loss_masks.append(loss_mask) + positions.append(position_ids) + restore.append(rmap) + + return PaddedBatch( + physical_token_ids=tuple(physical_ids), + physical_attention_mask=tuple(masks), + physical_loss_mask=tuple(loss_masks), + physical_position_ids=tuple(positions), + pad_side=pad_side, + pad_token_id=pad_id, + padded_len=target_len, + restore_map=tuple(restore), + sample_ids=batch.sample_ids, + ) + + +def restore_logical_order_from_padded( + padded: PaddedBatch, + physical_values: Sequence[Sequence[Any]], +) -> dict[tuple[str, int], Any]: + """Map physical per-position values back to logical (sample_id, token_position).""" + if len(physical_values) != len(padded.restore_map): + raise WorkloadError("physical_values batch size mismatch") + out: dict[tuple[str, int], Any] = {} + for row_vals, row_map in zip(physical_values, padded.restore_map): + if len(row_vals) != len(row_map): + raise WorkloadError("physical_values seq length mismatch") + for val, key in zip(row_vals, row_map): + if key is None: + continue + if key in out: + raise WorkloadError(f"duplicate logical key {key}") + out[key] = val + return out + + +def permute_batch(batch: LogicalBatch, permutation: Sequence[int]) -> LogicalBatch: + n = len(batch.samples) + if sorted(permutation) != list(range(n)): + raise WorkloadError("permutation must be a permutation of sample indices") + samples = tuple(batch.samples[i] for i in permutation) + return LogicalBatch( + workload_id=batch.workload_id, + seed=batch.seed, + samples=samples, + cell_id=batch.cell_id, + ) + + +def batch_permutation_from_manifest(manifest: WS1Manifest | None = None) -> tuple[int, ...]: + m = manifest if manifest is not None else load_manifest() + perm = m.primary_matrix["batch_permutation"] + return tuple(int(x) for x in perm["permutation"]) + + +def singleton_aggregate_plan( + batch: LogicalBatch, + *, + denominator: str = "active_token_count_across_all_samples", +) -> SingletonAggregatePlan: + """N× B=1 schedule over the same multiset as one B=N run.""" + if not batch.samples: + raise WorkloadError("empty batch") + run_ids = tuple((s.sample_id,) for s in batch.samples) + return SingletonAggregatePlan( + sample_ids=batch.sample_ids, + run_sample_ids=run_ids, + aggregation_order=batch.sample_ids, + denominator=denominator, + token_multiset=batch.token_multiset(active_only=True), + ) + + +def same_logical_multiset(a: LogicalBatch, b: LogicalBatch, *, active_only: bool = True) -> bool: + return a.token_multiset(active_only=active_only) == b.token_multiset(active_only=active_only) + + +def profile_required_nodes( + manifest: WS1Manifest | None = None, profile_id: str = "cuda_bf16" +) -> list[dict[str, Any]]: + m = manifest if manifest is not None else load_manifest() + if profile_id not in m.backend_profiles: + raise WorkloadError(f"unknown profile_id {profile_id!r}") + return [dict(n) for n in m.backend_profiles[profile_id]["required_nodes"]] + + +def profile_missing_required_nodes( + manifest: WS1Manifest | None = None, profile_id: str = "triton_cuda_bf16" +) -> list[str]: + nodes = profile_required_nodes(manifest, profile_id) + return [n["node"] for n in nodes if n.get("status") == "missing_required"] + + +def get_case(manifest: WS1Manifest | None = None, case_id: str = "") -> dict[str, Any]: + m = manifest if manifest is not None else load_manifest() + for case in m.representative_cases: + if case["case_id"] == case_id: + return dict(case) + raise WorkloadError(f"unknown case_id {case_id!r}") + + +def case_ids(manifest: WS1Manifest | None = None) -> tuple[str, ...]: + m = manifest if manifest is not None else load_manifest() + return tuple(c["case_id"] for c in m.representative_cases) + + +def assert_no_undeclared_randomness( + *, + declared_rng_sources: Iterable[str], + encountered_rng_sources: Iterable[str], +) -> None: + """Gate helper: any RNG source not declared in the manifest hard-fails.""" + allowed = set(declared_rng_sources) + bad = [s for s in encountered_rng_sources if s not in allowed] + if bad: + raise WorkloadError( + f"undeclared stochastic source(s) {bad}; policy is hard_fail" + ) + + +def fixture_hash( + manifest: WS1Manifest | None = None, + *, + batch: LogicalBatch | None = None, + extra: Mapping[str, Any] | None = None, +) -> str: + """Stable hash of workload identity-defining fields and logical fixtures.""" + m = manifest if manifest is not None else load_manifest() + logical = batch if batch is not None else build_logical_batch(m) + payload = _manifest_identity_payload(m.raw) + payload["selected_logical_batch"] = [ + list(x) for x in logical.token_multiset(active_only=False) + ] + payload["extra"] = dict(extra) if extra else {} + blob = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(blob).hexdigest() + + +def _manifest_identity_payload(raw: Mapping[str, Any]) -> dict[str, Any]: + return {k: raw[k] for k in _REQUIRED_TOP_LEVEL if k != "fixture_identity_sha256"} + + +def manifest_identity_hash(raw: Mapping[str, Any]) -> str: + blob = json.dumps( + _manifest_identity_payload(raw), sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(blob).hexdigest() + + +def _sequence_digest(values: Any) -> str: + blob = json.dumps(values, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(blob).hexdigest() + + +def weight_snapshot_hash(shards: Sequence[Mapping[str, Any]]) -> str: + """Hash canonical filename/SHA-256/size records for all weight shards.""" + records = sorted( + (str(s["filename"]), str(s["sha256"]), int(s["size_bytes"])) for s in shards + ) + blob = "".join(f"{name}\t{digest}\t{size}\n" for name, digest, size in records) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def reference_payload( + manifest: WS1Manifest | None = None, + *, + cell_id: str | None = None, + dtype: str = "bfloat16", +) -> dict[str, Any]: + """Payload emitted by scripts/ws1_reference.py (no full-model forward).""" + m = manifest if manifest is not None else load_manifest() + if dtype not in {"bfloat16", "bf16", "float32", "fp32"}: + raise WorkloadError(f"unsupported dtype {dtype!r}") + norm_dtype = "bfloat16" if dtype in {"bfloat16", "bf16"} else "float32" + batch = build_logical_batch(m, cell_id=cell_id) + cell = get_matrix_cell(m, cell_id) if cell_id else None + plan = singleton_aggregate_plan(batch) + chunk = chunk_plan_from_manifest(m) + chunked = apply_chunking(batch, chunk_size=chunk.chunk_size) + packed = apply_packing(batch) + padded_left = apply_padding(batch, pad_side="left", manifest=m) + padded_right = apply_padding(batch, pad_side="right", manifest=m) + return { + "workload_id": m.workload_id, + "seed": m.seed, + "dtype": norm_dtype, + "fixture_hash": fixture_hash(m, batch=batch), + "clip_interval": list(m.clip_interval), + "model_id": m.model_identity["model_id"], + "revision": m.model_identity["revision"], + "config_fingerprint": m.model_identity["config_fingerprint"], + "weight_snapshot": m.model_identity["weight_snapshot"], + "cell_id": cell_id, + "cell": cell, + "sample_ids": list(batch.sample_ids), + "active_token_count": batch.active_token_count(), + "singleton_aggregate": { + "aggregation_order": list(plan.aggregation_order), + "denominator": plan.denominator, + "num_runs": len(plan.run_sample_ids), + "token_multiset_len": len(plan.token_multiset), + }, + "chunk_plan": { + "seq_len": chunk.seq_len, + "chunk_size": chunk.chunk_size, + "num_chunks": chunk.num_chunks, + "chunk_spans": [list(s) for s in chunk.chunk_spans], + }, + "backend_profiles": list(m.backend_profiles.keys()), + "case_ids": list(case_ids(m)), + "profile_missing_required": { + pid: profile_missing_required_nodes(m, pid) for pid in m.backend_profiles + }, + "reference_outputs": { + "logical_token_ids_sha256": _sequence_digest( + [list(s.token_ids) for s in batch.samples] + ), + "logical_loss_mask_sha256": _sequence_digest( + [[int(t.is_active) for t in s.tokens()] for s in batch.samples] + ), + "padded_left_sha256": _sequence_digest( + [padded_left.physical_token_ids, padded_left.physical_attention_mask, + padded_left.physical_loss_mask, padded_left.physical_position_ids] + ), + "padded_right_sha256": _sequence_digest( + [padded_right.physical_token_ids, padded_right.physical_attention_mask, + padded_right.physical_loss_mask, padded_right.physical_position_ids] + ), + "chunked_sha256": _sequence_digest( + [chunked.physical_token_ids, chunked.physical_loss_mask, + chunked.restore_map, chunked.segment_offsets, chunked.segment_lengths] + ), + "packed_sha256": _sequence_digest( + [packed.physical_token_ids, packed.physical_loss_mask, + packed.restore_map, packed.segment_offsets, packed.segment_lengths] + ), + "short_fixture_sha256": _sequence_digest( + m.fixtures["short_full_model_fixture"] + ), + "long_fixture_sha256": _sequence_digest( + m.fixtures["long_full_model_fixture"] + ), + }, + } diff --git a/scripts/ws1_reference.py b/scripts/ws1_reference.py new file mode 100755 index 00000000..79af5b97 --- /dev/null +++ b/scripts/ws1_reference.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Emit WS1 C2 (#268) workload reference identity (no full-model forward). + +Example: + python scripts/ws1_reference.py --dtype bf16 --cell-id BN/full + python scripts/ws1_reference.py --emit-json - +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path + + +def _ensure_repo_on_path() -> None: + repo_root = Path(__file__).resolve().parents[1] + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + + +def _load_workload_module(): + """Load the pure-Python C2 module without importing torch-heavy package helpers.""" + module_path = Path(__file__).resolve().parents[1] / "rl_engine/testing/ws1_workload.py" + spec = importlib.util.spec_from_file_location("_ws1_workload_cli", module_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load workload module at {module_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Emit the pinned WS1 canonical workload reference payload: " + "workload_id, seed, dtype, fixture hash, model identity, and matrix cell." + ) + ) + parser.add_argument( + "--manifest", + type=Path, + default=None, + help="Optional path to ws1_manifest.json (default: package manifest).", + ) + parser.add_argument( + "--workload-id", + default=None, + help="If set, must match the manifest workload_id.", + ) + parser.add_argument( + "--seed", + type=int, + default=None, + help="If set, must match the manifest seed (does not reseed fixtures).", + ) + parser.add_argument( + "--dtype", + default="bf16", + help="Execution dtype label for the emission (bf16/bfloat16 or fp32/float32).", + ) + parser.add_argument( + "--cell-id", + default=None, + help="Optional primary matrix cell_id (e.g. BN/full).", + ) + parser.add_argument( + "--emit-json", + default=None, + metavar="PATH", + help="Write full JSON payload to PATH, or '-' for stdout only JSON.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + _ensure_repo_on_path() + workload = _load_workload_module() + WorkloadError = workload.WorkloadError + + args = build_parser().parse_args(argv) + try: + manifest = workload.load_manifest(args.manifest) + if args.workload_id is not None and args.workload_id != manifest.workload_id: + raise WorkloadError( + f"--workload-id {args.workload_id!r} does not match manifest " + f"{manifest.workload_id!r}" + ) + if args.seed is not None and int(args.seed) != manifest.seed: + raise WorkloadError( + f"--seed {args.seed} does not match manifest seed {manifest.seed}" + ) + payload = workload.reference_payload( + manifest, cell_id=args.cell_id, dtype=args.dtype + ) + except WorkloadError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + if args.emit_json == "-": + json.dump(payload, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + return 0 + + # Human-readable summary (always includes the three required identity fields). + print(f"workload_id: {payload['workload_id']}") + print(f"seed: {payload['seed']}") + print(f"dtype: {payload['dtype']}") + print(f"fixture_hash: {payload['fixture_hash']}") + print(f"model_id: {payload['model_id']}") + print(f"revision: {payload['revision']}") + print(f"clip_interval: {payload['clip_interval']}") + if payload.get("cell_id"): + print(f"cell_id: {payload['cell_id']}") + print(f"active_token_count: {payload['active_token_count']}") + print(f"chunk_spans: {payload['chunk_plan']['chunk_spans']}") + missing = payload["profile_missing_required"] + for profile_id, nodes in missing.items(): + if nodes: + print(f"profile {profile_id} missing_required: {nodes}") + + if args.emit_json: + out_path = Path(args.emit_json) + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, sort_keys=True) + fh.write("\n") + print(f"wrote: {out_path}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_ws1_workload.py b/tests/test_ws1_workload.py new file mode 100644 index 00000000..1333b900 --- /dev/null +++ b/tests/test_ws1_workload.py @@ -0,0 +1,494 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""WS1 C2 (#268) canonical workload / logical identity tests (CPU-only).""" + +from __future__ import annotations + +import json +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +REFERENCE_SCRIPT = REPO_ROOT / "scripts" / "ws1_reference.py" +CONTRACT_PATH = REPO_ROOT / "rl_engine/kernels/gtest/tolerance_contract.json" +OPERATOR_SPECS_PATH = REPO_ROOT / "rl_engine/kernels/gtest/operator_specs.py" + + +def _load_pure_workload_module(): + path = REPO_ROOT / "rl_engine/testing/ws1_workload.py" + spec = importlib.util.spec_from_file_location("_ws1_workload_tests", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +ws1 = _load_pure_workload_module() +WorkloadError = ws1.WorkloadError +WS1Manifest = ws1.WS1Manifest +apply_padding = ws1.apply_padding +apply_chunking = ws1.apply_chunking +apply_packing = ws1.apply_packing +assert_no_undeclared_randomness = ws1.assert_no_undeclared_randomness +batch_permutation_from_manifest = ws1.batch_permutation_from_manifest +build_chunk_plan = ws1.build_chunk_plan +build_logical_batch = ws1.build_logical_batch +case_ids = ws1.case_ids +chunk_plan_from_manifest = ws1.chunk_plan_from_manifest +default_manifest_path = ws1.default_manifest_path +fixture_hash = ws1.fixture_hash +get_case = ws1.get_case +get_matrix_cell = ws1.get_matrix_cell +load_manifest = ws1.load_manifest +matrix_cell_ids = ws1.matrix_cell_ids +permute_batch = ws1.permute_batch +profile_missing_required_nodes = ws1.profile_missing_required_nodes +profile_required_nodes = ws1.profile_required_nodes +reference_payload = ws1.reference_payload +restore_logical_order_from_padded = ws1.restore_logical_order_from_padded +restore_logical_order = ws1.restore_logical_order +same_logical_multiset = ws1.same_logical_multiset +singleton_aggregate_plan = ws1.singleton_aggregate_plan +validate_manifest = ws1.validate_manifest + + +def load_contract(): + return json.loads(CONTRACT_PATH.read_text(encoding="utf-8")) + +REQUIRED_CELLS = { + "B1-singleton_aggregate/full", + "BN/full", + "B1-singleton_aggregate/chunked", + "BN/chunked", +} + + +@pytest.fixture(scope="module") +def manifest(): + return load_manifest() + + +def test_default_manifest_path_exists(): + path = default_manifest_path() + assert path.is_file() + assert path.name == "ws1_manifest.json" + + +def test_manifest_loads_and_validates(manifest): + assert manifest.workload_id.startswith("ws1-qwen3-8b-dense") + assert manifest.seed == 20260812 + validate_manifest(manifest.raw) + + +def test_model_identity_is_full_qwen3_8b(manifest): + fp = manifest.model_identity["config_fingerprint"] + assert fp["num_hidden_layers"] == 36 + assert fp["hidden_size"] == 4096 + assert fp["num_attention_heads"] == 32 + assert fp["num_key_value_heads"] == 8 + assert fp["head_dim"] == 128 + assert fp["vocab_size"] == 151936 + assert fp["intermediate_size"] == 12288 + assert fp["tie_word_embeddings"] is False + assert fp["qk_norm"] is True + assert manifest.model_identity["exit_forbids_architecture_shrink"] is True + weight = manifest.model_identity["weight_snapshot"] + assert weight["total_size_bytes"] > 0 + assert weight["pin_method"] + assert len(weight["shards"]) == 5 + assert weight["content_hash"] == ( + "fc664a19c52c82b6f5ddb33d4fe2723181daeb93a344b16fee6369963e5a13a5" + ) + + +def test_clip_interval_pinned_and_aligns_with_c1(manifest): + assert list(manifest.clip_interval) == [0.8, 1.2] + contract = load_contract() + # C1 stores default_clip_interval under chain_logprob_aggregates. + c1_interval = contract["chain_logprob_aggregates"]["default_clip_interval"] + assert list(c1_interval) == list(manifest.clip_interval) + + +def test_forbidden_comparison_roles_align_with_c1(manifest): + forbidden = set(manifest.chain_semantics["forbidden_comparison_roles"]) + assert "baseline" in forbidden + assert "singleton_aggregate" in forbidden + contract = load_contract() + c1_forbidden = set(contract["comparison_roles"]["forbidden"]) + assert forbidden == c1_forbidden + + +def test_primary_matrix_2x2_and_n(manifest): + assert set(matrix_cell_ids(manifest)) == REQUIRED_CELLS + assert int(manifest.primary_matrix["N"]) > 1 + for cell_id in REQUIRED_CELLS: + cell = get_matrix_cell(manifest, cell_id) + assert "batch_mode" in cell + assert cell["batch_mode"] in {"singleton_aggregate", "batched"} + # Naming boundary: never treat singleton_aggregate as a C1 role field. + assert "comparison_lhs_role" not in cell + assert "comparison_rhs_role" not in cell + + +def test_chunk_plan_multi_chunk_non_divisible(manifest): + plan = chunk_plan_from_manifest(manifest) + assert plan.num_chunks >= 2 + assert plan.seq_len % plan.chunk_size != 0 + # Reconstruct full coverage without overlap. + covered = [] + for start, end in plan.chunk_spans: + covered.extend(range(start, end)) + assert covered == list(range(plan.seq_len)) + + +def test_logical_batch_reproducible_and_hash_stable(manifest): + a = build_logical_batch(manifest) + b = build_logical_batch(manifest) + assert a.sample_ids == b.sample_ids + assert a.token_multiset(active_only=False) == b.token_multiset(active_only=False) + assert fixture_hash(manifest, batch=a) == fixture_hash(manifest, batch=b) + assert len(fixture_hash(manifest)) == 64 + + +def test_fixture_hash_covers_all_manifest_identity_fields(manifest): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + original = fixture_hash(manifest) + raw["fixtures"]["loss_mask"]["prompt_tokens_active"] = True + changed = WS1Manifest(raw=raw, path=default_manifest_path()) + assert fixture_hash(changed) != original + + +def test_same_workload_id_rejects_unversioned_manifest_change(): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + raw["chain_semantics"]["temperature"] = 0.5 + with pytest.raises(WorkloadError, match="fixture_identity_sha256"): + validate_manifest(raw) + + +def test_short_long_and_varlen_fixtures_are_materialized(manifest): + fixtures = manifest.fixtures + for name in ("short_full_model_fixture", "long_full_model_fixture"): + fixture = fixtures[name] + assert len(fixture["token_ids"]) == fixture["seq_len"] + assert fixture["candidate_case_ids"] + batch = build_logical_batch(manifest) + assert [sample.seq_len for sample in batch.samples] == fixtures["varlen_seq_lens"] + assert fixtures["prompt_lens"] == [sample.prompt_len for sample in batch.samples] + assert fixtures["completion_lens"] == [ + sample.seq_len - sample.prompt_len for sample in batch.samples + ] + assert fixtures["max_completion_len"] == max(fixtures["completion_lens"]) + assert "primary_completion_len" not in fixtures + + +def test_chain_semantics_report_and_actual_boundaries(manifest): + sem = manifest.chain_semantics + assert "tolerance_contract.json" in sem["tf32_policy_ref"] + assert set(sem["report_naming"]["forbidden_in_reports"]) >= { + "baseline", + "singleton_aggregate", + } + assert ( + sem["report_naming"]["singleton_aggregate_is"] + == "c2_execution_aggregation_mode_only" + ) + assert ( + sem["backend_actual_semantics"]["c2_actual_backend_id"] + == "registry_resolved_expected_candidate" + ) + assert "C8" in sem["backend_actual_semantics"]["runtime_observed_actual_owner"] + boundary = manifest.raw["provenance_boundary"] + assert "full_model_forward" in boundary["not_in_c2"] + assert "runtime_kernel_dispatch_observation" in boundary["not_in_c2"] + + +def test_stale_primary_completion_len_rejected(): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + raw["fixtures"]["primary_completion_len"] = 8 + with pytest.raises(WorkloadError, match="primary_completion_len is forbidden"): + validate_manifest(raw) + + +def test_active_tokens_are_completion_only(manifest): + batch = build_logical_batch(manifest) + for sample in batch.samples: + for tok in sample.tokens(): + if tok.token_position < sample.prompt_len: + assert not tok.is_active + else: + assert tok.is_active + assert batch.active_token_count() == sum( + sample.seq_len - sample.prompt_len for sample in batch.samples + ) + + +@pytest.mark.parametrize("pad_side", ["right", "left"]) +def test_padding_restores_logical_identity(manifest, pad_side): + batch = build_logical_batch(manifest) + padded = apply_padding(batch, pad_side=pad_side, manifest=manifest) + # Physical values encode a unique marker per logical key. + physical_values = [] + for row_map in padded.restore_map: + row = [] + for key in row_map: + if key is None: + row.append(None) + else: + row.append(f"{key[0]}@{key[1]}") + physical_values.append(row) + restored = restore_logical_order_from_padded(padded, physical_values) + expected_keys = set(batch.logical_keys(active_only=False)) + assert set(restored.keys()) == expected_keys + for sample in batch.samples: + for pos in range(sample.seq_len): + assert restored[(sample.sample_id, pos)] == f"{sample.sample_id}@{pos}" + + +def test_batch_permutation_restores_multiset(manifest): + batch = build_logical_batch(manifest) + perm = batch_permutation_from_manifest(manifest) + permuted = permute_batch(batch, perm) + assert permuted.sample_ids != batch.sample_ids + # Multiset equality is order-sensitive in token_multiset (fixed order). + # After sorting by sample_id, the pairs must match. + def sorted_multiset(b): + return tuple(sorted(b.token_multiset(active_only=True))) + + assert sorted_multiset(batch) == sorted_multiset(permuted) + # Restoring original order via inverse permutation. + inverse = [0] * len(perm) + for new_i, old_i in enumerate(perm): + inverse[old_i] = new_i + # samples in permuted are batch.samples[perm[i]]; map back: + restored_samples = [] + for old_i in range(len(batch.samples)): + # find which permuted index holds original old_i + new_i = list(perm).index(old_i) + restored_samples.append(permuted.samples[new_i]) + restored = ws1.LogicalBatch( + workload_id=batch.workload_id, + seed=batch.seed, + samples=tuple(restored_samples), + ) + assert same_logical_multiset(batch, restored) + + +def test_singleton_aggregate_matches_bn_multiset(manifest): + bn = build_logical_batch(manifest, cell_id="BN/full") + plan = singleton_aggregate_plan(bn) + assert plan.sample_ids == bn.sample_ids + assert len(plan.run_sample_ids) == len(bn.samples) + assert all(len(run) == 1 for run in plan.run_sample_ids) + # Rebuild B1 runs and concatenate multiset in fixed order. + combined = [] + for (sid,) in plan.run_sample_ids: + run = build_logical_batch(manifest, sample_ids=[sid]) + combined.extend(run.token_multiset(active_only=True)) + assert tuple(combined) == bn.token_multiset(active_only=True) + assert tuple(combined) == plan.token_multiset + + +def test_chunk_positions_cover_logical_keys(manifest): + batch = build_logical_batch(manifest) + chunk_size = manifest.primary_matrix["chunk"]["chunk_size_tokens"] + for sample in batch.samples: + plan = build_chunk_plan(sample.seq_len, chunk_size) + keys = [] + for start, end in plan.chunk_spans: + for pos in range(start, end): + keys.append((sample.sample_id, pos)) + expected = [(sample.sample_id, pos) for pos in range(sample.seq_len)] + assert keys == expected + + +def test_chunk_and_pack_layouts_restore_identity(manifest): + batch = build_logical_batch(manifest) + chunked = apply_chunking(batch, chunk_size=7) + packed = apply_packing(batch) + for layout in (chunked, packed): + values = [f"{sid}@{pos}" for sid, pos in layout.restore_map] + restored = restore_logical_order(layout, values) + assert set(restored) == set(batch.logical_keys()) + assert len(layout.physical_token_ids) == len(layout.restore_map) + assert chunked.segment_lengths[-1] == 5 + assert packed.segment_lengths == (11, 16, 13, 19) + + +def test_stochastic_policy_hard_fails_undeclared_rng(manifest): + policy = manifest.raw["stochastic_policy"] + assert policy["dropout"] == 0.0 + assert policy["sampling_in_logprob_parity"] is False + assert policy["undeclared_randomness"] == "hard_fail" + declared = {policy["rng_source"]} + assert_no_undeclared_randomness( + declared_rng_sources=declared, + encountered_rng_sources=[policy["rng_source"]], + ) + with pytest.raises(WorkloadError, match="undeclared stochastic"): + assert_no_undeclared_randomness( + declared_rng_sources=declared, + encountered_rng_sources=["torch.randn_unseeded"], + ) + + +def test_backend_profiles_enumerate_required_nodes(manifest): + for profile_id in ("cuda_bf16", "triton_cuda_bf16"): + nodes = profile_required_nodes(manifest, profile_id) + names = {n["node"] for n in nodes} + for required in ( + "embedding", + "rms_norm", + "det_gemm", + "attention", + "rope", + "swiglu", + "lm_head", + "logprob", + "batch_invariant_logp", + ): + assert required in names + for node in nodes: + assert node["status"] in {"declared", "missing_required"} + if node["status"] == "declared": + assert node["expected_backend_id"] + assert node["expected_kernel_config_id"] + assert node["algorithm_property"] + + +def test_triton_profile_records_missing_required_not_na(manifest): + missing = profile_missing_required_nodes(manifest, "triton_cuda_bf16") + # Honest red nodes based on current operator_specs candidates. + assert "embedding" in missing + assert "lm_head" in missing + for node in profile_required_nodes(manifest, "triton_cuda_bf16"): + if node["node"] in missing: + assert node["status"] == "missing_required" + assert node.get("expected_backend_id") in (None, "") + + +def test_representative_cases_stable_ids_and_pins(manifest): + ids = case_ids(manifest) + assert len(ids) == len(set(ids)) + families = {get_case(manifest, cid)["family"] for cid in ids} + assert {"gemm", "attention", "logprob"} <= families + for cid in ids: + case = get_case(manifest, cid) + assert case["architecture_identity"] == "full_qwen3_8b_dense" + assert case["expected_backend_id"] == case["actual_backend_id"] + assert case["expected_kernel_config_id"] == case["actual_kernel_config_id"] + assert case["provenance_status"] == "registry_resolved_runtime_pending" + assert case["provenance_evidence"]["resolved_path"] == case["actual_kernel_config_id"] + assert case["algorithm_property"] + assert "shape" in case + for profile in ("cuda_bf16", "triton_cuda_bf16"): + cases = [ + get_case(manifest, cid) + for cid in ids + if profile in get_case(manifest, cid)["profile_ids"] + ] + assert {c["family"] for c in cases} == {"gemm", "attention", "logprob"} + assert len({c["shape"]["M"] for c in cases if c["family"] == "gemm"}) >= 2 + attention_modes = { + c["shape"]["mode"] for c in cases if c["family"] == "attention" + } + assert attention_modes == {"prefill", "decode"} + + +def test_declared_candidates_resolve_to_real_operator_specs(manifest): + source = OPERATOR_SPECS_PATH.read_text(encoding="utf-8") + spec_map = manifest.raw["capabilities"]["operator_spec_map"] + for node, spec_name in spec_map.items(): + assert f'"{spec_name}": OperatorSpec(' in source, node + for case in manifest.representative_cases: + evidence = case["provenance_evidence"] + resolved_class = evidence["resolved_path"].rsplit(".", 1)[1] + assert resolved_class in source + assert f'"{evidence["candidate_name"]}"' in source + algorithm_path, line = evidence["algorithm_source"].rsplit(":", 1) + algorithm_file = REPO_ROOT / algorithm_path + assert algorithm_file.is_file() + assert 1 <= int(line) <= len(algorithm_file.read_text(encoding="utf-8").splitlines()) + + +def test_capabilities_packing_and_qk_norm(manifest): + caps = manifest.raw["capabilities"] + assert caps["qk_norm"]["status"] == "required_on_chain" + packing = caps["packing"] + assert packing["status"] == "supported" + assert manifest.fixtures["packing"]["packed_fixture"]["total_tokens"] == 59 + + +def test_missing_weight_hash_rejected(): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + del raw["model_identity"]["weight_snapshot"]["content_hash"] + with pytest.raises(WorkloadError, match="content_hash"): + validate_manifest(raw) + + +def test_packing_cannot_be_marked_na_when_supported(): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + raw["fixtures"]["packing"]["status"] = "n_a_with_capability_proof" + with pytest.raises(WorkloadError, match="must pin a supported packed fixture"): + validate_manifest(raw) + + +def test_architecture_shrink_rejected(): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + raw["model_identity"]["config_fingerprint"]["num_hidden_layers"] = 2 + with pytest.raises(WorkloadError, match="does not match official"): + validate_manifest(raw) + + +def test_missing_matrix_cell_rejected(): + raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) + raw["primary_matrix"]["cells"] = raw["primary_matrix"]["cells"][:3] + with pytest.raises(WorkloadError, match="primary_matrix.cells"): + validate_manifest(raw) + + +def test_reference_payload_contains_required_fields(manifest): + payload = reference_payload(manifest, cell_id="BN/full", dtype="bf16") + assert payload["workload_id"] == manifest.workload_id + assert payload["seed"] == manifest.seed + assert payload["dtype"] == "bfloat16" + assert payload["fixture_hash"] == fixture_hash(manifest) + assert payload["cell_id"] == "BN/full" + assert payload["clip_interval"] == [0.8, 1.2] + + +def test_ws1_reference_cli_emits_identity(): + proc = subprocess.run( + [ + sys.executable, + str(REFERENCE_SCRIPT), + "--dtype", + "bf16", + "--cell-id", + "BN/full", + "--emit-json", + "-", + ], + check=False, + capture_output=True, + text=True, + cwd=str(REPO_ROOT), + ) + assert proc.returncode == 0, proc.stderr + payload = json.loads(proc.stdout) + assert "workload_id" in payload + assert "seed" in payload + assert payload["dtype"] == "bfloat16" + assert len(payload["fixture_hash"]) == 64 + + +def test_build_chunk_plan_edges(): + plan = build_chunk_plan(16, 7) + assert plan.chunk_spans == ((0, 7), (7, 14), (14, 16)) + with pytest.raises(WorkloadError): + build_chunk_plan(8, 0) From fca16563e48c06a283a2e96c38dbe8f67cb22c7b Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 15:28:52 +0800 Subject: [PATCH 06/10] fix(ws1): address PR 292 review feedback --- docs/contributing/testing.md | 3 +- docs/design/ws1-c2-268-closeout-evidence.md | 24 +- docs/design/ws1-c2-268-workload-plan.md | 14 +- rl_engine/kernels/gtest/op_checks.py | 64 ++--- rl_engine/kernels/gtest/tolerance.py | 34 ++- .../kernels/gtest/tolerance_contract.json | 3 + rl_engine/testing/__init__.py | 4 + rl_engine/testing/ws1_manifest.json | 221 +++++++++-------- rl_engine/testing/ws1_workload.py | 232 ++++++++++++------ scripts/ws1_candidate_evidence.py | 196 +++++++++++++++ scripts/ws1_reference.py | 15 +- tests/test_op_checks.py | 56 +++++ tests/test_tolerance_contract.py | 21 +- tests/test_ws1_candidate_evidence.py | 43 ++++ tests/test_ws1_workload.py | 87 +++++-- 15 files changed, 760 insertions(+), 257 deletions(-) create mode 100755 scripts/ws1_candidate_evidence.py create mode 100644 tests/test_ws1_candidate_evidence.py diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 749b203e..967298bc 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -7,7 +7,8 @@ RL-Kernel uses focused tests for dispatch behavior and operator accuracy. Primary entry for single-operator forward/backward checks against a PyTorch gold path: ```bash -python scripts/check_operator.py --op logp --candidate pytorch --device cpu --dtype fp32 +python scripts/check_operator.py --op logp --candidate pytorch --device cpu --dtype fp32 \ + --batch 1 --seq 2 --vocab 17 ``` Full usage (register `OP_SPECS`, build inputs, CLI flags, and the WS1 four-judgment diff --git a/docs/design/ws1-c2-268-closeout-evidence.md b/docs/design/ws1-c2-268-closeout-evidence.md index 72bcd5b4..82388d91 100644 --- a/docs/design/ws1-c2-268-closeout-evidence.md +++ b/docs/design/ws1-c2-268-closeout-evidence.md @@ -1,6 +1,7 @@ # WS1 C2 (#268) Closeout Evidence -**Issue:** #268 · **Parent:** #266 · **Workload:** `ws1-qwen3-8b-dense-primary-v3` +**Issue:** #268 · **Parent:** #266 · **Workload:** `ws1-qwen3-8b-dense-primary-v4` + **Branch:** `feat/ws1-c2-canonical-workload-268` ## Deliverables @@ -10,6 +11,7 @@ | `rl_engine/testing/ws1_manifest.json` | SSOT workload identity / matrix / profiles / cases | | `rl_engine/testing/ws1_workload.py` | Load, validate, logical identity, pad/pack/chunk restore | | `scripts/ws1_reference.py` | One-command reference emission | +| `scripts/ws1_candidate_evidence.py` | Executable CUDA/Triton candidate provenance | | `tests/test_ws1_workload.py` | CPU acceptance tests | | `docs/design/ws1-c2-268-workload-plan.md` | Landing plan | | `docs/design/ws1-c2-268-closeout-evidence.md` | This map | @@ -27,14 +29,14 @@ | 2×2 + perm + multi-chunk non-divisible + pad/varlen | **Pass** | primary matrix + varlen samples `[11,16,13,19]` | | clip_interval for clipfrac0 | **Pass** | `[0.8, 1.2]` aligned with C1 | | Dropout/sampling/RNG policy; undeclared hard-fail | **Pass** | `stochastic_policy` + helper test | -| Short + representative fixtures hit declared candidates | **Pass\*** | fixture `candidate_case_ids` + registry resolution tests | +| Short + representative fixtures hit declared candidates | **Pass** | fixture-derived shapes + runtime candidate evidence runner | | Stable case_id for C8/C10/C11 reference | **Pass** | `representative_cases[].case_id` | -| expected + actual backend/kernel + algorithm property | **Pass\*** | registry-resolved actual; runtime observation owned by C8+ (declared in manifest) | +| expected + actual backend/kernel + algorithm property | **Pass** | runner executes each case, records actual class path, compares it to expected, and checks outputs | | One command emits reference (workload ID, seed, dtype) | **Pass** | `scripts/ws1_reference.py` | | Packing / QK-Norm / required ops status | **Pass** | packing supported + packed fixture; qk_norm required | | Both profiles enumerate required nodes; no untracked missing | **Pass** | Triton gaps are `missing_required` (red, tracked) | -\*C2 binds **registry-resolved** candidate paths. Live GPU dispatch observation is explicitly out of C2 (`provenance_boundary`) and owned by C3/C8/C10/C11. +C2 executes all representative cases. Full-model dispatch provenance remains owned by C3/C8/C10/C11; this does not claim the C9/C10 full-model gate. ## Verification commands @@ -42,15 +44,25 @@ # From repo root with PYTHONPATH=repo root (or editable install) python -m pytest tests/test_ws1_workload.py -q python scripts/ws1_reference.py --dtype bf16 --cell-id BN/full --emit-json - +python scripts/ws1_candidate_evidence.py --emit-json ws1-c2-runtime-evidence.json ``` -Expected: all C2 tests green; CLI prints `workload_id`, `seed`, `dtype`, `fixture_hash`, and `reference_outputs` digests. +Expected: all C2 tests green; the reference CLI emits identity/digests; the candidate runner executes all CUDA/Triton cases and reports `passed: true` with runtime-observed actual paths. + +Validated on 2026-08-12: + +- NVIDIA GeForce RTX 3060 Laptop GPU, SM86, single GPU +- PyTorch `2.8.0+cu128`, CUDA runtime `12.8`, Triton `3.4.0`, Python `3.13.3` +- Representative runtime evidence: 10/10 CUDA + Triton cases passed +- Focused review/workload/contract suite: 81 passed (including CUDA/Triton runtime evidence) +- Full repository CUDA/Triton pytest: 1622 tests collected; exit code 0 (1501 passed, 121 hardware/CI skips) +- `pre-commit run --all-files`: all 7 hooks passed ## Residual (explicitly not #268) | Item | Owner | | --- | --- | -| Runtime observed actual backend on GPU | C3 / C8 / C10 / C11 | +| Full-model runtime observed actual backend | C3 / C8 / C10 / C11 | | Triton `missing_required`: embedding, lm_head, logprob | later candidate work / Blocker; tracked red in C2 | | #150 numerical asserts / full-model e2e | C9 / C10 | | Full WS1 EXIT | #266 after C1–C11 | diff --git a/docs/design/ws1-c2-268-workload-plan.md b/docs/design/ws1-c2-268-workload-plan.md index e05a6f4a..af56904e 100644 --- a/docs/design/ws1-c2-268-workload-plan.md +++ b/docs/design/ws1-c2-268-workload-plan.md @@ -1,7 +1,9 @@ # WS1 C2 (#268) Landing Plan — Canonical Workload & Logical Identity -**Parent:** #266 · **Issue:** #268 · **Depends on:** C1 (#267) contract roles only -**Branch:** `feat/ws1-c2-canonical-workload-268` (from `feat/ws1-c1-tolerance-contract-267@81ddd65`) +**Parent:** #266 · **Issue:** #268 · **Depends on:** C1 (#267) contract roles only + +**Branch:** `feat/ws1-c2-canonical-workload-268` (from `feat/ws1-c1-tolerance-contract-267@81ddd65`) + **Does not modify:** C1 branch tip --- @@ -140,13 +142,13 @@ Changing any pinned field → new `case_id` / revision. ```text load_manifest() / validate_manifest() -build_logical_batch(workload_id) -> LogicalBatch +build_logical_batch(manifest=None, *, cell_id=None, sample_ids=None) -> LogicalBatch samples: list[LogicalSample] # sample_id, token_ids, positions, loss_mask, ... apply_padding(batch) / apply_chunking(batch) / apply_packing(batch) -> physical layout restore_logical_order(physical, values) -> aligned values keyed by (sample_id, token_position) singleton_aggregate_plan(N samples) -> execution schedule for B1×N vs BN fixture_hash(batch|manifest) -> stable hex -matrix_cells() / get_cell(cell_id) +matrix_cell_ids() / get_matrix_cell(manifest, cell_id) profile_required_nodes(profile_id) get_case(case_id) ``` @@ -171,7 +173,7 @@ python scripts/ws1_reference.py \ ``` Emits: workload_id, seed, dtype, fixture_hash, model identity pins, cell descriptor, -clip_interval, profile ids, and deterministic logical/pad/chunk/pack/short/long tensor digests. +clip_interval, profile ids, and deterministic logical/pad/chunk/pack/short/long tensor digests. Does **not** run full 8B forward (owned by C9/C10); may emit tensor fixture digests for token/mask tensors only. --- @@ -261,5 +263,5 @@ CPU-only; no GPU / no weight download required for C2 unit tests. - [x] `pytest tests/test_ws1_workload.py -q` green — 33 passed (CPU). - [x] `python scripts/ws1_reference.py` emits workload_id / seed / dtype / fixture digests. - [x] Closeout evidence: `docs/design/ws1-c2-268-closeout-evidence.md`. -- [x] #268 residual closeout: per-sample `completion_lens`, TF32 ref, report naming, registry-vs-runtime actual boundary. +- [x] #268 residual closeout: per-sample `completion_lens`, TF32 ref, report naming, fixture-derived representative shapes, and executable CUDA/Triton actual provenance. - [x] Explicit non-claim: does not close #266 or turn Triton missing_required green. diff --git a/rl_engine/kernels/gtest/op_checks.py b/rl_engine/kernels/gtest/op_checks.py index 1bada7b3..e9354cb6 100644 --- a/rl_engine/kernels/gtest/op_checks.py +++ b/rl_engine/kernels/gtest/op_checks.py @@ -9,9 +9,9 @@ import torch +from rl_engine.kernels.gtest.tolerance import BackendProvenance, ContractResolveError +from rl_engine.kernels.gtest.tolerance import _dtype_name as _normalize_dtype_name from rl_engine.kernels.gtest.tolerance import ( - BackendProvenance, - ContractResolveError, load_contract, resolve_tolerance, validate_backend_provenance, @@ -159,7 +159,9 @@ def _run_candidate( f"{candidate.provenance.actual_backend!r}" ) for case in cases: - if _dtype_name(case.dtype) != candidate.provenance.execution_dtype: + case_dtype = _normalize_dtype_name(case.dtype) + provenance_dtype = _normalize_dtype_name(candidate.provenance.execution_dtype) + if case_dtype != provenance_dtype: raise ContractResolveError( f"case {case.name!r} dtype {case.dtype} does not match " f"provenance execution_dtype {candidate.provenance.execution_dtype!r}" @@ -243,16 +245,8 @@ def _run_case_backward( ).outputs # Gradient thresholds come from the independent gradient_accuracy judgment # (#267); they must not silently inherit forward_accuracy rows. - atol, rtol = _resolve_tolerance( - contract, - op_class=case.op_class, - dtype=case.dtype, - arch_key=candidate.arch_key, - backend_profile=(candidate.provenance.backend_profile if candidate.provenance else None), - judgment="gradient_accuracy", - ) - gradient_spec = ( - resolve_tolerance( + if "judgments" in contract: + gradient_spec = resolve_tolerance( contract, judgment="gradient_accuracy", op_class=case.op_class, @@ -262,9 +256,19 @@ def _run_case_backward( candidate.provenance.backend_profile if candidate.provenance else None ), ) - if "judgments" in contract - else None - ) + atol, rtol = gradient_spec.atol, gradient_spec.rtol + else: + gradient_spec = None + atol, rtol = _resolve_tolerance( + contract, + op_class=case.op_class, + dtype=case.dtype, + arch_key=candidate.arch_key, + backend_profile=( + candidate.provenance.backend_profile if candidate.provenance else None + ), + judgment="gradient_accuracy", + ) grad_checks = [ _compare_output( candidate_grad, @@ -307,16 +311,8 @@ def _compare_case_outputs( f"candidate {candidate.name!r} returned {len(candidate_outputs)} outputs, " f"gold returned {len(gold_outputs)}" ) - atol, rtol = _resolve_tolerance( - contract, - op_class=case.op_class, - dtype=case.dtype, - arch_key=candidate.arch_key, - backend_profile=(candidate.provenance.backend_profile if candidate.provenance else None), - judgment="forward_accuracy", - ) - forward_spec = ( - resolve_tolerance( + if "judgments" in contract: + forward_spec = resolve_tolerance( contract, judgment="forward_accuracy", op_class=case.op_class, @@ -326,9 +322,19 @@ def _compare_case_outputs( candidate.provenance.backend_profile if candidate.provenance else None ), ) - if "judgments" in contract - else None - ) + atol, rtol = forward_spec.atol, forward_spec.rtol + else: + forward_spec = None + atol, rtol = _resolve_tolerance( + contract, + op_class=case.op_class, + dtype=case.dtype, + arch_key=candidate.arch_key, + backend_profile=( + candidate.provenance.backend_profile if candidate.provenance else None + ), + judgment="forward_accuracy", + ) if candidate.provenance is not None: for candidate_output, gold_output in zip(candidate_outputs, gold_outputs, strict=True): candidate_dtype = _dtype_name(candidate_output.dtype) diff --git a/rl_engine/kernels/gtest/tolerance.py b/rl_engine/kernels/gtest/tolerance.py index 0d2ae2e7..d9afdfae 100644 --- a/rl_engine/kernels/gtest/tolerance.py +++ b/rl_engine/kernels/gtest/tolerance.py @@ -259,7 +259,12 @@ def validate_backend_provenance( policy = resolve_dtype_policy(contract) if provenance.backend_profile not in policy.backend_profiles: raise ContractResolveError(f"unknown backend_profile {provenance.backend_profile!r}") - profile_contract = contract["policy"]["backend_profile_contracts"][provenance.backend_profile] + profile_contracts = contract["policy"]["backend_profile_contracts"] + if provenance.backend_profile not in profile_contracts: + raise ContractResolveError( + f"missing backend_profile_contracts entry for {provenance.backend_profile!r}" + ) + profile_contract = profile_contracts[provenance.backend_profile] expected_backend = str(profile_contract["backend_family"]) for field_name, actual in ( ("requested_backend", provenance.requested_backend), @@ -922,6 +927,14 @@ def _dtype_name(dtype: str | Any) -> str: "torch.bfloat16": "bfloat16", "torch.float16": "float16", "torch.float8": "float8", + "torch.float8_e4m3fn": "float8", + "torch.float8_e5m2": "float8", + "torch.float8_e4m3fnuz": "float8", + "torch.float8_e5m2fnuz": "float8", + "float8_e4m3fn": "float8", + "float8_e5m2": "float8", + "float8_e4m3fnuz": "float8", + "float8_e5m2fnuz": "float8", "fp32": "float32", "bf16": "bfloat16", "fp16": "float16", @@ -940,9 +953,19 @@ def _dtype_name(dtype: str | Any) -> str: "torch.float32": "float32", "torch.bfloat16": "bfloat16", "torch.float16": "float16", + "torch.float8": "float8", + "torch.float8_e4m3fn": "float8", + "torch.float8_e5m2": "float8", + "torch.float8_e4m3fnuz": "float8", + "torch.float8_e5m2fnuz": "float8", "float32": "float32", "bfloat16": "bfloat16", "float16": "float16", + "float8": "float8", + "float8_e4m3fn": "float8", + "float8_e5m2": "float8", + "float8_e4m3fnuz": "float8", + "float8_e5m2fnuz": "float8", } # torch.dtype str is like "torch.float32" as_str = str(dtype) @@ -959,6 +982,15 @@ def _dtype_name(dtype: str | Any) -> str: return "bfloat16" if dtype is torch.float16: return "float16" + for attr in ( + "float8_e4m3fn", + "float8_e5m2", + "float8_e4m3fnuz", + "float8_e5m2fnuz", + ): + torch_dtype = getattr(torch, attr, None) + if torch_dtype is not None and dtype is torch_dtype: + return "float8" except ImportError: # pragma: no cover pass raise ContractResolveError(f"unsupported dtype: {dtype!r}") diff --git a/rl_engine/kernels/gtest/tolerance_contract.json b/rl_engine/kernels/gtest/tolerance_contract.json index e7645b75..8acdae9f 100644 --- a/rl_engine/kernels/gtest/tolerance_contract.json +++ b/rl_engine/kernels/gtest/tolerance_contract.json @@ -126,6 +126,8 @@ }, "gradient_accuracy": { "default_mode": "tolerance", + "calibration_status": "provisional_pending_measured_backward_evidence", + "calibration_note": "C1 keeps gradient rows independent from forward rows. The initial values intentionally match the forward table until per-operator CUDA and Triton backward error distributions are recorded; revise them only from measured evidence.", "by_op_class": { "elementwise": { "float32": {"status": "applicable", "mode": "tolerance", "atol": 1.0e-5, "rtol": 1.0e-5}, @@ -217,6 +219,7 @@ "approx_kl0": { "formula": "mean(exp(dlogp) - 1 - dlogp)", "pass_rule": "value <= threshold", + "threshold_rationale": "The initial C1 thresholds intentionally preserve the same drift scale as max_abs_dlogp for compatibility. max_abs_dlogp is therefore the stricter guard in this version; approx_kl0 remains a required reported metric pending measured chain-level distributions, after which its threshold may be tightened without introducing an unevidenced C2-local value.", "by_execution_dtype": { "bfloat16": {"threshold": 5.0e-2}, "float32": {"threshold": 1.0e-5}, diff --git a/rl_engine/testing/__init__.py b/rl_engine/testing/__init__.py index eff1be01..1d5708ac 100644 --- a/rl_engine/testing/__init__.py +++ b/rl_engine/testing/__init__.py @@ -19,12 +19,14 @@ WorkloadError, WS1Manifest, apply_chunking, + apply_padding, apply_packing, build_logical_batch, fixture_hash, load_manifest, reference_payload, restore_logical_order, + restore_logical_order_from_padded, ) __all__ = [ @@ -35,6 +37,7 @@ "WorkloadError", "active_token_count", "apply_chunking", + "apply_padding", "apply_packing", "build_logical_batch", "compute_policy_ratio", @@ -46,6 +49,7 @@ "masked_sum", "reference_payload", "restore_logical_order", + "restore_logical_order_from_padded", "selected_logprobs_reference", "summarize_kernel_drift", ] diff --git a/rl_engine/testing/ws1_manifest.json b/rl_engine/testing/ws1_manifest.json index b3e69cb9..6691f84f 100644 --- a/rl_engine/testing/ws1_manifest.json +++ b/rl_engine/testing/ws1_manifest.json @@ -1,6 +1,6 @@ { - "version": "ws1-c2-v3", - "workload_id": "ws1-qwen3-8b-dense-primary-v3", + "version": "ws1-c2-v4", + "workload_id": "ws1-qwen3-8b-dense-primary-v4", "seed": 20260812, "model_identity": { "model_id": "Qwen/Qwen3-8B", @@ -108,15 +108,14 @@ "note": "C2 freezes naming rules; C3+ emit reports that must obey these roles." }, "backend_actual_semantics": { - "c2_actual_backend_id": "registry_resolved_expected_candidate", - "c2_actual_kernel_config_id": "operator_specs_candidate_path", - "runtime_observed_actual_owner": [ + "c2_representative_actual_source": "scripts/ws1_candidate_evidence.py runtime execution", + "full_model_runtime_observed_actual_owner": [ "C3", "C8", "C10", "C11" ], - "note": "For C2, actual_* equals expected_* after operator_specs resolution. GPU runtime provenance that proves a live kernel hit is owned by later closeout children; missing required Triton nodes stay status=missing_required (red)." + "note": "C2 executes every representative case and records runtime-observed actual backend/kernel provenance. Later children own full-model dispatch provenance; missing required Triton nodes stay status=missing_required (red)." } }, "stochastic_policy": { @@ -365,8 +364,10 @@ ], "note": "Shorter sequence on full architecture+weights only; never shrinks layers/hidden/heads/vocab.", "candidate_case_ids": [ - "gemm-m127-k4096-n4096-no-splitk-v1", - "logp-vocab151936-btok17-reduction-boundary-v1" + "gemm-short-m8-k4096-n4096-cuda-v2", + "gemm-short-m8-k4096-n4096-triton-v2", + "logp-short-vocab151936-t4-cuda-v2", + "logp-short-vocab151936-t4-triton-v2" ] }, "long_full_model_fixture": { @@ -409,8 +410,8 @@ ], "note": "Long fixed sequence on the same full architecture and pinned weight snapshot.", "candidate_case_ids": [ - "attn-decode-gqa-b1-sq1-skv129-no-splitkv-v1", - "attn-triton-prefill-gqa-b2-sq33-skv33-no-splitkv-v1" + "attn-long-decode-gqa-b1-sq1-skv32-cuda-v2", + "attn-long-decode-gqa-b1-sq1-skv32-triton-v2" ] }, "representative_full_model_fixture": { @@ -425,8 +426,10 @@ ], "note": "Primary variable-length matrix fixture; full architecture+weights.", "candidate_case_ids": [ - "gemm-m256-k4096-n12288-no-splitk-v1", - "logp-bi-triton-vocab151936-btok15-v1" + "gemm-primary-m59-k4096-n12288-cuda-v2", + "gemm-primary-m59-k4096-n12288-triton-v2", + "attn-primary-prefill-gqa-b4-sq19-skv19-cuda-v2", + "attn-primary-prefill-gqa-b4-sq19-skv19-triton-v2" ] }, "prompt_lens": [ @@ -707,319 +710,339 @@ }, "representative_cases": [ { - "case_id": "gemm-m127-k4096-n4096-no-splitk-v1", + "case_id": "gemm-short-m8-k4096-n4096-cuda-v2", "family": "gemm", - "revision": 1, + "revision": 2, + "fixture_id": "short_full_model_seq8", + "operator_spec": "det_gemm", "shape": { - "M": 127, + "M": 8, "K": 4096, "N": 4096, - "note": "Non-tile-aligned flattened-token M on full-model projection K/N." + "note": "Short-fixture flattened-token M; non-tile-aligned on full-model projection K/N." }, "expected_backend_id": "cuda", "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", "actual_backend_id": "cuda", "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_k_deterministic_gemm", "profile_ids": [ "cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "cuda", "resolved_path": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", - "algorithm_source": "rl_engine/kernels/ops/cuda/matmul/det_gemm.py:3", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "csrc/cuda/gemm/det_gemm_kernel.cu:det_gemm_naive", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id gemm-short-m8-k4096-n4096-cuda-v2" } }, { - "case_id": "gemm-m256-k4096-n12288-no-splitk-v1", + "case_id": "gemm-primary-m59-k4096-n12288-cuda-v2", "family": "gemm", - "revision": 1, + "revision": 2, + "fixture_id": "rep_full_model_seq16", + "operator_spec": "det_gemm", "shape": { - "M": 256, + "M": 59, "K": 4096, "N": 12288, - "note": "Gate/up projection width (intermediate_size)." + "note": "Primary varlen fixture total tokens; full gate/up projection width." }, "expected_backend_id": "cuda", "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", "actual_backend_id": "cuda", "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_k_deterministic_gemm", "profile_ids": [ "cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "cuda", "resolved_path": "rl_engine.kernels.ops.cuda.matmul.det_gemm.DetGemmOp", - "algorithm_source": "rl_engine/kernels/ops/cuda/matmul/det_gemm.py:3", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "csrc/cuda/gemm/det_gemm_kernel.cu:det_gemm_naive", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id gemm-primary-m59-k4096-n12288-cuda-v2" } }, { - "case_id": "gemm-triton-m63-k4096-n4096-no-splitk-v1", + "case_id": "gemm-short-m8-k4096-n4096-triton-v2", "family": "gemm", - "revision": 1, + "revision": 2, + "fixture_id": "short_full_model_seq8", + "operator_spec": "det_gemm", "shape": { - "M": 63, + "M": 8, "K": 4096, "N": 4096, - "note": "Non-tile-aligned M on Triton det_gemm path." + "note": "Short-fixture flattened-token M on Triton det_gemm path." }, "expected_backend_id": "triton", "expected_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", "actual_backend_id": "triton", "actual_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_k_deterministic_gemm", "profile_ids": [ "triton_cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "triton", "resolved_path": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", - "algorithm_source": "rl_engine/kernels/ops/triton/matmul/det_gemm.py:3", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "rl_engine/kernels/ops/triton/matmul/det_gemm.py:_det_gemm_kernel", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id gemm-short-m8-k4096-n4096-triton-v2" } }, { - "case_id": "gemm-triton-m256-k4096-n12288-no-splitk-v1", + "case_id": "gemm-primary-m59-k4096-n12288-triton-v2", "family": "gemm", - "revision": 1, + "revision": 2, + "fixture_id": "rep_full_model_seq16", + "operator_spec": "det_gemm", "shape": { - "M": 256, + "M": 59, "K": 4096, "N": 12288, - "note": "Second Triton M and full gate/up projection width." + "note": "Primary varlen fixture total tokens and full gate/up projection width." }, "expected_backend_id": "triton", "expected_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", "actual_backend_id": "triton", "actual_kernel_config_id": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_k_deterministic_gemm", "profile_ids": [ "triton_cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "triton", "resolved_path": "rl_engine.kernels.ops.triton.matmul.det_gemm.TritonDetGemmOp", - "algorithm_source": "rl_engine/kernels/ops/triton/matmul/det_gemm.py:3", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "rl_engine/kernels/ops/triton/matmul/det_gemm.py:_det_gemm_kernel", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id gemm-primary-m59-k4096-n12288-triton-v2" } }, { - "case_id": "attn-prefill-gqa-b2-sq31-skv31-no-splitkv-v1", + "case_id": "attn-primary-prefill-gqa-b4-sq19-skv19-cuda-v2", "family": "attention", - "revision": 1, + "revision": 2, + "fixture_id": "rep_full_model_seq16", + "operator_spec": "attention", "shape": { - "B": 2, + "B": 4, "Hq": 32, "Hkv": 8, - "Sq": 31, - "Skv": 31, + "Sq": 19, + "Skv": 19, "D": 128, "mode": "prefill", - "note": "Non-tile-aligned sequence; GQA as official." + "note": "Primary max-varlen prefill; non-tile-aligned sequence and official GQA." }, "expected_backend_id": "cuda", "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", "actual_backend_id": "cuda", "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_kv_batch_invariant_attention", "profile_ids": [ "cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "cuda", "resolved_path": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", - "algorithm_source": "rl_engine/kernels/ops/cuda/attention/deterministic_attn.py:3", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "csrc/cuda/attention/deterministic_attention.cu:deterministic_attention_forward", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id attn-primary-prefill-gqa-b4-sq19-skv19-cuda-v2" } }, { - "case_id": "attn-decode-gqa-b1-sq1-skv129-no-splitkv-v1", + "case_id": "attn-long-decode-gqa-b1-sq1-skv32-cuda-v2", "family": "attention", - "revision": 1, + "revision": 2, + "fixture_id": "long_full_model_seq32", + "operator_spec": "attention", "shape": { "B": 1, "Hq": 32, "Hkv": 8, "Sq": 1, - "Skv": 129, + "Skv": 32, "D": 128, "mode": "decode", - "note": "Decode step with non-tile-aligned KV length." + "note": "Decode step over the fixed long fixture KV length." }, "expected_backend_id": "cuda", "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", "actual_backend_id": "cuda", "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_kv_batch_invariant_attention", "profile_ids": [ "cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "cuda", "resolved_path": "rl_engine.kernels.ops.cuda.attention.deterministic_attn.DeterministicAttentionOp", - "algorithm_source": "rl_engine/kernels/ops/cuda/attention/deterministic_attn.py:3", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "csrc/cuda/attention/deterministic_attention.cu:deterministic_attention_forward", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id attn-long-decode-gqa-b1-sq1-skv32-cuda-v2" } }, { - "case_id": "attn-triton-prefill-gqa-b2-sq33-skv33-no-splitkv-v1", + "case_id": "attn-primary-prefill-gqa-b4-sq19-skv19-triton-v2", "family": "attention", - "revision": 1, + "revision": 2, + "fixture_id": "rep_full_model_seq16", + "operator_spec": "attention", "shape": { - "B": 2, + "B": 4, "Hq": 32, "Hkv": 8, - "Sq": 33, - "Skv": 33, + "Sq": 19, + "Skv": 19, "D": 128, "mode": "prefill", - "note": "Triton batch-invariant attention; non-power-of-two seq." + "note": "Triton primary max-varlen prefill; non-power-of-two sequence." }, "expected_backend_id": "triton", "expected_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", "actual_backend_id": "triton", "actual_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_kv_batch_invariant_attention", "profile_ids": [ "triton_cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "triton", "resolved_path": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", - "algorithm_source": "rl_engine/kernels/ops/triton/attention/standard_attn.py:1", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "rl_engine/kernels/ops/triton/attention/standard_attn.py:_standard_attn_fwd_kernel", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id attn-primary-prefill-gqa-b4-sq19-skv19-triton-v2" } }, { - "case_id": "attn-triton-decode-gqa-b1-sq1-skv129-no-splitkv-v1", + "case_id": "attn-long-decode-gqa-b1-sq1-skv32-triton-v2", "family": "attention", - "revision": 1, + "revision": 2, + "fixture_id": "long_full_model_seq32", + "operator_spec": "attention", "shape": { "B": 1, "Hq": 32, "Hkv": 8, "Sq": 1, - "Skv": 129, + "Skv": 32, "D": 128, "mode": "decode", - "note": "Triton decode with non-tile-aligned KV length." + "note": "Triton decode over the fixed long fixture KV length." }, "expected_backend_id": "triton", "expected_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", "actual_backend_id": "triton", "actual_kernel_config_id": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "no_split_kv_batch_invariant_attention", "profile_ids": [ "triton_cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "triton", "resolved_path": "rl_engine.kernels.ops.triton.attention.standard_attn.TritonBatchInvariantAttentionOp", - "algorithm_source": "rl_engine/kernels/ops/triton/attention/standard_attn.py:1", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "rl_engine/kernels/ops/triton/attention/standard_attn.py:_standard_attn_fwd_kernel", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id attn-long-decode-gqa-b1-sq1-skv32-triton-v2" } }, { - "case_id": "logp-vocab151936-btok17-reduction-boundary-v1", + "case_id": "logp-short-vocab151936-t4-cuda-v2", "family": "logprob", - "revision": 1, + "revision": 2, + "fixture_id": "short_full_model_seq8", + "operator_spec": "logp", "shape": { "B": 1, - "T": 17, + "T": 4, "vocab": 151936, - "note": "Full vocab; token count crosses common 16-aligned reduction boundary." + "note": "Short-fixture active selected tokens; full vocab crosses the CUDA reduction boundary." }, "expected_backend_id": "cuda", "expected_kernel_config_id": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", "actual_backend_id": "cuda", "actual_kernel_config_id": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "deterministic_selected_logprob", "profile_ids": [ "cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "cuda", "resolved_path": "rl_engine.kernels.ops.cuda.loss.logp.FusedLogpGenericOp", - "algorithm_source": "rl_engine/kernels/ops/cuda/loss/logp.py:213", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "csrc/fused_logp_kernel.cu:fused_logp_forward_kernel", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id logp-short-vocab151936-t4-cuda-v2" } }, { - "case_id": "logp-bi-triton-vocab151936-btok15-v1", + "case_id": "logp-short-vocab151936-t4-triton-v2", "family": "logprob", - "revision": 1, + "revision": 2, + "fixture_id": "short_full_model_seq8", + "operator_spec": "batch_invariant_logp", "shape": { "B": 1, - "T": 15, + "T": 4, "vocab": 151936, - "note": "Triton batch-invariant logp on full vocab; non-aligned T." + "note": "Short-fixture active selected tokens; full vocab crosses Triton BLOCK_V reductions." }, "expected_backend_id": "triton", "expected_kernel_config_id": "rl_engine.kernels.ops.triton.loss.batch_invariant_logp.TritonBatchInvariantLogpOp", "actual_backend_id": "triton", "actual_kernel_config_id": "rl_engine.kernels.ops.triton.loss.batch_invariant_logp.TritonBatchInvariantLogpOp", - "provenance_status": "registry_resolved_runtime_pending", + "provenance_status": "runtime_evidence_required", "algorithm_property": "batch_invariant_logprob_reduction", "profile_ids": [ "triton_cuda_bf16" ], "architecture_identity": "full_qwen3_8b_dense", "provenance_evidence": { - "kind": "operator_specs_registry_resolution", + "kind": "runtime_execution_via_operator_specs", "registry": "rl_engine/kernels/gtest/operator_specs.py", "candidate_name": "triton", "resolved_path": "rl_engine.kernels.ops.triton.loss.batch_invariant_logp.TritonBatchInvariantLogpOp", - "algorithm_source": "rl_engine/kernels/ops/triton/loss/batch_invariant_logp.py:1", - "runtime_evidence_owner": "C8/C10/C11" + "algorithm_source": "rl_engine/kernels/ops/triton/loss/batch_invariant_logp.py:_batch_invariant_logp_kernel", + "runtime_evidence_command": "python scripts/ws1_candidate_evidence.py --case-id logp-short-vocab151936-t4-triton-v2" } } ], - "fixture_identity_sha256": "c2ec565a575aa3a02c3a27d89ffba3162d93455efd53a7fcb1de11f7e9db7f3d", + "fixture_identity_sha256": "9ebc2c68f411622656c66ab93fa35f39a417e6521cea3d179448626fd1a82675", "provenance_boundary": { - "c2_scope": "logical_workload_identity_and_registry_path_binding", + "c2_scope": "logical_workload_identity_and_executed_representative_candidate_binding", "not_in_c2": [ "full_model_forward", "numerical_150_asserts", - "runtime_kernel_dispatch_observation", + "full_model_runtime_kernel_dispatch_observation", "multi_gpu" ], "runtime_evidence_owner": [ diff --git a/rl_engine/testing/ws1_workload.py b/rl_engine/testing/ws1_workload.py index 02d21964..b0d5c0a2 100644 --- a/rl_engine/testing/ws1_workload.py +++ b/rl_engine/testing/ws1_workload.py @@ -74,6 +74,13 @@ class WorkloadError(ValueError): """Raised when the WS1 workload manifest or fixture is invalid.""" +def _require(mapping: Mapping[str, Any], key: str, *, context: str) -> Any: + """Return mapping[key] or raise WorkloadError (never bare KeyError).""" + if key not in mapping: + raise WorkloadError(f"{context} missing {key!r}") + return mapping[key] + + @dataclass(frozen=True) class LogicalToken: """One active or inactive logical token position.""" @@ -276,6 +283,7 @@ def validate_manifest(raw: Mapping[str, Any]) -> None: _validate_capabilities(raw["capabilities"]) _validate_backend_profiles(raw["backend_profiles"], raw["capabilities"]) _validate_representative_cases(raw["representative_cases"]) + _validate_fixture_case_bindings(raw["fixtures"], raw["representative_cases"]) expected_identity = manifest_identity_hash(raw) if raw["fixture_identity_sha256"] != expected_identity: raise WorkloadError( @@ -309,15 +317,14 @@ def _validate_model_identity(identity: Mapping[str, Any]) -> None: "content_hash_algorithm", "content_hash", "shards", + "weight_files_total_size_bytes", ): if key not in weight: raise WorkloadError(f"weight_snapshot missing {key!r}") shards = weight["shards"] if not isinstance(shards, list) or not shards: raise WorkloadError("weight_snapshot.shards must be a non-empty list") - if int(weight["weight_files_total_size_bytes"]) != sum( - int(s["size_bytes"]) for s in shards - ): + if int(weight["weight_files_total_size_bytes"]) != sum(int(s["size_bytes"]) for s in shards): raise WorkloadError("weight_snapshot file total does not match shard sizes") for shard in shards: digest = str(shard.get("sha256", "")) @@ -378,14 +385,12 @@ def _validate_chain_semantics(sem: Mapping[str, Any]) -> None: actual_sem = sem["backend_actual_semantics"] if not isinstance(actual_sem, Mapping): raise WorkloadError("backend_actual_semantics must be an object") - if actual_sem.get("c2_actual_backend_id") != "registry_resolved_expected_candidate": - raise WorkloadError( - "C2 actual_backend_id semantics must be registry_resolved_expected_candidate" - ) - if "C8" not in actual_sem.get("runtime_observed_actual_owner", []): - raise WorkloadError( - "backend_actual_semantics must assign runtime observed actuals to C8+" - ) + if actual_sem.get("c2_representative_actual_source") != ( + "scripts/ws1_candidate_evidence.py runtime execution" + ): + raise WorkloadError("C2 representative actual provenance must come from runtime execution") + if "C8" not in actual_sem.get("full_model_runtime_observed_actual_owner", []): + raise WorkloadError("backend_actual_semantics must assign full-model actuals to C8+") def _validate_stochastic_policy(policy: Mapping[str, Any]) -> None: @@ -401,22 +406,24 @@ def _validate_stochastic_policy(policy: Mapping[str, Any]) -> None: def _validate_primary_matrix(matrix: Mapping[str, Any], fixtures: Mapping[str, Any]) -> None: - n = int(matrix["N"]) + n = int(_require(matrix, "N", context="primary_matrix")) if n <= 1: raise WorkloadError("primary_matrix.N must be > 1") - sample_ids = list(matrix["sample_ids"]) + sample_ids = list(_require(matrix, "sample_ids", context="primary_matrix")) if len(sample_ids) != n: raise WorkloadError("sample_ids length must equal N") if len(set(sample_ids)) != n: raise WorkloadError("sample_ids must be unique") perm = matrix.get("batch_permutation", {}) if perm.get("enabled"): - p = list(perm["permutation"]) + p = list(_require(perm, "permutation", context="primary_matrix.batch_permutation")) if sorted(p) != list(range(n)): raise WorkloadError("batch_permutation.permutation must be a permutation of [0..N)") - chunk = matrix["chunk"] - chunk_size = int(chunk["chunk_size_tokens"]) - seq_len = int(fixtures["primary_seq_len"]) + chunk = _require(matrix, "chunk", context="primary_matrix") + if not isinstance(chunk, Mapping): + raise WorkloadError("primary_matrix.chunk must be an object") + chunk_size = int(_require(chunk, "chunk_size_tokens", context="primary_matrix.chunk")) + seq_len = int(_require(fixtures, "primary_seq_len", context="fixtures")) if chunk_size <= 0: raise WorkloadError("chunk_size_tokens must be positive") plan = build_chunk_plan(seq_len, chunk_size) @@ -425,7 +432,7 @@ def _validate_primary_matrix(matrix: Mapping[str, Any], fixtures: Mapping[str, A if chunk.get("non_divisible_case") and seq_len % chunk_size == 0: raise WorkloadError("non_divisible_case requires seq_len % chunk_size != 0") - cells = matrix["cells"] + cells = _require(matrix, "cells", context="primary_matrix") if not isinstance(cells, list): raise WorkloadError("primary_matrix.cells must be a list") cell_ids = [c["cell_id"] for c in cells] @@ -454,8 +461,8 @@ def _validate_fixtures(fixtures: Mapping[str, Any], matrix: Mapping[str, Any]) - f"fixtures.samples order/ids must match primary_matrix.sample_ids " f"{expected_ids}, got {got_ids}" ) - primary_seq = int(fixtures["primary_seq_len"]) - declared_varlen = [int(x) for x in fixtures["varlen_seq_lens"]] + primary_seq = int(_require(fixtures, "primary_seq_len", context="fixtures")) + declared_varlen = [int(x) for x in _require(fixtures, "varlen_seq_lens", context="fixtures")] if declared_varlen != [int(s["seq_len"]) for s in samples]: raise WorkloadError("varlen_seq_lens must match fixtures.samples seq_len values") for sample in samples: @@ -471,15 +478,11 @@ def _validate_fixtures(fixtures: Mapping[str, Any], matrix: Mapping[str, Any]) - raise WorkloadError("primary_seq_len must equal the maximum varlen sequence length") # Per-sample prompt/completion lengths are authoritative (no stale scalar pin). expected_prompt_lens = [int(s["prompt_len"]) for s in samples] - expected_completion_lens = [ - int(s["seq_len"]) - int(s["prompt_len"]) for s in samples - ] + expected_completion_lens = [int(s["seq_len"]) - int(s["prompt_len"]) for s in samples] if list(fixtures.get("prompt_lens", [])) != expected_prompt_lens: raise WorkloadError("fixtures.prompt_lens must match per-sample prompt_len values") if list(fixtures.get("completion_lens", [])) != expected_completion_lens: - raise WorkloadError( - "fixtures.completion_lens must match per-sample (seq_len - prompt_len)" - ) + raise WorkloadError("fixtures.completion_lens must match per-sample (seq_len - prompt_len)") if int(fixtures.get("max_completion_len", -1)) != max(expected_completion_lens): raise WorkloadError("fixtures.max_completion_len must equal max(completion_lens)") if "primary_completion_len" in fixtures: @@ -487,10 +490,14 @@ def _validate_fixtures(fixtures: Mapping[str, Any], matrix: Mapping[str, Any]) - "fixtures.primary_completion_len is forbidden under varlen primary samples; " "use completion_lens / max_completion_len" ) - padding = fixtures["padding"] + padding = _require(fixtures, "padding", context="fixtures") + if not isinstance(padding, Mapping): + raise WorkloadError("fixtures.padding must be an object") if "right" not in padding["modes"] or "left" not in padding["modes"]: raise WorkloadError("padding.modes must include left and right") - packing = fixtures["packing"] + packing = _require(fixtures, "packing", context="fixtures") + if not isinstance(packing, Mapping): + raise WorkloadError("fixtures.packing must be an object") if packing["status"] not in { "supported", "n_a_with_capability_proof", @@ -502,9 +509,13 @@ def _validate_fixtures(fixtures: Mapping[str, Any], matrix: Mapping[str, Any]) - raise WorkloadError("packing op is present, so C2 must pin a supported packed fixture") if not packing.get("packed_fixture"): raise WorkloadError("supported packing requires packed_fixture") - for name in ("short_full_model_fixture", "long_full_model_fixture"): + for name in ( + "short_full_model_fixture", + "long_full_model_fixture", + "representative_full_model_fixture", + ): fixture = fixtures[name] - if len(fixture["token_ids"]) != int(fixture["seq_len"]): + if "token_ids" in fixture and len(fixture["token_ids"]) != int(fixture["seq_len"]): raise WorkloadError(f"{name} token_ids length mismatch") if not fixture.get("candidate_case_ids"): raise WorkloadError(f"{name} must reference representative case IDs") @@ -538,9 +549,7 @@ def _validate_backend_profiles( if name not in profiles: raise WorkloadError(f"backend_profiles missing required profile {name!r}") required_ops = [ - e["op"] - for e in capabilities["required_chain_ops"] - if e["status"] == "required" + e["op"] for e in capabilities["required_chain_ops"] if e["status"] == "required" ] for name, profile in profiles.items(): nodes = profile.get("required_nodes") @@ -601,6 +610,8 @@ def _validate_representative_cases(cases: Sequence[Mapping[str, Any]]) -> None: "provenance_evidence", "algorithm_property", "architecture_identity", + "fixture_id", + "operator_spec", ): if key not in case: raise WorkloadError(f"case {case.get('case_id')} missing {key!r}") @@ -608,21 +619,21 @@ def _validate_representative_cases(cases: Sequence[Mapping[str, Any]]) -> None: raise WorkloadError( f"case {case['case_id']} must pin architecture_identity=full_qwen3_8b_dense" ) - if case["provenance_status"] != "registry_resolved_runtime_pending": - raise WorkloadError( - f"case {case['case_id']} must distinguish registry resolution from runtime" - ) + if case["provenance_status"] != "runtime_evidence_required": + raise WorkloadError(f"case {case['case_id']} must require runtime candidate evidence") if case["actual_backend_id"] != case["expected_backend_id"]: raise WorkloadError(f"case {case['case_id']} actual backend mismatch") if case["actual_kernel_config_id"] != case["expected_kernel_config_id"]: raise WorkloadError(f"case {case['case_id']} actual kernel mismatch") evidence = case["provenance_evidence"] - if evidence.get("kind") != "operator_specs_registry_resolution": - raise WorkloadError(f"case {case['case_id']} lacks registry provenance") + if evidence.get("kind") != "runtime_execution_via_operator_specs": + raise WorkloadError(f"case {case['case_id']} lacks runtime provenance command") if evidence.get("resolved_path") != case["actual_kernel_config_id"]: raise WorkloadError(f"case {case['case_id']} evidence path mismatch") if not evidence.get("algorithm_source"): raise WorkloadError(f"case {case['case_id']} lacks algorithm source proof") + if not evidence.get("runtime_evidence_command"): + raise WorkloadError(f"case {case['case_id']} lacks runtime evidence command") for profile in _REQUIRED_PROFILES: profile_cases = [c for c in cases if profile in c.get("profile_ids", [])] for family in ("gemm", "attention", "logprob"): @@ -634,13 +645,75 @@ def _validate_representative_cases(cases: Sequence[Mapping[str, Any]]) -> None: gemm_m = {int(c["shape"]["M"]) for c in profile_cases if c["family"] == "gemm"} if len(gemm_m) < 2: raise WorkloadError(f"profile {profile} GEMM cases require multiple M values") - attn_modes = { - c["shape"]["mode"] for c in profile_cases if c["family"] == "attention" - } + attn_modes = {c["shape"]["mode"] for c in profile_cases if c["family"] == "attention"} if attn_modes != {"prefill", "decode"}: raise WorkloadError(f"profile {profile} attention cases require prefill+decode") +def _validate_fixture_case_bindings( + fixtures: Mapping[str, Any], cases: Sequence[Mapping[str, Any]] +) -> None: + """Require every fixture→case edge to describe a shape produced by that fixture.""" + fixture_names = ( + "short_full_model_fixture", + "long_full_model_fixture", + "representative_full_model_fixture", + ) + by_fixture_id = {fixtures[name]["fixture_id"]: fixtures[name] for name in fixture_names} + by_case_id = {case["case_id"]: case for case in cases} + + for fixture_id, fixture in by_fixture_id.items(): + for case_id in fixture["candidate_case_ids"]: + if case_id not in by_case_id: + raise WorkloadError(f"fixture {fixture_id} references unknown case {case_id!r}") + if by_case_id[case_id]["fixture_id"] != fixture_id: + raise WorkloadError( + f"fixture {fixture_id} references case {case_id!r} bound to " + f"{by_case_id[case_id]['fixture_id']!r}" + ) + + referenced = { + case_id for fixture in by_fixture_id.values() for case_id in fixture["candidate_case_ids"] + } + if referenced != set(by_case_id): + raise WorkloadError("every representative case must be referenced by its source fixture") + + short = fixtures["short_full_model_fixture"] + long = fixtures["long_full_model_fixture"] + primary_total_tokens = sum(int(sample["seq_len"]) for sample in fixtures["samples"]) + primary_max_seq = max(int(sample["seq_len"]) for sample in fixtures["samples"]) + expected_shapes = { + "short_full_model_seq8": { + "gemm": {"M": int(short["seq_len"])}, + "logprob": {"B": 1, "T": int(short["seq_len"]) - int(short["prompt_len"])}, + }, + "long_full_model_seq32": { + "attention": {"B": 1, "Sq": 1, "Skv": int(long["seq_len"]), "mode": "decode"} + }, + "rep_full_model_seq16": { + "gemm": {"M": primary_total_tokens}, + "attention": { + "B": len(fixtures["samples"]), + "Sq": primary_max_seq, + "Skv": primary_max_seq, + "mode": "prefill", + }, + }, + } + for case in cases: + required = expected_shapes[case["fixture_id"]][case["family"]] + mismatched = { + key: (case["shape"].get(key), value) + for key, value in required.items() + if case["shape"].get(key) != value + } + if mismatched: + raise WorkloadError( + f"case {case['case_id']} shape does not derive from fixture " + f"{case['fixture_id']}: {mismatched}" + ) + + def build_logical_batch( manifest: WS1Manifest | None = None, *, @@ -768,7 +841,7 @@ def restore_logical_order( if len(physical_values) != len(layout.restore_map): raise WorkloadError("physical_values length does not match restore map") out: dict[tuple[str, int], Any] = {} - for key, value in zip(layout.restore_map, physical_values): + for key, value in zip(layout.restore_map, physical_values, strict=True): if key in out: raise WorkloadError(f"duplicate logical key {key}") out[key] = value @@ -810,16 +883,15 @@ def apply_padding( pad_count = target_len - sample.seq_len pad_tokens = (pad_id,) * pad_count pad_restore: tuple[None, ...] = (None,) * pad_count - logical_restore = tuple( - (sample.sample_id, pos) for pos in range(sample.seq_len) - ) + logical_restore = tuple((sample.sample_id, pos) for pos in range(sample.seq_len)) if pad_side == "right": ids = sample.token_ids + pad_tokens mask = (1,) * sample.seq_len + (0,) * pad_count rmap = logical_restore + pad_restore - loss_mask = tuple( - int(pos >= sample.prompt_len) for pos in range(sample.seq_len) - ) + (0,) * pad_count + loss_mask = ( + tuple(int(pos >= sample.prompt_len) for pos in range(sample.seq_len)) + + (0,) * pad_count + ) position_ids = tuple(range(sample.seq_len)) + (0,) * pad_count else: ids = pad_tokens + sample.token_ids @@ -856,10 +928,10 @@ def restore_logical_order_from_padded( if len(physical_values) != len(padded.restore_map): raise WorkloadError("physical_values batch size mismatch") out: dict[tuple[str, int], Any] = {} - for row_vals, row_map in zip(physical_values, padded.restore_map): + for row_vals, row_map in zip(physical_values, padded.restore_map, strict=True): if len(row_vals) != len(row_map): raise WorkloadError("physical_values seq length mismatch") - for val, key in zip(row_vals, row_map): + for val, key in zip(row_vals, row_map, strict=True): if key is None: continue if key in out: @@ -947,9 +1019,7 @@ def assert_no_undeclared_randomness( allowed = set(declared_rng_sources) bad = [s for s in encountered_rng_sources if s not in allowed] if bad: - raise WorkloadError( - f"undeclared stochastic source(s) {bad}; policy is hard_fail" - ) + raise WorkloadError(f"undeclared stochastic source(s) {bad}; policy is hard_fail") def fixture_hash( @@ -962,16 +1032,15 @@ def fixture_hash( m = manifest if manifest is not None else load_manifest() logical = batch if batch is not None else build_logical_batch(m) payload = _manifest_identity_payload(m.raw) - payload["selected_logical_batch"] = [ - list(x) for x in logical.token_multiset(active_only=False) - ] + payload["selected_logical_batch"] = [list(x) for x in logical.token_multiset(active_only=False)] payload["extra"] = dict(extra) if extra else {} blob = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(blob).hexdigest() def _manifest_identity_payload(raw: Mapping[str, Any]) -> dict[str, Any]: - return {k: raw[k] for k in _REQUIRED_TOP_LEVEL if k != "fixture_identity_sha256"} + # Hash every declared section so future manifest keys cannot escape identity. + return {k: v for k, v in raw.items() if k != "fixture_identity_sha256"} def manifest_identity_hash(raw: Mapping[str, Any]) -> str: @@ -988,9 +1057,7 @@ def _sequence_digest(values: Any) -> str: def weight_snapshot_hash(shards: Sequence[Mapping[str, Any]]) -> str: """Hash canonical filename/SHA-256/size records for all weight shards.""" - records = sorted( - (str(s["filename"]), str(s["sha256"]), int(s["size_bytes"])) for s in shards - ) + records = sorted((str(s["filename"]), str(s["sha256"]), int(s["size_bytes"])) for s in shards) blob = "".join(f"{name}\t{digest}\t{size}\n" for name, digest, size in records) return hashlib.sha256(blob.encode("utf-8")).hexdigest() @@ -1041,6 +1108,7 @@ def reference_payload( "chunk_spans": [list(s) for s in chunk.chunk_spans], }, "backend_profiles": list(m.backend_profiles.keys()), + "backend_actual_semantics": m.chain_semantics["backend_actual_semantics"], "case_ids": list(case_ids(m)), "profile_missing_required": { pid: profile_missing_required_nodes(m, pid) for pid in m.backend_profiles @@ -1053,26 +1121,40 @@ def reference_payload( [[int(t.is_active) for t in s.tokens()] for s in batch.samples] ), "padded_left_sha256": _sequence_digest( - [padded_left.physical_token_ids, padded_left.physical_attention_mask, - padded_left.physical_loss_mask, padded_left.physical_position_ids] + [ + padded_left.physical_token_ids, + padded_left.physical_attention_mask, + padded_left.physical_loss_mask, + padded_left.physical_position_ids, + ] ), "padded_right_sha256": _sequence_digest( - [padded_right.physical_token_ids, padded_right.physical_attention_mask, - padded_right.physical_loss_mask, padded_right.physical_position_ids] + [ + padded_right.physical_token_ids, + padded_right.physical_attention_mask, + padded_right.physical_loss_mask, + padded_right.physical_position_ids, + ] ), "chunked_sha256": _sequence_digest( - [chunked.physical_token_ids, chunked.physical_loss_mask, - chunked.restore_map, chunked.segment_offsets, chunked.segment_lengths] + [ + chunked.physical_token_ids, + chunked.physical_loss_mask, + chunked.restore_map, + chunked.segment_offsets, + chunked.segment_lengths, + ] ), "packed_sha256": _sequence_digest( - [packed.physical_token_ids, packed.physical_loss_mask, - packed.restore_map, packed.segment_offsets, packed.segment_lengths] - ), - "short_fixture_sha256": _sequence_digest( - m.fixtures["short_full_model_fixture"] - ), - "long_fixture_sha256": _sequence_digest( - m.fixtures["long_full_model_fixture"] + [ + packed.physical_token_ids, + packed.physical_loss_mask, + packed.restore_map, + packed.segment_offsets, + packed.segment_lengths, + ] ), + "short_fixture_sha256": _sequence_digest(m.fixtures["short_full_model_fixture"]), + "long_fixture_sha256": _sequence_digest(m.fixtures["long_full_model_fixture"]), }, } diff --git a/scripts/ws1_candidate_evidence.py b/scripts/ws1_candidate_evidence.py new file mode 100755 index 00000000..7c3cccf6 --- /dev/null +++ b/scripts/ws1_candidate_evidence.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Execute WS1 C2 representative CUDA/Triton cases and emit runtime provenance.""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import platform +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import torch + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from rl_engine.kernels.gtest import run_operator_suite # noqa: E402 +from rl_engine.kernels.gtest.operator_specs import make_candidate, make_operator_case # noqa: E402 +from rl_engine.testing.ws1_workload import WorkloadError, load_manifest # noqa: E402 + + +def _object_path(value: Any) -> str: + cls = value.__class__ + return f"{cls.__module__}.{cls.__qualname__}" + + +def _case_args(case: dict[str, Any], seed: int) -> SimpleNamespace: + shape = case["shape"] + operator_spec = case["operator_spec"] + common: dict[str, Any] = { + "op": operator_spec, + "candidate": case["expected_backend_id"], + "arch_key": None, + "input_mode": "random", + "constant_value": 0.25, + "token_value": 0, + "normalized_dim": 4096, + "k_dim": 4096, + "n_dim": 4096, + "theta": 1.0e6, + "eps": 1.0e-6, + "seed": seed, + } + if operator_spec == "det_gemm": + common.update(batch=1, seq=shape["M"], k_dim=shape["K"], n_dim=shape["N"]) + elif operator_spec == "attention": + common.update( + batch=shape["B"], + seq=shape["Sq"], + skv=shape["Skv"], + n_heads=shape["Hq"], + n_kv_heads=shape["Hkv"], + causal=1, + use_padding=0, + scale_mode="default", + ) + elif operator_spec in {"logp", "batch_invariant_logp"}: + common.update(batch=shape["B"], seq=shape["T"], vocab=shape["vocab"]) + else: + raise WorkloadError(f"unsupported representative operator_spec {operator_spec!r}") + return SimpleNamespace(**common) + + +def run_case(case: dict[str, Any], *, seed: int, device: torch.device) -> dict[str, Any]: + args = _case_args(case, seed) + candidate = make_candidate(args) + actual_path = _object_path(candidate.fn) + if candidate.backend != case["expected_backend_id"]: + raise WorkloadError( + f"case {case['case_id']} resolved backend {candidate.backend!r}, expected " + f"{case['expected_backend_id']!r}" + ) + if actual_path != case["expected_kernel_config_id"]: + raise WorkloadError( + f"case {case['case_id']} resolved kernel {actual_path!r}, expected " + f"{case['expected_kernel_config_id']!r}" + ) + + operator_case = make_operator_case(args, torch.bfloat16, device) + report = run_operator_suite( + case["operator_spec"], candidates=[candidate], cases=[operator_case] + ) + torch.cuda.synchronize(device) + candidate_report = report.candidates[0] + output_checks = [ + { + "shape": list(output.shape), + "dtype": output.candidate_dtype, + "max_abs_error": output.max_abs_error, + "passed": output.passed, + } + for checked_case in candidate_report.cases + for output in checked_case.outputs + ] + return { + "case_id": case["case_id"], + "fixture_id": case["fixture_id"], + "operator_spec": case["operator_spec"], + "expected_backend_id": case["expected_backend_id"], + "actual_backend_id": candidate.backend, + "expected_kernel_config_id": case["expected_kernel_config_id"], + "actual_kernel_config_id": actual_path, + "algorithm_property": case["algorithm_property"], + "shape": case["shape"], + "runtime_status": "passed" if report.passed else "failed", + "outputs": output_checks, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run manifest-pinned WS1 representative candidates on a real GPU." + ) + parser.add_argument("--manifest", type=Path, default=None) + parser.add_argument( + "--profile", + action="append", + choices=("cuda_bf16", "triton_cuda_bf16"), + help="Profile to run; repeatable. Defaults to both required profiles.", + ) + parser.add_argument("--case-id", action="append", help="Optional case_id filter.") + parser.add_argument("--emit-json", default="-", help="Output path, or '-' for stdout.") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if not torch.cuda.is_available(): + print("error: CUDA is required for runtime candidate evidence", file=sys.stderr) + return 2 + + try: + manifest = load_manifest(args.manifest) + profiles = set(args.profile or ("cuda_bf16", "triton_cuda_bf16")) + selected_ids = set(args.case_id or ()) + cases = [ + case + for case in manifest.representative_cases + if profiles.intersection(case["profile_ids"]) + and (not selected_ids or case["case_id"] in selected_ids) + ] + if selected_ids - {case["case_id"] for case in cases}: + unknown = sorted(selected_ids - {case["case_id"] for case in cases}) + raise WorkloadError(f"unknown or profile-filtered case IDs: {unknown}") + device = torch.device("cuda:0") + log_stream = sys.stderr if args.emit_json == "-" else sys.stdout + with contextlib.redirect_stdout(log_stream): + results = [ + run_case(case, seed=manifest.seed + i, device=device) + for i, case in enumerate(cases) + ] + except (RuntimeError, ValueError, WorkloadError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + props = torch.cuda.get_device_properties(device) + payload = { + "schema_version": "ws1-c2-runtime-provenance-v1", + "workload_id": manifest.workload_id, + "fixture_identity_sha256": manifest.raw["fixture_identity_sha256"], + "execution_dtype": "bfloat16", + "device": { + "index": device.index, + "name": props.name, + "compute_capability": f"sm{props.major}{props.minor}", + "execution_world_size": 1, + }, + "software": { + "python": platform.python_version(), + "torch": torch.__version__, + "cuda_runtime": torch.version.cuda, + }, + "profiles": sorted(profiles), + "passed": bool(results) and all(result["runtime_status"] == "passed" for result in results), + "cases": results, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if args.emit_json == "-": + sys.stdout.write(rendered) + else: + path = Path(args.emit_json) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered, encoding="utf-8") + print(f"wrote: {path}") + return 0 if payload["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ws1_reference.py b/scripts/ws1_reference.py index 79af5b97..7e677cba 100755 --- a/scripts/ws1_reference.py +++ b/scripts/ws1_reference.py @@ -18,12 +18,6 @@ from pathlib import Path -def _ensure_repo_on_path() -> None: - repo_root = Path(__file__).resolve().parents[1] - if str(repo_root) not in sys.path: - sys.path.insert(0, str(repo_root)) - - def _load_workload_module(): """Load the pure-Python C2 module without importing torch-heavy package helpers.""" module_path = Path(__file__).resolve().parents[1] / "rl_engine/testing/ws1_workload.py" @@ -80,7 +74,6 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: - _ensure_repo_on_path() workload = _load_workload_module() WorkloadError = workload.WorkloadError @@ -93,12 +86,8 @@ def main(argv: list[str] | None = None) -> int: f"{manifest.workload_id!r}" ) if args.seed is not None and int(args.seed) != manifest.seed: - raise WorkloadError( - f"--seed {args.seed} does not match manifest seed {manifest.seed}" - ) - payload = workload.reference_payload( - manifest, cell_id=args.cell_id, dtype=args.dtype - ) + raise WorkloadError(f"--seed {args.seed} does not match manifest seed {manifest.seed}") + payload = workload.reference_payload(manifest, cell_id=args.cell_id, dtype=args.dtype) except WorkloadError as exc: print(f"error: {exc}", file=sys.stderr) return 2 diff --git a/tests/test_op_checks.py b/tests/test_op_checks.py index de2ceb22..bcbe89f0 100644 --- a/tests/test_op_checks.py +++ b/tests/test_op_checks.py @@ -221,6 +221,39 @@ def test_ws1_report_persists_roles_and_backend_provenance(): assert "baseline" not in data["cases"][0]["outputs"][0] +def test_ws1_report_accepts_triton_backend_provenance(): + provenance = BackendProvenance( + backend_profile="triton_cuda_bf16", + requested_backend="triton", + actual_backend="triton", + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, + ) + report = run_operator_suite( + "logp", + candidates=[ + CandidateSpec( + name="triton-logp", + backend="triton", + fn=NativeLogpOp(), + provenance=provenance, + ) + ], + cases=[_logp_case("bf16", torch.bfloat16, seed=15)], + ) + output = report.candidates[0].cases[0].outputs[0] + assert output.judgment == "forward_accuracy" + assert output.comparison_lhs_role == "bf16_candidate" + assert output.comparison_rhs_role == "fp32_reference" + data = report.to_dict()["candidates"][0] + assert data["backend_provenance"]["actual_backend"] == "triton" + assert "baseline" not in data["cases"][0]["outputs"][0] + + def test_ws1_report_rejects_backend_provenance_mismatch(): provenance = BackendProvenance( backend_profile="cuda_bf16", @@ -278,6 +311,29 @@ def wrong_output_dtype(logits, token_ids): cases=[_logp_case("bf16", torch.bfloat16, seed=14)], ) + wrong_gold_case = _logp_case("bf16", torch.bfloat16, seed=14) + wrong_gold_case = OperatorCase( + name=wrong_gold_case.name, + op_class=wrong_gold_case.op_class, + dtype=wrong_gold_case.dtype, + inputs=wrong_gold_case.inputs, + gold_fn=lambda **inputs: NativeLogpOp().forward(**inputs), + grad_input_names=wrong_gold_case.grad_input_names, + ) + with pytest.raises(ContractResolveError, match="gold output dtype"): + run_operator_suite( + "logp", + candidates=[ + CandidateSpec( + name="wrong-gold-output", + backend="cuda", + fn=NativeLogpOp(), + provenance=provenance, + ) + ], + cases=[wrong_gold_case], + ) + def test_candidate_arch_key_uses_tolerance_override(): def slightly_shifted_logp(logits, token_ids): diff --git a/tests/test_tolerance_contract.py b/tests/test_tolerance_contract.py index 4fcbe23f..3c73d80c 100644 --- a/tests/test_tolerance_contract.py +++ b/tests/test_tolerance_contract.py @@ -332,6 +332,16 @@ def test_chain_aggregate_named_resolve(): resolve_chain_aggregate_thresholds(contract, "mean_abs_dlogp", "bfloat16") +def test_provisional_thresholds_record_calibration_rationale(): + contract = load_contract() + gradient = contract["judgments"]["gradient_accuracy"] + assert gradient["calibration_status"] == "provisional_pending_measured_backward_evidence" + assert "measured evidence" in gradient["calibration_note"] + + approx_kl0 = contract["chain_logprob_aggregates"]["metrics"]["approx_kl0"] + assert "max_abs_dlogp is therefore the stricter guard" in approx_kl0["threshold_rationale"] + + def test_compute_logprob_aggregates_formulas(): # lhs - rhs = [0.0, 0.1, -0.2] lhs = torch.tensor([1.0, 2.1, 0.8], dtype=torch.float32) @@ -469,10 +479,15 @@ def test_clipfrac0_counts_ratios_outside_the_interval(): def test_clip_interval_endpoints_count_as_inside(): - lo, hi = 0.5, 2.0 + # Drive endpoints through the same float32 exp path the implementation uses so + # ratio0 lands exactly on the clip interval bounds (no log/exp float round-trip). + dlogp = torch.tensor([-1.0, 1.0], dtype=torch.float32) + ratio0 = torch.exp(dlogp) + lo = float(ratio0[0].item()) + hi = float(ratio0[1].item()) agg = compute_logprob_aggregates( - torch.tensor([math.log(lo), math.log(hi)]), - torch.zeros(2), + dlogp, + torch.zeros(2, dtype=torch.float32), torch.ones(2, dtype=torch.bool), contract=load_contract(), report_kind="train_infer_logprob_parity", diff --git a/tests/test_ws1_candidate_evidence.py b/tests/test_ws1_candidate_evidence.py new file mode 100644 index 00000000..201bce41 --- /dev/null +++ b/tests/test_ws1_candidate_evidence.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""GPU acceptance coverage for WS1 C2 representative candidate provenance.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +REPO_ROOT = Path(__file__).resolve().parents[1] +EVIDENCE_SCRIPT = REPO_ROOT / "scripts" / "ws1_candidate_evidence.py" + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="WS1 candidate evidence requires CUDA") +def test_ws1_cuda_and_triton_candidate_runtime_provenance(): + proc = subprocess.run( + [sys.executable, str(EVIDENCE_SCRIPT), "--emit-json", "-"], + check=False, + capture_output=True, + text=True, + cwd=str(REPO_ROOT), + timeout=600, + ) + assert proc.returncode == 0, proc.stderr + payload = json.loads(proc.stdout) + assert payload["passed"] is True + assert payload["profiles"] == ["cuda_bf16", "triton_cuda_bf16"] + assert payload["device"]["index"] == 0 + assert payload["device"]["execution_world_size"] == 1 + assert len(payload["cases"]) == 10 + assert {case["actual_backend_id"] for case in payload["cases"]} == {"cuda", "triton"} + for case in payload["cases"]: + assert case["runtime_status"] == "passed" + assert case["actual_backend_id"] == case["expected_backend_id"] + assert case["actual_kernel_config_id"] == case["expected_kernel_config_id"] + assert case["outputs"] + assert all(output["passed"] for output in case["outputs"]) diff --git a/tests/test_ws1_workload.py b/tests/test_ws1_workload.py index 1333b900..ef02f28c 100644 --- a/tests/test_ws1_workload.py +++ b/tests/test_ws1_workload.py @@ -5,18 +5,20 @@ from __future__ import annotations -import json import importlib.util +import json import subprocess import sys from pathlib import Path import pytest +from rl_engine.kernels.gtest.operator_specs import OP_SPECS + REPO_ROOT = Path(__file__).resolve().parents[1] REFERENCE_SCRIPT = REPO_ROOT / "scripts" / "ws1_reference.py" +CANDIDATE_EVIDENCE_SCRIPT = REPO_ROOT / "scripts" / "ws1_candidate_evidence.py" CONTRACT_PATH = REPO_ROOT / "rl_engine/kernels/gtest/tolerance_contract.json" -OPERATOR_SPECS_PATH = REPO_ROOT / "rl_engine/kernels/gtest/operator_specs.py" def _load_pure_workload_module(): @@ -61,6 +63,7 @@ def _load_pure_workload_module(): def load_contract(): return json.loads(CONTRACT_PATH.read_text(encoding="utf-8")) + REQUIRED_CELLS = { "B1-singleton_aggregate/full", "BN/full", @@ -194,18 +197,14 @@ def test_chain_semantics_report_and_actual_boundaries(manifest): "baseline", "singleton_aggregate", } + assert sem["report_naming"]["singleton_aggregate_is"] == "c2_execution_aggregation_mode_only" assert ( - sem["report_naming"]["singleton_aggregate_is"] - == "c2_execution_aggregation_mode_only" - ) - assert ( - sem["backend_actual_semantics"]["c2_actual_backend_id"] - == "registry_resolved_expected_candidate" + sem["backend_actual_semantics"]["c2_representative_actual_source"] + == "scripts/ws1_candidate_evidence.py runtime execution" ) - assert "C8" in sem["backend_actual_semantics"]["runtime_observed_actual_owner"] + assert "C8" in sem["backend_actual_semantics"]["full_model_runtime_observed_actual_owner"] boundary = manifest.raw["provenance_boundary"] assert "full_model_forward" in boundary["not_in_c2"] - assert "runtime_kernel_dispatch_observation" in boundary["not_in_c2"] def test_stale_primary_completion_len_rejected(): @@ -255,6 +254,7 @@ def test_batch_permutation_restores_multiset(manifest): perm = batch_permutation_from_manifest(manifest) permuted = permute_batch(batch, perm) assert permuted.sample_ids != batch.sample_ids + # Multiset equality is order-sensitive in token_multiset (fixed order). # After sorting by sample_id, the pairs must match. def sorted_multiset(b): @@ -268,9 +268,7 @@ def sorted_multiset(b): # samples in permuted are batch.samples[perm[i]]; map back: restored_samples = [] for old_i in range(len(batch.samples)): - # find which permuted index holds original old_i - new_i = list(perm).index(old_i) - restored_samples.append(permuted.samples[new_i]) + restored_samples.append(permuted.samples[inverse[old_i]]) restored = ws1.LogicalBatch( workload_id=batch.workload_id, seed=batch.seed, @@ -382,8 +380,9 @@ def test_representative_cases_stable_ids_and_pins(manifest): assert case["architecture_identity"] == "full_qwen3_8b_dense" assert case["expected_backend_id"] == case["actual_backend_id"] assert case["expected_kernel_config_id"] == case["actual_kernel_config_id"] - assert case["provenance_status"] == "registry_resolved_runtime_pending" + assert case["provenance_status"] == "runtime_evidence_required" assert case["provenance_evidence"]["resolved_path"] == case["actual_kernel_config_id"] + assert case["provenance_evidence"]["runtime_evidence_command"] assert case["algorithm_property"] assert "shape" in case for profile in ("cuda_bf16", "triton_cuda_bf16"): @@ -394,26 +393,51 @@ def test_representative_cases_stable_ids_and_pins(manifest): ] assert {c["family"] for c in cases} == {"gemm", "attention", "logprob"} assert len({c["shape"]["M"] for c in cases if c["family"] == "gemm"}) >= 2 - attention_modes = { - c["shape"]["mode"] for c in cases if c["family"] == "attention" - } + attention_modes = {c["shape"]["mode"] for c in cases if c["family"] == "attention"} assert attention_modes == {"prefill", "decode"} +def test_fixture_case_shapes_are_derived_from_fixed_fixtures(manifest): + fixtures = manifest.fixtures + cases = {case["case_id"]: case for case in manifest.representative_cases} + for fixture_name in ( + "short_full_model_fixture", + "long_full_model_fixture", + "representative_full_model_fixture", + ): + fixture = fixtures[fixture_name] + for case_id in fixture["candidate_case_ids"]: + assert cases[case_id]["fixture_id"] == fixture["fixture_id"] + + short_cases = [ + cases[case_id] for case_id in fixtures["short_full_model_fixture"]["candidate_case_ids"] + ] + assert {case["shape"]["M"] for case in short_cases if case["family"] == "gemm"} == {8} + assert {case["shape"]["T"] for case in short_cases if case["family"] == "logprob"} == {4} + primary_cases = [ + cases[case_id] + for case_id in fixtures["representative_full_model_fixture"]["candidate_case_ids"] + ] + assert {case["shape"]["M"] for case in primary_cases if case["family"] == "gemm"} == {59} + assert { + (case["shape"]["B"], case["shape"]["Sq"], case["shape"]["Skv"]) + for case in primary_cases + if case["family"] == "attention" + } == {(4, 19, 19)} + + def test_declared_candidates_resolve_to_real_operator_specs(manifest): - source = OPERATOR_SPECS_PATH.read_text(encoding="utf-8") spec_map = manifest.raw["capabilities"]["operator_spec_map"] for node, spec_name in spec_map.items(): - assert f'"{spec_name}": OperatorSpec(' in source, node + assert spec_name in OP_SPECS, node for case in manifest.representative_cases: evidence = case["provenance_evidence"] - resolved_class = evidence["resolved_path"].rsplit(".", 1)[1] - assert resolved_class in source - assert f'"{evidence["candidate_name"]}"' in source - algorithm_path, line = evidence["algorithm_source"].rsplit(":", 1) + spec = OP_SPECS[case["operator_spec"]] + assert spec.candidate_paths[evidence["candidate_name"]] == evidence["resolved_path"] + algorithm_path, symbol = evidence["algorithm_source"].rsplit(":", 1) algorithm_file = REPO_ROOT / algorithm_path assert algorithm_file.is_file() - assert 1 <= int(line) <= len(algorithm_file.read_text(encoding="utf-8").splitlines()) + assert symbol in algorithm_file.read_text(encoding="utf-8") def test_capabilities_packing_and_qk_norm(manifest): @@ -460,6 +484,7 @@ def test_reference_payload_contains_required_fields(manifest): assert payload["fixture_hash"] == fixture_hash(manifest) assert payload["cell_id"] == "BN/full" assert payload["clip_interval"] == [0.8, 1.2] + assert "c2_representative_actual_source" in payload["backend_actual_semantics"] def test_ws1_reference_cli_emits_identity(): @@ -478,6 +503,7 @@ def test_ws1_reference_cli_emits_identity(): capture_output=True, text=True, cwd=str(REPO_ROOT), + timeout=120, ) assert proc.returncode == 0, proc.stderr payload = json.loads(proc.stdout) @@ -487,6 +513,19 @@ def test_ws1_reference_cli_emits_identity(): assert len(payload["fixture_hash"]) == 64 +def test_candidate_evidence_cli_help_is_available(): + proc = subprocess.run( + [sys.executable, str(CANDIDATE_EVIDENCE_SCRIPT), "--help"], + check=False, + capture_output=True, + text=True, + cwd=str(REPO_ROOT), + timeout=60, + ) + assert proc.returncode == 0, proc.stderr + assert "representative candidates on a real GPU" in proc.stdout + + def test_build_chunk_plan_edges(): plan = build_chunk_plan(16, 7) assert plan.chunk_spans == ((0, 7), (7, 14), (14, 16)) From 362562e4d067cc65ca814b421883152ece378815 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 15:35:26 +0800 Subject: [PATCH 07/10] style(testing): apply isort export ordering --- rl_engine/testing/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rl_engine/testing/__init__.py b/rl_engine/testing/__init__.py index 1d5708ac..8a66f5de 100644 --- a/rl_engine/testing/__init__.py +++ b/rl_engine/testing/__init__.py @@ -19,8 +19,8 @@ WorkloadError, WS1Manifest, apply_chunking, - apply_padding, apply_packing, + apply_padding, build_logical_batch, fixture_hash, load_manifest, From b41c6f852f4c4c0b0bd0c02fc1f385dd431e16d2 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 15:40:36 +0800 Subject: [PATCH 08/10] fix(ws1): satisfy mypy workload validation --- rl_engine/testing/ws1_workload.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rl_engine/testing/ws1_workload.py b/rl_engine/testing/ws1_workload.py index b0d5c0a2..d3ae608a 100644 --- a/rl_engine/testing/ws1_workload.py +++ b/rl_engine/testing/ws1_workload.py @@ -330,10 +330,10 @@ def _validate_model_identity(identity: Mapping[str, Any]) -> None: digest = str(shard.get("sha256", "")) if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): raise WorkloadError("every weight shard must pin a lowercase SHA-256") - expected = weight_snapshot_hash(shards) + expected_content_hash = weight_snapshot_hash(shards) if weight["content_hash_algorithm"] != "sha256-of-sorted-shard-records-v1": raise WorkloadError("unsupported weight_snapshot content_hash_algorithm") - if weight["content_hash"] != expected: + if weight["content_hash"] != expected_content_hash: raise WorkloadError("weight_snapshot content_hash does not match shard records") @@ -682,7 +682,7 @@ def _validate_fixture_case_bindings( long = fixtures["long_full_model_fixture"] primary_total_tokens = sum(int(sample["seq_len"]) for sample in fixtures["samples"]) primary_max_seq = max(int(sample["seq_len"]) for sample in fixtures["samples"]) - expected_shapes = { + expected_shapes: dict[str, dict[str, dict[str, Any]]] = { "short_full_model_seq8": { "gemm": {"M": int(short["seq_len"])}, "logprob": {"B": 1, "T": int(short["seq_len"]) - int(short["prompt_len"])}, From f1bfbc530a96593788c8825a195706aa04e8a4b7 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 17:35:00 +0800 Subject: [PATCH 09/10] feat(ws1): land C3 forward config-invariance harness (#269) Add the shared forward accuracy/invariance API, C2 config matrix, backend provenance fail-closed checks, selected-logprob smoke, GPU gate CLI, CPU tests, and closeout evidence for WS1 C3. --- .github/workflows/ci.yml | 1 + docs/design/ws1-c3-269-closeout-evidence.md | 64 ++ rl_engine/kernels/gtest/__init__.py | 18 + rl_engine/kernels/gtest/forward_invariance.py | 726 ++++++++++++++++++ scripts/check_forward_invariance.py | 262 +++++++ tests/test_forward_invariance.py | 462 +++++++++++ 6 files changed, 1533 insertions(+) create mode 100644 docs/design/ws1-c3-269-closeout-evidence.md create mode 100644 rl_engine/kernels/gtest/forward_invariance.py create mode 100644 scripts/check_forward_invariance.py create mode 100644 tests/test_forward_invariance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92cd0433..5d7225a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,7 @@ jobs: run: | python -m pytest rl_engine/tests/test_dispatch.py -v PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest tests/test_attention_correctness.py -q -rs + python -m pytest tests/test_forward_invariance.py -q - name: Run Attention Ground-Truth Tests (CPU-safe) run: | diff --git a/docs/design/ws1-c3-269-closeout-evidence.md b/docs/design/ws1-c3-269-closeout-evidence.md new file mode 100644 index 00000000..a876476c --- /dev/null +++ b/docs/design/ws1-c3-269-closeout-evidence.md @@ -0,0 +1,64 @@ +# WS1 C3 (#269) closeout evidence + +**Parent:** #266 · **Depends on:** #267 / #268 · **Scope:** shared forward harness only + +## Acceptance map + +| #269 criterion | Evidence | +| --- | --- | +| Accuracy and invariance separate | `ForwardInvarianceReport.accuracy_reports` and `invariance_reports` | +| Batch/chunk bitwise after logical unpadding | C1 `forward_invariance` resolver plus exact C2 logical-key validation | +| C2 transforms | `build_config_matrix`: fixed 2×2 matrix, permutation, packing, left/right padding | +| Diagnostics | tensor name, config pair, max/mean absolute error, max relative error | +| Backend provenance | profile, requested/actual backend, candidate/kernel id, device, CC, dtype, seed, fallback reason | +| Silent/cross-profile fallback | missing or mismatched provenance fails; CLI rejects candidate/profile mismatch | +| Selected-logprob smoke | C1 `max_abs_dlogp`, `approx_kl0`, and `clipfrac0` verdict | +| CUDA and Triton same schema | one API/CLI/report schema; both profile contracts are parametrically tested | +| No private thresholds | all tensor and aggregate thresholds resolve through the C1 contract | + +CPU-safe contract regression: + +```bash +python -m pytest -q \ + tests/test_tolerance_contract.py \ + tests/test_ws1_workload.py \ + tests/test_forward_invariance.py \ + tests/test_op_checks.py +``` + +Required-profile runtime examples (must run on CUDA hardware and must not be skipped): + +```bash +python scripts/check_forward_invariance.py \ + --op logp --candidate cuda \ + --backend-profile cuda_bf16 --json + +python scripts/check_forward_invariance.py \ + --op batch_invariant_logp --candidate triton \ + --backend-profile triton_cuda_bf16 --json +``` + +The CLI exits red when CUDA is unavailable, a candidate is absent, the C2 node is +`missing_required`, the compute capability cannot run a declared SM90 candidate, provenance +does not match the profile, or any accuracy/invariance/logprob verdict fails. + +## Runtime verification + +Verified on NVIDIA GeForce RTX 3060 Laptop GPU (`sm86`) with PyTorch 2.8.0+cu128: + +| Gate | Result | +| --- | --- | +| Full pytest suite | `1524 passed, 121 skipped` | +| Full pre-commit | trailing whitespace, EOF, YAML, large-file, black, isort, flake8 passed | +| `cuda_bf16` / generic CUDA logp C3 matrix | passed; all invariance max-abs errors `0.0` | +| `triton_cuda_bf16` / Triton batch-invariant-logp C3 matrix | passed; all invariance max-abs errors `0.0` | +| CUDA operator accuracy check | passed, max absolute error `0.0287590` | +| Triton operator accuracy check | passed, max absolute error `9.536743e-07` | + +The CUDA profile uses the manifest-declared generic CUDA logp candidate on SM86. No SM90 +candidate or fallback path is claimed on this device. + +## Parent boundary + +This closes only C3. It supplies the report and canonicalization contract that C10 must reuse. +It does not claim the full-model, backward, KV-cache, or CI EXIT requirements of #266. diff --git a/rl_engine/kernels/gtest/__init__.py b/rl_engine/kernels/gtest/__init__.py index a12db99e..0fa103c9 100644 --- a/rl_engine/kernels/gtest/__init__.py +++ b/rl_engine/kernels/gtest/__init__.py @@ -1,6 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026 RL-Kernel Contributors +from .forward_invariance import ( + AccuracyReport, + ConfigSpec, + ForwardInvarianceReport, + InvarianceReport, + LogprobSmokeResult, + TensorComparisonDetail, + assert_forward_batch_invariant, + build_config_matrix, +) from .op_checks import CandidateSpec, OperatorCase, run_operator_suite from .tolerance import ( BackendProvenance, @@ -15,8 +25,16 @@ ) __all__ = [ + "AccuracyReport", "CandidateSpec", + "ConfigSpec", + "ForwardInvarianceReport", + "InvarianceReport", + "LogprobSmokeResult", "OperatorCase", + "TensorComparisonDetail", + "assert_forward_batch_invariant", + "build_config_matrix", "run_operator_suite", "BackendProvenance", "ContractError", diff --git a/rl_engine/kernels/gtest/forward_invariance.py b/rl_engine/kernels/gtest/forward_invariance.py new file mode 100644 index 00000000..f837334d --- /dev/null +++ b/rl_engine/kernels/gtest/forward_invariance.py @@ -0,0 +1,726 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""WS1 C3 (#269): Forward config-invariance and backend provenance harness. + +Provides a shared forward accuracy/invariance API so downstream gates (C8, C10) +do not invent private thresholds, canonicalize wrong tokens, or compare outputs +from silently-fallback backends. + +All thresholds come from the C1 tolerance contract. Logical identity and config +transforms come from the C2 canonical workload. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass +from typing import Any + +import torch + +from rl_engine.kernels.gtest.tolerance import ( + BackendProvenance, + ContractResolveError, + LogprobAggregateVerdict, +) +from rl_engine.kernels.gtest.tolerance import _dtype_name as _normalize_dtype_name +from rl_engine.kernels.gtest.tolerance import ( + compute_logprob_aggregates, + default_clip_interval, + judge_logprob_aggregates, + load_contract, + resolve_comparison_roles, + resolve_tolerance, + validate_backend_provenance, +) +from rl_engine.testing.ws1_workload import ( + LogicalBatch, + PaddedBatch, + PhysicalLayout, + WS1Manifest, + apply_chunking, + apply_packing, + apply_padding, + batch_permutation_from_manifest, + build_logical_batch, + chunk_plan_from_manifest, + load_manifest, + permute_batch, + restore_logical_order, + restore_logical_order_from_padded, +) + + +@dataclass(frozen=True) +class ConfigSpec: + """One workload configuration (batch/chunk/padding/packing variant).""" + + config_id: str + transform_kind: str + logical_batch: LogicalBatch + physical_layout: PhysicalLayout | PaddedBatch + is_canonical: bool = False + + +@dataclass(frozen=True) +class TensorComparisonDetail: + """Per-tensor comparison result with full diagnostics.""" + + tensor_name: str + config_pair: tuple[str, str] + shape: tuple[int, ...] + dtype: str + max_abs_error: float + mean_abs_error: float + max_rel_error: float + atol: float + rtol: float + passed: bool + judgment: str + comparison_lhs_role: str + comparison_rhs_role: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class AccuracyReport: + """Forward accuracy: bf16_candidate vs fp32_reference.""" + + config_id: str + op_class: str + dtype: str + backend_profile: str + details: tuple[TensorComparisonDetail, ...] + passed: bool + backend_provenance: BackendProvenance | None = None + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + if self.backend_provenance is not None: + data["backend_provenance"] = self.backend_provenance.to_dict() + return data + + +@dataclass(frozen=True) +class InvarianceReport: + """Forward invariance: transformed vs canonical (bitwise atol=0 rtol=0).""" + + canonical_config_id: str + transformed_config_id: str + transform_kind: str + op_class: str + dtype: str + backend_profile: str + details: tuple[TensorComparisonDetail, ...] + passed: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class LogprobSmokeResult: + """Selected-logprob aggregate smoke on fixed workload.""" + + config_id: str + backend_profile: str + verdict: LogprobAggregateVerdict + passed: bool + + def to_dict(self) -> dict[str, Any]: + return { + "config_id": self.config_id, + "backend_profile": self.backend_profile, + "verdict": self.verdict.to_dict(), + "passed": self.passed, + } + + +@dataclass(frozen=True) +class ForwardInvarianceReport: + """Suite-level report combining accuracy, invariance, and logprob smoke.""" + + op_name: str + backend_profile: str + accuracy_reports: tuple[AccuracyReport, ...] + invariance_reports: tuple[InvarianceReport, ...] + logprob_smoke: LogprobSmokeResult | None + backend_provenance: BackendProvenance | None + candidate_id: str + device: str + compute_capability: str | None + seed: int + fallback_reason: str | None + passed: bool + provenance_valid: bool + metadata_valid: bool + + def to_dict(self) -> dict[str, Any]: + return { + "op_name": self.op_name, + "backend_profile": self.backend_profile, + "accuracy_reports": [r.to_dict() for r in self.accuracy_reports], + "invariance_reports": [r.to_dict() for r in self.invariance_reports], + "logprob_smoke": (self.logprob_smoke.to_dict() if self.logprob_smoke else None), + "backend_provenance": ( + self.backend_provenance.to_dict() if self.backend_provenance else None + ), + "candidate_id": self.candidate_id, + "device": self.device, + "compute_capability": self.compute_capability, + "seed": self.seed, + "fallback_reason": self.fallback_reason, + "passed": self.passed, + "provenance_valid": self.provenance_valid, + "metadata_valid": self.metadata_valid, + } + + +def build_config_matrix( + manifest: WS1Manifest | None = None, +) -> list[ConfigSpec]: + """Build the C2 primary 2x2 matrix + permutation + padding + packing configs.""" + + m = manifest if manifest is not None else load_manifest() + chunk_plan = chunk_plan_from_manifest(m) + batch_bn = build_logical_batch(m) + configs: list[ConfigSpec] = [] + + packed_bn = apply_packing(batch_bn) + configs.append( + ConfigSpec( + config_id="BN/full", + transform_kind="canonical", + logical_batch=batch_bn, + physical_layout=packed_bn, + is_canonical=True, + ) + ) + + chunked_bn = apply_chunking(batch_bn, chunk_size=chunk_plan.chunk_size) + configs.append( + ConfigSpec( + config_id="BN/chunked", + transform_kind="chunk", + logical_batch=batch_bn, + physical_layout=chunked_bn, + ) + ) + + for sample in batch_bn.samples: + single_batch = LogicalBatch( + workload_id=batch_bn.workload_id, + seed=batch_bn.seed, + samples=(sample,), + cell_id="B1-singleton_aggregate/full", + ) + packed_single = apply_packing(single_batch) + configs.append( + ConfigSpec( + config_id=f"B1-singleton_aggregate/full/{sample.sample_id}", + transform_kind="batch_size", + logical_batch=single_batch, + physical_layout=packed_single, + ) + ) + + for sample in batch_bn.samples: + single_batch = LogicalBatch( + workload_id=batch_bn.workload_id, + seed=batch_bn.seed, + samples=(sample,), + cell_id="B1-singleton_aggregate/chunked", + ) + chunked_single = apply_chunking(single_batch, chunk_size=chunk_plan.chunk_size) + configs.append( + ConfigSpec( + config_id=f"B1-singleton_aggregate/chunked/{sample.sample_id}", + transform_kind="chunk", + logical_batch=single_batch, + physical_layout=chunked_single, + ) + ) + + perm = batch_permutation_from_manifest(m) + permuted = permute_batch(batch_bn, perm) + packed_perm = apply_packing(permuted) + configs.append( + ConfigSpec( + config_id="BN/permuted", + transform_kind="permutation", + logical_batch=permuted, + physical_layout=packed_perm, + ) + ) + + padded_right = apply_padding(batch_bn, pad_side="right", manifest=m) + configs.append( + ConfigSpec( + config_id="BN/padded_right", + transform_kind="padding", + logical_batch=batch_bn, + physical_layout=padded_right, + is_canonical=False, + ) + ) + + padded_left = apply_padding(batch_bn, pad_side="left", manifest=m) + configs.append( + ConfigSpec( + config_id="BN/padded_left", + transform_kind="padding", + logical_batch=batch_bn, + physical_layout=padded_left, + is_canonical=False, + ) + ) + + return configs + + +def _compare_logical_tensors( + canonical: torch.Tensor, + transformed: torch.Tensor, + *, + judgment: str, + contract: Mapping[str, Any], + op_class: str, + dtype: str | torch.dtype, + backend_profile: str | None = None, + tensor_name: str = "output", + config_pair: tuple[str, str] = ("canonical", "transformed"), +) -> TensorComparisonDetail: + """Compare two tensors aligned to the same logical token order.""" + + spec = resolve_tolerance( + contract, + judgment=judgment, + op_class=op_class, + dtype=dtype, + backend_profile=backend_profile, + ) + atol, rtol = spec.atol, spec.rtol + roles = resolve_comparison_roles(contract, judgment) + + canonical_fp32 = canonical.float() + transformed_fp32 = transformed.float() + + if canonical_fp32.shape != transformed_fp32.shape: + return TensorComparisonDetail( + tensor_name=tensor_name, + config_pair=config_pair, + shape=tuple(transformed_fp32.shape), + dtype=_normalize_dtype_name(transformed.dtype), + max_abs_error=float("inf"), + mean_abs_error=float("inf"), + max_rel_error=float("inf"), + atol=atol, + rtol=rtol, + passed=False, + judgment=judgment, + comparison_lhs_role=roles.comparison_lhs_role, + comparison_rhs_role=roles.comparison_rhs_role, + ) + + abs_error = (canonical_fp32 - transformed_fp32).abs() + if abs_error.numel() == 0: + max_abs = 0.0 + mean_abs = 0.0 + max_rel = 0.0 + else: + max_abs = float(abs_error.max().item()) + mean_abs = float(abs_error.mean().item()) + rel_error = abs_error / canonical_fp32.abs().clamp_min(1e-12) + max_rel = float(rel_error.max().item()) + + passed = bool(torch.allclose(transformed_fp32, canonical_fp32, atol=atol, rtol=rtol)) + + return TensorComparisonDetail( + tensor_name=tensor_name, + config_pair=config_pair, + shape=tuple(canonical_fp32.shape), + dtype=_normalize_dtype_name(canonical.dtype), + max_abs_error=max_abs, + mean_abs_error=mean_abs, + max_rel_error=max_rel, + atol=atol, + rtol=rtol, + passed=passed, + judgment=judgment, + comparison_lhs_role=roles.comparison_lhs_role, + comparison_rhs_role=roles.comparison_rhs_role, + ) + + +def _validate_provenance( + contract: Mapping[str, Any], + provenance: BackendProvenance | None, + backend_profile: str, +) -> bool: + """Validate backend provenance; return False if silent/cross-profile fallback.""" + + if provenance is None: + return False + try: + validate_backend_provenance(contract, provenance) + except ContractResolveError: + return False + if provenance.backend_profile != backend_profile: + return False + return True + + +def _collect_logical_outputs( + op: Callable[..., Any] | Any, + config: ConfigSpec, + *, + op_kwargs: Mapping[str, Any] | None = None, +) -> dict[tuple[str, int], torch.Tensor]: + """Run op on a config and restore outputs to logical (sample_id, position) order.""" + + kwargs = dict(op_kwargs) if op_kwargs else {} + if hasattr(op, "forward") and callable(op.forward): + raw_output = op.forward(config=config, **kwargs) + else: + raw_output = op(config=config, **kwargs) + + if isinstance(raw_output, dict): + return raw_output + + if isinstance(raw_output, torch.Tensor): + if isinstance(config.physical_layout, PaddedBatch): + if raw_output.shape != ( + len(config.physical_layout.restore_map), + config.physical_layout.padded_len, + ): + raise ValueError( + f"padded output shape {tuple(raw_output.shape)} does not match " + f"({len(config.physical_layout.restore_map)}, " + f"{config.physical_layout.padded_len})" + ) + return restore_logical_order_from_padded(config.physical_layout, list(raw_output)) + flat = raw_output.reshape(-1) + return restore_logical_order(config.physical_layout, list(flat)) + + raise TypeError(f"op must return dict or Tensor, got {type(raw_output)!r}") + + +def _align_and_compare_invariance( + canonical_map: dict[tuple[str, int], Any], + transformed_map: dict[tuple[str, int], Any], + *, + contract: Mapping[str, Any], + op_class: str, + dtype: str | torch.dtype, + backend_profile: str | None, + canonical_id: str, + transformed_id: str, + tensor_name: str = "output", + expected_keys: set[tuple[str, int]], +) -> TensorComparisonDetail: + """Align two logical output maps and compare for bitwise invariance.""" + + canonical_keys = set(canonical_map) + transformed_keys = set(transformed_map) + if ( + not expected_keys + or not expected_keys.issubset(canonical_keys) + or not expected_keys.issubset(transformed_keys) + ): + return TensorComparisonDetail( + tensor_name=tensor_name, + config_pair=(canonical_id, transformed_id), + shape=(0,), + dtype=( + _normalize_dtype_name(dtype) + if isinstance(dtype, str) + else _normalize_dtype_name(dtype) + ), + max_abs_error=float("inf"), + mean_abs_error=float("inf"), + max_rel_error=float("inf"), + atol=0.0, + rtol=0.0, + passed=False, + judgment="forward_invariance", + comparison_lhs_role="transformed_config", + comparison_rhs_role="canonical_config", + ) + + shared_keys = sorted(expected_keys) + canonical_vals = torch.stack([torch.as_tensor(canonical_map[k]) for k in shared_keys]) + transformed_vals = torch.stack([torch.as_tensor(transformed_map[k]) for k in shared_keys]) + + return _compare_logical_tensors( + canonical_vals, + transformed_vals, + judgment="forward_invariance", + contract=contract, + op_class=op_class, + dtype=dtype, + backend_profile=backend_profile, + tensor_name=tensor_name, + config_pair=(canonical_id, transformed_id), + ) + + +def assert_forward_batch_invariant( + op: Callable[..., Any] | Any, + configs: Sequence[ConfigSpec] | None = None, + contract: Mapping[str, Any] | None = None, + *, + manifest: WS1Manifest | None = None, + backend_profile: str, + provenance: BackendProvenance | None = None, + gold_fn: Callable[..., Any] | None = None, + op_class: str = "logprob", + dtype: torch.dtype = torch.bfloat16, + op_name: str = "operator", + op_kwargs: Mapping[str, Any] | None = None, + include_logprob_smoke: bool = True, + active_only: bool = True, + candidate_id: str = "unspecified", + device: str = "unspecified", + compute_capability: str | None = None, + fallback_reason: str | None = None, +) -> ForwardInvarianceReport: + """Run forward config-invariance and accuracy checks. + + This is the sole C3 API. C10 must reuse this harness/report schema. + + Args: + op: Operator callable. Must accept (config=ConfigSpec, **op_kwargs) and + return either a dict[(sample_id, position) -> Tensor] or a flat Tensor. + configs: Config matrix; built from C2 manifest if None. + contract: C1 tolerance contract; loaded from default path if None. + manifest: C2 workload manifest; loaded from default path if None. + backend_profile: Required profile id (cuda_bf16 or triton_cuda_bf16). + provenance: Runtime-observed backend provenance. Missing provenance fails closed. + gold_fn: FP32 reference callable for accuracy checks. + op_class: Operator class for tolerance resolution. + dtype: Execution dtype. + op_name: Name for reporting. + op_kwargs: Extra kwargs passed to op. + include_logprob_smoke: Whether to run logprob aggregate smoke. + active_only: Only compare active (non-prompt) tokens for invariance. + + Returns: + ForwardInvarianceReport with accuracy, invariance, and logprob sub-reports. + """ + + loaded_contract = dict(contract or load_contract()) + m = manifest if manifest is not None else load_manifest() + config_list = list(configs) if configs is not None else build_config_matrix(m) + if not config_list: + raise ValueError("configs must contain at least one configuration") + if gold_fn is None: + raise ValueError("gold_fn is required for forward accuracy") + if include_logprob_smoke and op_class != "logprob": + raise ValueError("selected-logprob smoke requires op_class='logprob'") + + provenance_valid = _validate_provenance(loaded_contract, provenance, backend_profile) + if not provenance_valid and fallback_reason is None: + fallback_reason = "missing or contract-invalid backend provenance" + metadata_valid = ( + candidate_id != "unspecified" + and device != "unspecified" + and compute_capability is not None + and fallback_reason is None + ) + + canonical_config = next((c for c in config_list if c.is_canonical), config_list[0]) + canonical_outputs = _collect_logical_outputs(op, canonical_config, op_kwargs=op_kwargs) + + def expected_keys(config: ConfigSpec) -> set[tuple[str, int]]: + return set(config.logical_batch.logical_keys(active_only=active_only)) + + def validate_keys( + outputs: Mapping[tuple[str, int], Any], config: ConfigSpec, label: str + ) -> None: + required = expected_keys(config) + allowed = set(config.logical_batch.logical_keys(active_only=False)) + actual = set(outputs) + if not required.issubset(actual) or not actual.issubset(allowed): + raise ValueError( + f"{label} output keys for {config.config_id!r} do not match the " + "C2 logical identity" + ) + + canonical_keys = expected_keys(canonical_config) + validate_keys(canonical_outputs, canonical_config, "canonical") + + invariance_reports: list[InvarianceReport] = [] + for config in config_list: + if config.is_canonical: + continue + transformed_outputs = _collect_logical_outputs(op, config, op_kwargs=op_kwargs) + validate_keys(transformed_outputs, config, "transformed") + detail = _align_and_compare_invariance( + canonical_outputs, + transformed_outputs, + contract=loaded_contract, + op_class=op_class, + dtype=dtype, + backend_profile=backend_profile, + canonical_id=canonical_config.config_id, + transformed_id=config.config_id, + expected_keys=expected_keys(config), + ) + invariance_reports.append( + InvarianceReport( + canonical_config_id=canonical_config.config_id, + transformed_config_id=config.config_id, + transform_kind=config.transform_kind, + op_class=op_class, + dtype=_normalize_dtype_name(dtype), + backend_profile=backend_profile, + details=(detail,), + passed=detail.passed, + ) + ) + + accuracy_reports: list[AccuracyReport] = [] + for config in config_list: + candidate_outputs = ( + canonical_outputs + if config.is_canonical + else _collect_logical_outputs(op, config, op_kwargs=op_kwargs) + ) + gold_outputs = _collect_logical_outputs(gold_fn, config, op_kwargs=op_kwargs) + keys = expected_keys(config) + validate_keys(candidate_outputs, config, "candidate accuracy") + validate_keys(gold_outputs, config, "reference accuracy") + ordered_keys = sorted(keys) + candidate_vals = torch.stack([torch.as_tensor(candidate_outputs[k]) for k in ordered_keys]) + gold_vals = torch.stack([torch.as_tensor(gold_outputs[k]) for k in ordered_keys]) + acc_detail = _compare_logical_tensors( + gold_vals, + candidate_vals, + judgment="forward_accuracy", + contract=loaded_contract, + op_class=op_class, + dtype=dtype, + backend_profile=backend_profile, + tensor_name="selected_logprob" if op_class == "logprob" else "output", + config_pair=(config.config_id, "fp32_reference"), + ) + accuracy_reports.append( + AccuracyReport( + config_id=config.config_id, + op_class=op_class, + dtype=_normalize_dtype_name(dtype), + backend_profile=backend_profile, + details=(acc_detail,), + passed=acc_detail.passed, + backend_provenance=provenance, + ) + ) + + logprob_smoke: LogprobSmokeResult | None = None + if include_logprob_smoke: + logprob_smoke = _run_logprob_smoke( + canonical_outputs, + gold_fn, + canonical_config, + loaded_contract, + m, + backend_profile=backend_profile, + op_kwargs=op_kwargs, + active_keys=canonical_keys, + ) + + all_invariance_passed = all(r.passed for r in invariance_reports) + all_accuracy_passed = all(r.passed for r in accuracy_reports) + smoke_passed = logprob_smoke.passed if logprob_smoke is not None else True + + overall_passed = ( + all_invariance_passed + and all_accuracy_passed + and smoke_passed + and provenance_valid + and metadata_valid + ) + + return ForwardInvarianceReport( + op_name=op_name, + backend_profile=backend_profile, + accuracy_reports=tuple(accuracy_reports), + invariance_reports=tuple(invariance_reports), + logprob_smoke=logprob_smoke, + backend_provenance=provenance, + candidate_id=candidate_id, + device=device, + compute_capability=compute_capability, + seed=m.seed, + fallback_reason=fallback_reason, + passed=overall_passed, + provenance_valid=provenance_valid, + metadata_valid=metadata_valid, + ) + + +def _run_logprob_smoke( + candidate_outputs: dict[tuple[str, int], Any], + gold_fn: Callable[..., Any] | Any, + config: ConfigSpec, + contract: Mapping[str, Any], + manifest: WS1Manifest, + *, + backend_profile: str, + op_kwargs: Mapping[str, Any] | None = None, + active_keys: set[tuple[str, int]] | None = None, +) -> LogprobSmokeResult: + """Run selected-logprob aggregate smoke check.""" + + gold_outputs = _collect_logical_outputs(gold_fn, config, op_kwargs=op_kwargs) + if active_keys is not None: + shared = sorted(k for k in candidate_outputs if k in gold_outputs and k in active_keys) + else: + shared = sorted(k for k in candidate_outputs if k in gold_outputs) + + if not shared: + raise ContractResolveError("no shared active tokens for logprob smoke") + + lhs_logp = torch.stack([torch.as_tensor(candidate_outputs[k]).float() for k in shared]) + rhs_logp = torch.stack([torch.as_tensor(gold_outputs[k]).float() for k in shared]) + active_mask = torch.ones(len(shared), dtype=torch.bool) + + clip_interval = default_clip_interval(contract) + roles = resolve_comparison_roles(contract, "forward_accuracy") + + aggregates = compute_logprob_aggregates( + lhs_logp, + rhs_logp, + active_mask, + contract=contract, + report_kind="forward_accuracy", + clip_interval=clip_interval, + comparison_lhs_role=roles.comparison_lhs_role, + comparison_rhs_role=roles.comparison_rhs_role, + ) + verdict = judge_logprob_aggregates( + aggregates, + contract, + execution_dtype="bfloat16", + ) + return LogprobSmokeResult( + config_id=config.config_id, + backend_profile=backend_profile, + verdict=verdict, + passed=verdict.passed, + ) + + +__all__ = [ + "AccuracyReport", + "ConfigSpec", + "ForwardInvarianceReport", + "InvarianceReport", + "LogprobSmokeResult", + "TensorComparisonDetail", + "assert_forward_batch_invariant", + "build_config_matrix", +] diff --git a/scripts/check_forward_invariance.py b/scripts/check_forward_invariance.py new file mode 100644 index 00000000..d4f8dd9e --- /dev/null +++ b/scripts/check_forward_invariance.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Run the WS1 C3 selected-logprob forward invariance gate on a real GPU.""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +from typing import Any + +import torch + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from rl_engine.kernels.gtest import ( # noqa: E402 + BackendProvenance, + ConfigSpec, + assert_forward_batch_invariant, + load_contract, +) +from rl_engine.kernels.gtest.operator_specs import OP_SPECS, _load_object # noqa: E402 +from rl_engine.kernels.gtest.tolerance import resolve_dtype_policy # noqa: E402 +from rl_engine.testing.ws1_workload import PaddedBatch, load_manifest # noqa: E402 + + +def _object_path(value: Any) -> str: + cls = value.__class__ + return f"{cls.__module__}.{cls.__qualname__}" + + +def _profile_node(manifest: Any, profile: str, op_name: str) -> dict[str, Any]: + node_name = "logprob" if op_name == "logp" else op_name + nodes = manifest.backend_profiles[profile]["required_nodes"] + node = next((dict(item) for item in nodes if item["node"] == node_name), None) + if node is None: + raise RuntimeError(f"profile {profile!r} does not declare node {node_name!r}") + if node["status"] != "declared": + raise RuntimeError( + f"profile {profile!r} node {node_name!r} is {node['status']!r}; " + "missing required candidates are red, not fallback or N/A" + ) + return node + + +def _candidate_family(candidate: str) -> str: + if candidate.startswith("cuda"): + return "cuda" + if candidate == "triton": + return "triton" + return candidate + + +def _validate_candidate_selection( + *, manifest: Any, profile: str, op_name: str, candidate: str +) -> dict[str, Any]: + node = _profile_node(manifest, profile, op_name) + expected_family = manifest.backend_profiles[profile]["backend_family"] + actual_family = _candidate_family(candidate) + if actual_family != expected_family: + raise RuntimeError( + f"candidate {candidate!r} belongs to {actual_family!r}, but profile " + f"{profile!r} requires {expected_family!r}" + ) + if candidate != node["expected_backend_id"]: + raise RuntimeError( + f"candidate {candidate!r} does not match the C2 declaration " + f"{node['expected_backend_id']!r} for {profile}/{node['node']}" + ) + return node + + +def _physical_rows( + config: ConfigSpec, +) -> tuple[list[tuple[str, int] | None], tuple[int, ...]]: + layout = config.physical_layout + if isinstance(layout, PaddedBatch): + keys = [key for row in layout.restore_map for key in row] + return keys, (len(layout.restore_map), layout.padded_len) + return list(layout.restore_map), (len(layout.restore_map),) + + +def _make_inputs( + config: ConfigSpec, + *, + device: torch.device, + dtype: torch.dtype, + vocab_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Create row-local deterministic logits from C2 logical identity.""" + + keys, leading_shape = _physical_rows(config) + vocab_axis = torch.arange(vocab_size, device=device, dtype=torch.int64) + rows: list[torch.Tensor] = [] + targets: list[int] = [] + token_by_key = { + (token.sample_id, token.token_position): token.token_id + for sample in config.logical_batch.samples + for token in sample.tokens() + } + for key in keys: + if key is None: + position, token_id = 0, 0 + else: + position = key[1] + token_id = token_by_key[key] + # Integer construction makes each logical row independent of batching, + # chunking, permutation, padding, and RNG consumption order. + values = ((vocab_axis + token_id * 17 + position * 13) % 257) - 128 + rows.append((values.to(torch.float32) / 1024.0).to(dtype)) + targets.append(token_id % vocab_size) + logits = torch.stack(rows).reshape(leading_shape + (vocab_size,)) + target_tensor = torch.tensor(targets, device=device, dtype=torch.long).reshape(leading_shape) + return logits, target_tensor + + +def _make_runner( + operator: Any, + *, + device: torch.device, + dtype: torch.dtype, + vocab_size: int, + reference: bool, +): + def run(config: ConfigSpec, **_: Any) -> torch.Tensor: + logits, targets = _make_inputs(config, device=device, dtype=dtype, vocab_size=vocab_size) + if reference: + logits = logits.float() + return operator(logits, targets) + + return run + + +def _summarize(report: Any) -> None: + print( + f"op={report.op_name} profile={report.backend_profile} " + f"candidate={report.candidate_id} passed={report.passed}" + ) + print( + f" device={report.device} cc={report.compute_capability} seed={report.seed} " + f"provenance_valid={report.provenance_valid}" + ) + for acc in report.accuracy_reports: + detail = acc.details[0] + print( + f" accuracy config={acc.config_id} max_abs={detail.max_abs_error:.8e} " + f"max_rel={detail.max_rel_error:.8e} passed={acc.passed}" + ) + for inv in report.invariance_reports: + detail = inv.details[0] + print( + f" invariance pair={detail.config_pair} transform={inv.transform_kind} " + f"max_abs={detail.max_abs_error:.8e} passed={inv.passed}" + ) + if report.logprob_smoke is not None: + print(f" selected_logprob_smoke passed={report.logprob_smoke.passed}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="WS1 C3 forward invariance GPU gate") + parser.add_argument("--op", choices=("logp", "batch_invariant_logp"), default="logp") + parser.add_argument( + "--candidate", required=True, help="Manifest-declared CUDA/Triton candidate" + ) + parser.add_argument( + "--backend-profile", + choices=("cuda_bf16", "triton_cuda_bf16"), + required=True, + ) + parser.add_argument("--device", default="cuda") + parser.add_argument("--vocab", type=int, default=151936) + parser.add_argument("--json", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + device = torch.device(args.device) + if device.type != "cuda" or not torch.cuda.is_available(): + raise SystemExit("ERROR: C3 required-profile evidence requires an available CUDA device") + if args.vocab <= 240: + raise SystemExit("ERROR: --vocab must cover every fixed C2 workload token id") + + contract = load_contract() + manifest = load_manifest() + node = _validate_candidate_selection( + manifest=manifest, + profile=args.backend_profile, + op_name=args.op, + candidate=args.candidate, + ) + spec = OP_SPECS[args.op] + if args.candidate not in spec.candidate_paths: + raise SystemExit(f"ERROR: operator {args.op!r} has no candidate {args.candidate!r}") + + candidate_op = _load_object(spec.candidate_paths[args.candidate])() + gold_op = _load_object(spec.gold_path)() + gold_method = getattr(gold_op, spec.gold_method) + policy = resolve_dtype_policy(contract) + family = _candidate_family(args.candidate) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + cc_tuple = torch.cuda.get_device_capability(device) + cc = f"sm{cc_tuple[0]}{cc_tuple[1]}" + if args.candidate == "cuda-sm90" and cc_tuple[0] != 9: + raise SystemExit( + "ERROR: cuda-sm90 candidate requested on non-SM90 hardware; fallback forbidden" + ) + + provenance = BackendProvenance( + backend_profile=args.backend_profile, + requested_backend=manifest.backend_profiles[args.backend_profile]["backend_family"], + actual_backend=family, + execution_dtype=policy.execution_dtype, + accumulation_dtype=policy.accumulation_dtype, + output_dtype=policy.output_dtype_default, + reference_dtype=policy.reference_dtype, + candidate_tf32_enabled=torch.backends.cuda.matmul.allow_tf32, + reference_tf32_enabled=torch.backends.cuda.matmul.allow_tf32, + ) + report = assert_forward_batch_invariant( + _make_runner( + candidate_op, + device=device, + dtype=torch.bfloat16, + vocab_size=args.vocab, + reference=False, + ), + contract=contract, + manifest=manifest, + backend_profile=args.backend_profile, + provenance=provenance, + gold_fn=_make_runner( + gold_method, + device=device, + dtype=torch.bfloat16, + vocab_size=args.vocab, + reference=True, + ), + op_class="logprob", + dtype=torch.bfloat16, + op_name=args.op, + candidate_id=f"{_object_path(candidate_op)}::{node['expected_kernel_config_id']}", + device=f"{device}:{torch.cuda.get_device_name(device)}", + compute_capability=cc, + ) + + if args.json: + print(json.dumps(report.to_dict(), indent=2, default=str)) + else: + _summarize(report) + if not report.passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_forward_invariance.py b/tests/test_forward_invariance.py new file mode 100644 index 00000000..d3f89ed8 --- /dev/null +++ b/tests/test_forward_invariance.py @@ -0,0 +1,462 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Unit tests for WS1 C3 forward config-invariance harness.""" + +from __future__ import annotations + +from typing import Any + +import pytest +import torch + +from rl_engine.kernels.gtest.forward_invariance import ( + ConfigSpec, + ForwardInvarianceReport, + TensorComparisonDetail, + _validate_provenance, +) +from rl_engine.kernels.gtest.forward_invariance import ( + assert_forward_batch_invariant as _assert_forward_batch_invariant, +) +from rl_engine.kernels.gtest.forward_invariance import build_config_matrix +from rl_engine.kernels.gtest.tolerance import BackendProvenance, load_contract, resolve_tolerance +from rl_engine.testing.ws1_workload import LogicalBatch, LogicalSample, PaddedBatch, load_manifest + + +def assert_forward_batch_invariant(*args: Any, **kwargs: Any) -> ForwardInvarianceReport: + """Supply explicit synthetic runtime metadata for CPU-safe harness tests.""" + + kwargs.setdefault("candidate_id", "synthetic-test-candidate") + kwargs.setdefault("device", "cpu:test-double") + kwargs.setdefault("compute_capability", "synthetic") + return _assert_forward_batch_invariant(*args, **kwargs) + + +@pytest.fixture() +def contract() -> dict[str, Any]: + return load_contract() + + +@pytest.fixture() +def manifest(): + return load_manifest() + + +@pytest.fixture() +def simple_batch() -> LogicalBatch: + samples = ( + LogicalSample(sample_id="s0", token_ids=(1, 2, 3, 4), prompt_len=2, seq_len=4), + LogicalSample(sample_id="s1", token_ids=(5, 6, 7, 8), prompt_len=1, seq_len=4), + ) + return LogicalBatch(workload_id="test", seed=42, samples=samples) + + +def _make_identity_op(value: float = 1.0): + """Op that returns identical outputs regardless of config (batch-invariant).""" + + def op(config: ConfigSpec, **kwargs: Any) -> dict[tuple[str, int], torch.Tensor]: + result: dict[tuple[str, int], torch.Tensor] = {} + for sample in config.logical_batch.samples: + for tok in sample.active_tokens(): + result[(tok.sample_id, tok.token_position)] = torch.tensor( + value, dtype=torch.bfloat16 + ) + return result + + return op + + +def _make_drifting_op(drift: float = 0.1): + """Op that adds drift per sample to break invariance.""" + + def op(config: ConfigSpec, **kwargs: Any) -> dict[tuple[str, int], torch.Tensor]: + result: dict[tuple[str, int], torch.Tensor] = {} + for idx, sample in enumerate(config.logical_batch.samples): + for tok in sample.active_tokens(): + result[(tok.sample_id, tok.token_position)] = torch.tensor( + 1.0 + idx * drift, dtype=torch.bfloat16 + ) + return result + + return op + + +def _make_provenance( + backend_profile: str = "cuda_bf16", + requested: str = "cuda", + actual: str = "cuda", +) -> BackendProvenance: + return BackendProvenance( + backend_profile=backend_profile, + requested_backend=requested, + actual_backend=actual, + execution_dtype="bfloat16", + accumulation_dtype="float32", + output_dtype="bfloat16", + reference_dtype="float32", + candidate_tf32_enabled=False, + reference_tf32_enabled=False, + ) + + +class TestReportStructure: + def test_accuracy_and_invariance_reported_separately(self, contract, manifest): + op = _make_identity_op() + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(1.0), + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + assert isinstance(report, ForwardInvarianceReport) + assert hasattr(report, "accuracy_reports") + assert hasattr(report, "invariance_reports") + assert isinstance(report.accuracy_reports, tuple) + assert isinstance(report.invariance_reports, tuple) + assert len(report.invariance_reports) > 0 + assert len(report.accuracy_reports) == len(build_config_matrix(manifest)) + + def test_report_contains_required_runtime_metadata(self, contract, manifest): + report = assert_forward_batch_invariant( + _make_identity_op(), + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + op_class="logprob", + include_logprob_smoke=False, + candidate_id="cuda-test-kernel", + device="cuda:0:test-device", + compute_capability="sm90", + ) + payload = report.to_dict() + assert payload["candidate_id"] == "cuda-test-kernel" + assert payload["device"] == "cuda:0:test-device" + assert payload["compute_capability"] == "sm90" + assert payload["seed"] == manifest.seed + assert payload["fallback_reason"] is None + + def test_missing_runtime_metadata_fails_closed(self, contract, manifest): + report = _assert_forward_batch_invariant( + _make_identity_op(), + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + include_logprob_smoke=False, + ) + assert report.provenance_valid + assert not report.metadata_valid + assert not report.passed + + def test_report_contains_max_abs_rel_tensor_name(self, contract, manifest): + op = _make_identity_op() + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + for inv in report.invariance_reports: + for detail in inv.details: + assert isinstance(detail, TensorComparisonDetail) + assert detail.tensor_name is not None + assert detail.max_abs_error is not None + assert detail.max_rel_error is not None + assert detail.config_pair is not None + assert len(detail.config_pair) == 2 + + +class TestInvariance: + def test_invariance_bitwise_zero_tolerance(self, contract, manifest): + op = _make_identity_op() + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + for inv in report.invariance_reports: + for detail in inv.details: + assert detail.judgment == "forward_invariance" + assert detail.atol == 0.0 + assert detail.rtol == 0.0 + + def test_identity_op_passes_invariance(self, contract, manifest): + op = _make_identity_op() + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + for inv in report.invariance_reports: + assert inv.passed, f"invariance failed for {inv.transformed_config_id}" + assert report.passed + + def test_logical_unpadding_before_compare(self, contract, manifest): + op = _make_identity_op() + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + active_only=True, + ) + for inv in report.invariance_reports: + assert inv.passed + + def test_padding_configs_use_c2_padded_layout(self, manifest): + padded = [c for c in build_config_matrix(manifest) if c.transform_kind == "padding"] + assert {c.physical_layout.pad_side for c in padded} == {"left", "right"} + assert all(isinstance(c.physical_layout, PaddedBatch) for c in padded) + + def test_missing_active_token_hard_fails(self, contract, manifest): + def incomplete(config: ConfigSpec, **kwargs: Any): + result = _make_identity_op()(config, **kwargs) + result.pop(next(iter(result))) + return result + + with pytest.raises(ValueError, match="C2 logical identity"): + assert_forward_batch_invariant( + incomplete, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + include_logprob_smoke=False, + ) + + def test_padded_tensor_is_logically_unpadded(self, contract, manifest): + def physical_identity(config: ConfigSpec, **kwargs: Any): + layout = config.physical_layout + if isinstance(layout, PaddedBatch): + return torch.ones( + (len(layout.restore_map), layout.padded_len), dtype=torch.bfloat16 + ) + return torch.ones(len(layout.restore_map), dtype=torch.bfloat16) + + report = assert_forward_batch_invariant( + physical_identity, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=physical_identity, + include_logprob_smoke=False, + ) + padding_reports = [r for r in report.invariance_reports if r.transform_kind == "padding"] + assert len(padding_reports) == 2 + assert all(r.passed for r in padding_reports) + + +class TestAccuracy: + def test_missing_reference_is_rejected(self, contract, manifest): + with pytest.raises(ValueError, match="gold_fn is required"): + assert_forward_batch_invariant( + _make_identity_op(), + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=None, + include_logprob_smoke=False, + ) + + def test_accuracy_uses_c1_tolerances(self, contract, manifest): + op = _make_identity_op(1.0) + gold = _make_identity_op(1.0) + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=gold, + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + for acc in report.accuracy_reports: + for detail in acc.details: + assert detail.judgment == "forward_accuracy" + spec = resolve_tolerance( + contract, + judgment="forward_accuracy", + op_class="logprob", + dtype=torch.bfloat16, + backend_profile="cuda_bf16", + ) + assert detail.atol == spec.atol + assert detail.rtol == spec.rtol + + def test_no_private_thresholds(self, contract, manifest): + op = _make_identity_op(1.0) + gold = _make_identity_op(1.0) + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=gold, + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + for acc in report.accuracy_reports: + for detail in acc.details: + spec = resolve_tolerance( + contract, + judgment=detail.judgment, + op_class=acc.op_class, + dtype=torch.bfloat16, + backend_profile=acc.backend_profile, + ) + assert detail.atol == spec.atol + assert detail.rtol == spec.rtol + + +class TestBackendProvenance: + def test_valid_provenance_passes(self, contract): + provenance = _make_provenance("cuda_bf16", "cuda", "cuda") + assert _validate_provenance(contract, provenance, "cuda_bf16") is True + + def test_silent_fallback_rejected(self, contract): + provenance = _make_provenance("cuda_bf16", "cuda", "triton") + assert _validate_provenance(contract, provenance, "cuda_bf16") is False + + def test_cross_profile_fallback_rejected(self, contract): + provenance = _make_provenance("triton_cuda_bf16", "triton", "triton") + assert _validate_provenance(contract, provenance, "cuda_bf16") is False + + def test_none_provenance_fails_closed(self, contract): + assert _validate_provenance(contract, None, "cuda_bf16") is False + + @pytest.mark.parametrize( + ("profile", "family"), + [("cuda_bf16", "cuda"), ("triton_cuda_bf16", "triton")], + ) + def test_required_profiles_share_report_schema(self, contract, manifest, profile, family): + provenance = _make_provenance(profile, family, family) + report = assert_forward_batch_invariant( + _make_identity_op(), + contract=contract, + manifest=manifest, + backend_profile=profile, + provenance=provenance, + gold_fn=_make_identity_op(), + include_logprob_smoke=False, + ) + assert report.passed + assert set(report.to_dict()) == set( + ForwardInvarianceReport( + op_name="x", + backend_profile=profile, + accuracy_reports=(), + invariance_reports=(), + logprob_smoke=None, + backend_provenance=provenance, + candidate_id="x", + device="x", + compute_capability=None, + seed=manifest.seed, + fallback_reason=None, + passed=True, + provenance_valid=True, + metadata_valid=True, + ).to_dict() + ) + + def test_provenance_failure_fails_report(self, contract, manifest): + op = _make_identity_op() + bad_provenance = _make_provenance("cuda_bf16", "cuda", "triton") + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=bad_provenance, + gold_fn=_make_identity_op(), + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=False, + ) + assert report.provenance_valid is False + assert report.passed is False + + +class TestConfigMatrix: + def test_config_matrix_covers_c2_cells(self, manifest): + configs = build_config_matrix(manifest) + config_ids = [c.config_id for c in configs] + assert any("BN/full" in cid for cid in config_ids) + assert any("BN/chunked" in cid for cid in config_ids) + assert any("B1-singleton_aggregate/full" in cid for cid in config_ids) + assert any("B1-singleton_aggregate/chunked" in cid for cid in config_ids) + assert any("permuted" in cid for cid in config_ids) + assert any("padded_right" in cid for cid in config_ids) + assert any("padded_left" in cid for cid in config_ids) + + def test_canonical_config_exists(self, manifest): + configs = build_config_matrix(manifest) + canonical = [c for c in configs if c.is_canonical] + assert len(canonical) == 1 + assert canonical[0].config_id == "BN/full" + + +class TestLogprobSmoke: + def test_logprob_smoke_passes_for_identical(self, contract, manifest): + op = _make_identity_op(0.0) + gold = _make_identity_op(0.0) + report = assert_forward_batch_invariant( + op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=gold, + op_class="logprob", + dtype=torch.bfloat16, + op_name="test_op", + include_logprob_smoke=True, + ) + assert report.logprob_smoke is not None + assert report.logprob_smoke.passed From 2e4a30ab0af5e5397d4bd7eb21d5ba4cc17e8211 Mon Sep 17 00:00:00 2001 From: maxiaosong1124 Date: Wed, 12 Aug 2026 18:38:28 +0800 Subject: [PATCH 10/10] fix ws1 provenance review issues --- rl_engine/kernels/gtest/__init__.py | 2 + rl_engine/kernels/gtest/forward_invariance.py | 77 ++++++++++++++++--- rl_engine/kernels/gtest/op_checks.py | 31 +++----- rl_engine/kernels/gtest/tolerance.py | 26 +++++-- rl_engine/testing/ws1_workload.py | 23 ++++-- scripts/check_forward_invariance.py | 22 +++++- scripts/ws1_candidate_evidence.py | 60 +++++++++------ scripts/ws1_reference.py | 2 +- tests/test_forward_invariance.py | 45 +++++++++++ tests/test_ws1_workload.py | 2 +- 10 files changed, 221 insertions(+), 69 deletions(-) diff --git a/rl_engine/kernels/gtest/__init__.py b/rl_engine/kernels/gtest/__init__.py index 0fa103c9..9ab8967a 100644 --- a/rl_engine/kernels/gtest/__init__.py +++ b/rl_engine/kernels/gtest/__init__.py @@ -7,6 +7,7 @@ ForwardInvarianceReport, InvarianceReport, LogprobSmokeResult, + RuntimeObservation, TensorComparisonDetail, assert_forward_batch_invariant, build_config_matrix, @@ -31,6 +32,7 @@ "ForwardInvarianceReport", "InvarianceReport", "LogprobSmokeResult", + "RuntimeObservation", "OperatorCase", "TensorComparisonDetail", "assert_forward_batch_invariant", diff --git a/rl_engine/kernels/gtest/forward_invariance.py b/rl_engine/kernels/gtest/forward_invariance.py index f837334d..04d6fed2 100644 --- a/rl_engine/kernels/gtest/forward_invariance.py +++ b/rl_engine/kernels/gtest/forward_invariance.py @@ -23,13 +23,11 @@ BackendProvenance, ContractResolveError, LogprobAggregateVerdict, -) -from rl_engine.kernels.gtest.tolerance import _dtype_name as _normalize_dtype_name -from rl_engine.kernels.gtest.tolerance import ( compute_logprob_aggregates, default_clip_interval, judge_logprob_aggregates, load_contract, + normalize_dtype_name, resolve_comparison_roles, resolve_tolerance, validate_backend_provenance, @@ -51,6 +49,8 @@ restore_logical_order_from_padded, ) +_normalize_dtype_name = normalize_dtype_name + @dataclass(frozen=True) class ConfigSpec: @@ -63,6 +63,17 @@ class ConfigSpec: is_canonical: bool = False +@dataclass(frozen=True) +class RuntimeObservation: + """Runtime facts returned alongside one candidate output.""" + + output: Any + actual_backend: str + kernel_id: str + output_dtype: str + device: str + + @dataclass(frozen=True) class TensorComparisonDetail: """Per-tensor comparison result with full diagnostics.""" @@ -157,6 +168,7 @@ class ForwardInvarianceReport: passed: bool provenance_valid: bool metadata_valid: bool + observed_kernel_id: str | None = None def to_dict(self) -> dict[str, Any]: return { @@ -176,6 +188,7 @@ def to_dict(self) -> dict[str, Any]: "passed": self.passed, "provenance_valid": self.provenance_valid, "metadata_valid": self.metadata_valid, + "observed_kernel_id": self.observed_kernel_id, } @@ -378,7 +391,7 @@ def _collect_logical_outputs( config: ConfigSpec, *, op_kwargs: Mapping[str, Any] | None = None, -) -> dict[tuple[str, int], torch.Tensor]: +) -> tuple[dict[tuple[str, int], torch.Tensor], RuntimeObservation | None]: """Run op on a config and restore outputs to logical (sample_id, position) order.""" kwargs = dict(op_kwargs) if op_kwargs else {} @@ -387,8 +400,11 @@ def _collect_logical_outputs( else: raw_output = op(config=config, **kwargs) + observation = raw_output if isinstance(raw_output, RuntimeObservation) else None + if observation is not None: + raw_output = observation.output if isinstance(raw_output, dict): - return raw_output + return raw_output, observation if isinstance(raw_output, torch.Tensor): if isinstance(config.physical_layout, PaddedBatch): @@ -401,9 +417,12 @@ def _collect_logical_outputs( f"({len(config.physical_layout.restore_map)}, " f"{config.physical_layout.padded_len})" ) - return restore_logical_order_from_padded(config.physical_layout, list(raw_output)) + return ( + restore_logical_order_from_padded(config.physical_layout, list(raw_output)), + observation, + ) flat = raw_output.reshape(-1) - return restore_logical_order(config.physical_layout, list(flat)) + return restore_logical_order(config.physical_layout, list(flat)), observation raise TypeError(f"op must return dict or Tensor, got {type(raw_output)!r}") @@ -486,6 +505,9 @@ def assert_forward_batch_invariant( device: str = "unspecified", compute_capability: str | None = None, fallback_reason: str | None = None, + observed_actual_backend: str | None = None, + observed_kernel_id: str | None = None, + observed_output_dtype: str | None = None, ) -> ForwardInvarianceReport: """Run forward config-invariance and accuracy checks. @@ -530,9 +552,17 @@ def assert_forward_batch_invariant( and compute_capability is not None and fallback_reason is None ) + metadata_valid = metadata_valid and all( + value is not None + for value in (observed_actual_backend, observed_kernel_id, observed_output_dtype) + ) + if provenance is not None and observed_actual_backend is not None: + metadata_valid = metadata_valid and observed_actual_backend == provenance.actual_backend canonical_config = next((c for c in config_list if c.is_canonical), config_list[0]) - canonical_outputs = _collect_logical_outputs(op, canonical_config, op_kwargs=op_kwargs) + canonical_outputs, canonical_observation = _collect_logical_outputs( + op, canonical_config, op_kwargs=op_kwargs + ) def expected_keys(config: ConfigSpec) -> set[tuple[str, int]]: return set(config.logical_batch.logical_keys(active_only=active_only)) @@ -551,12 +581,34 @@ def validate_keys( canonical_keys = expected_keys(canonical_config) validate_keys(canonical_outputs, canonical_config, "canonical") + if canonical_observation is not None: + observed_device = str(canonical_observation.device) + report_device = str(device) + metadata_valid = metadata_valid and ( + provenance is not None + and canonical_observation.actual_backend == provenance.actual_backend + and canonical_observation.actual_backend == observed_actual_backend + and canonical_observation.kernel_id == observed_kernel_id + and _normalize_dtype_name(canonical_observation.output_dtype) + == _normalize_dtype_name(observed_output_dtype) + and ( + report_device == observed_device or report_device.startswith(observed_device + ":") + ) + and _normalize_dtype_name(canonical_observation.output_dtype) + == _normalize_dtype_name(next(iter(canonical_outputs.values())).dtype) + ) invariance_reports: list[InvarianceReport] = [] for config in config_list: if config.is_canonical: continue - transformed_outputs = _collect_logical_outputs(op, config, op_kwargs=op_kwargs) + transformed_outputs, observation = _collect_logical_outputs(op, config, op_kwargs=op_kwargs) + if canonical_observation is not None and observation is not None: + metadata_valid = metadata_valid and ( + observation.actual_backend == canonical_observation.actual_backend + and observation.kernel_id == canonical_observation.kernel_id + and observation.output_dtype == canonical_observation.output_dtype + ) validate_keys(transformed_outputs, config, "transformed") detail = _align_and_compare_invariance( canonical_outputs, @@ -587,9 +639,9 @@ def validate_keys( candidate_outputs = ( canonical_outputs if config.is_canonical - else _collect_logical_outputs(op, config, op_kwargs=op_kwargs) + else _collect_logical_outputs(op, config, op_kwargs=op_kwargs)[0] ) - gold_outputs = _collect_logical_outputs(gold_fn, config, op_kwargs=op_kwargs) + gold_outputs = _collect_logical_outputs(gold_fn, config, op_kwargs=op_kwargs)[0] keys = expected_keys(config) validate_keys(candidate_outputs, config, "candidate accuracy") validate_keys(gold_outputs, config, "reference accuracy") @@ -659,6 +711,7 @@ def validate_keys( passed=overall_passed, provenance_valid=provenance_valid, metadata_valid=metadata_valid, + observed_kernel_id=observed_kernel_id, ) @@ -675,7 +728,7 @@ def _run_logprob_smoke( ) -> LogprobSmokeResult: """Run selected-logprob aggregate smoke check.""" - gold_outputs = _collect_logical_outputs(gold_fn, config, op_kwargs=op_kwargs) + gold_outputs = _collect_logical_outputs(gold_fn, config, op_kwargs=op_kwargs)[0] if active_keys is not None: shared = sorted(k for k in candidate_outputs if k in gold_outputs and k in active_keys) else: diff --git a/rl_engine/kernels/gtest/op_checks.py b/rl_engine/kernels/gtest/op_checks.py index e9354cb6..6ae1b170 100644 --- a/rl_engine/kernels/gtest/op_checks.py +++ b/rl_engine/kernels/gtest/op_checks.py @@ -9,10 +9,11 @@ import torch -from rl_engine.kernels.gtest.tolerance import BackendProvenance, ContractResolveError -from rl_engine.kernels.gtest.tolerance import _dtype_name as _normalize_dtype_name from rl_engine.kernels.gtest.tolerance import ( + BackendProvenance, + ContractResolveError, load_contract, + normalize_dtype_name, resolve_tolerance, validate_backend_provenance, ) @@ -159,8 +160,8 @@ def _run_candidate( f"{candidate.provenance.actual_backend!r}" ) for case in cases: - case_dtype = _normalize_dtype_name(case.dtype) - provenance_dtype = _normalize_dtype_name(candidate.provenance.execution_dtype) + case_dtype = normalize_dtype_name(case.dtype) + provenance_dtype = normalize_dtype_name(candidate.provenance.execution_dtype) if case_dtype != provenance_dtype: raise ContractResolveError( f"case {case.name!r} dtype {case.dtype} does not match " @@ -337,14 +338,16 @@ def _compare_case_outputs( ) if candidate.provenance is not None: for candidate_output, gold_output in zip(candidate_outputs, gold_outputs, strict=True): - candidate_dtype = _dtype_name(candidate_output.dtype) - gold_dtype = _dtype_name(gold_output.dtype) - if candidate_dtype != candidate.provenance.output_dtype: + candidate_dtype = normalize_dtype_name(candidate_output.dtype) + gold_dtype = normalize_dtype_name(gold_output.dtype) + provenance_output_dtype = normalize_dtype_name(candidate.provenance.output_dtype) + provenance_reference_dtype = normalize_dtype_name(candidate.provenance.reference_dtype) + if candidate_dtype != provenance_output_dtype: raise ContractResolveError( f"candidate output dtype {candidate_dtype!r} disagrees with provenance " f"output_dtype {candidate.provenance.output_dtype!r}" ) - if gold_dtype != candidate.provenance.reference_dtype: + if gold_dtype != provenance_reference_dtype: raise ContractResolveError( f"gold output dtype {gold_dtype!r} disagrees with provenance " f"reference_dtype {candidate.provenance.reference_dtype!r}" @@ -518,7 +521,7 @@ def _resolve_tolerance( return float(spec.atol), float(spec.rtol) # Legacy fixtures used by some unit tests that inject a minimal contract. - dtype_name = _dtype_name(dtype) + dtype_name = normalize_dtype_name(dtype) if arch_key is not None: arch_values = ( contract["accuracy"] @@ -534,16 +537,6 @@ def _resolve_tolerance( return float(values["atol"]), float(values.get("rtol", 0.0)) -def _dtype_name(dtype: torch.dtype) -> str: - if dtype is torch.float32: - return "float32" - if dtype is torch.bfloat16: - return "bfloat16" - if dtype is torch.float16: - return "float16" - raise ValueError(f"unsupported dtype: {dtype}") - - def _compare_output( candidate: torch.Tensor, gold: torch.Tensor, diff --git a/rl_engine/kernels/gtest/tolerance.py b/rl_engine/kernels/gtest/tolerance.py index d9afdfae..1607e022 100644 --- a/rl_engine/kernels/gtest/tolerance.py +++ b/rl_engine/kernels/gtest/tolerance.py @@ -283,7 +283,7 @@ def validate_backend_provenance( "reference_dtype": policy.reference_dtype, } for field_name, expected in expected_dtypes.items(): - actual = _dtype_name(getattr(provenance, field_name)) + actual = normalize_dtype_name(getattr(provenance, field_name)) if actual != expected: raise ContractResolveError( f"backend provenance mismatch for {field_name}: expected " @@ -550,14 +550,17 @@ def compute_logprob_aggregates( if not (lo < hi): raise ContractResolveError(f"clip_interval requires lo < hi, got [{lo}, {hi}]") - lhs = torch.as_tensor(lhs_logp).detach().float().reshape(-1) - rhs = torch.as_tensor(rhs_logp).detach().float().reshape(-1) - mask = torch.as_tensor(active_mask).detach().reshape(-1).bool() - if lhs.shape != rhs.shape or lhs.shape != mask.shape: + lhs_tensor = torch.as_tensor(lhs_logp).detach().float() + rhs_tensor = torch.as_tensor(rhs_logp).detach().float() + mask_tensor = torch.as_tensor(active_mask).detach().bool() + if lhs_tensor.shape != rhs_tensor.shape or lhs_tensor.shape != mask_tensor.shape: raise ContractResolveError( - f"lhs/rhs/mask shape mismatch: {tuple(lhs.shape)} vs " - f"{tuple(rhs.shape)} vs {tuple(mask.shape)}" + f"lhs/rhs/mask shape mismatch: {tuple(lhs_tensor.shape)} vs " + f"{tuple(rhs_tensor.shape)} vs {tuple(mask_tensor.shape)}" ) + lhs = lhs_tensor.reshape(-1) + rhs = rhs_tensor.reshape(-1) + mask = mask_tensor.reshape(-1) active = int(mask.sum().item()) if active == 0: raise ContractResolveError("empty active-token set is a hard fail for logprob aggregates") @@ -918,7 +921,8 @@ def _lookup_cell( return base -def _dtype_name(dtype: str | Any) -> str: +def normalize_dtype_name(dtype: str | Any) -> str: + """Return the contract dtype name for a string alias or framework dtype.""" if isinstance(dtype, str): name = dtype # Accept torch-style aliases. @@ -996,6 +1000,11 @@ def _dtype_name(dtype: str | Any) -> str: raise ContractResolveError(f"unsupported dtype: {dtype!r}") +# Private compatibility alias for callers outside this package that have not +# migrated to the public normalizer yet. +_dtype_name = normalize_dtype_name + + __all__ = [ "ALL_DTYPES", "CHAIN_AGGREGATE_METRICS", @@ -1017,6 +1026,7 @@ def _dtype_name(dtype: str | Any) -> str: "default_clip_interval", "judge_logprob_aggregates", "load_contract", + "normalize_dtype_name", "resolve_chain_aggregate_thresholds", "resolve_comparison_roles", "resolve_dtype_policy", diff --git a/rl_engine/testing/ws1_workload.py b/rl_engine/testing/ws1_workload.py index d3ae608a..2ed3cca5 100644 --- a/rl_engine/testing/ws1_workload.py +++ b/rl_engine/testing/ws1_workload.py @@ -444,10 +444,13 @@ def _validate_primary_matrix(matrix: Mapping[str, Any], fixtures: Mapping[str, A mode = cell["batch_mode"] if mode not in ("singleton_aggregate", "batched"): raise WorkloadError(f"unknown batch_mode {mode!r}") - if mode == "singleton_aggregate" and "singleton_aggregate" in str( - cell.get("comparison_lhs_role", "") - ): - raise WorkloadError("singleton_aggregate must not be used as a comparison role") + for role_key in ("comparison_lhs_role", "comparison_rhs_role"): + role = str(cell.get(role_key, "")) + if role in _FORBIDDEN_COMPARISON_ROLES: + raise WorkloadError( + f"cell {cell.get('cell_id')!r}: {role_key} must not use " + f"forbidden comparison role {role!r}" + ) def _validate_fixtures(fixtures: Mapping[str, Any], matrix: Mapping[str, Any]) -> None: @@ -701,7 +704,17 @@ def _validate_fixture_case_bindings( }, } for case in cases: - required = expected_shapes[case["fixture_id"]][case["family"]] + fixture_shapes = expected_shapes.get(case["fixture_id"]) + if fixture_shapes is None: + raise WorkloadError( + f"case {case['case_id']}: unknown fixture_id {case['fixture_id']!r}" + ) + required = fixture_shapes.get(case["family"]) + if required is None: + raise WorkloadError( + f"case {case['case_id']}: fixture {case['fixture_id']!r} does not " + f"cover family {case['family']!r}" + ) mismatched = { key: (case["shape"].get(key), value) for key, value in required.items() diff --git a/scripts/check_forward_invariance.py b/scripts/check_forward_invariance.py index d4f8dd9e..d806e730 100644 --- a/scripts/check_forward_invariance.py +++ b/scripts/check_forward_invariance.py @@ -21,8 +21,10 @@ from rl_engine.kernels.gtest import ( # noqa: E402 BackendProvenance, ConfigSpec, + RuntimeObservation, assert_forward_batch_invariant, load_contract, + normalize_dtype_name, ) from rl_engine.kernels.gtest.operator_specs import OP_SPECS, _load_object # noqa: E402 from rl_engine.kernels.gtest.tolerance import resolve_dtype_policy # noqa: E402 @@ -126,12 +128,25 @@ def _make_runner( dtype: torch.dtype, vocab_size: int, reference: bool, + backend_family: str | None = None, + kernel_id: str | None = None, ): def run(config: ConfigSpec, **_: Any) -> torch.Tensor: logits, targets = _make_inputs(config, device=device, dtype=dtype, vocab_size=vocab_size) if reference: logits = logits.float() - return operator(logits, targets) + output = operator(logits, targets) + if reference: + return output + if backend_family is None or kernel_id is None: + raise RuntimeError("candidate telemetry must declare backend_family and kernel_id") + return RuntimeObservation( + output=output, + actual_backend=backend_family, + kernel_id=kernel_id, + output_dtype=normalize_dtype_name(output.dtype), + device=str(output.device), + ) return run @@ -230,6 +245,8 @@ def main() -> None: dtype=torch.bfloat16, vocab_size=args.vocab, reference=False, + backend_family=family, + kernel_id=_object_path(candidate_op), ), contract=contract, manifest=manifest, @@ -248,6 +265,9 @@ def main() -> None: candidate_id=f"{_object_path(candidate_op)}::{node['expected_kernel_config_id']}", device=f"{device}:{torch.cuda.get_device_name(device)}", compute_capability=cc, + observed_actual_backend=family, + observed_kernel_id=_object_path(candidate_op), + observed_output_dtype=policy.output_dtype_default, ) if args.json: diff --git a/scripts/ws1_candidate_evidence.py b/scripts/ws1_candidate_evidence.py index 7c3cccf6..dac3a932 100755 --- a/scripts/ws1_candidate_evidence.py +++ b/scripts/ws1_candidate_evidence.py @@ -32,8 +32,11 @@ def _object_path(value: Any) -> str: def _case_args(case: dict[str, Any], seed: int) -> SimpleNamespace: - shape = case["shape"] - operator_spec = case["operator_spec"] + try: + shape = case["shape"] + operator_spec = case["operator_spec"] + except KeyError as exc: + raise WorkloadError(f"candidate case missing {exc.args[0]!r}") from exc common: dict[str, Any] = { "op": operator_spec, "candidate": case["expected_backend_id"], @@ -48,23 +51,28 @@ def _case_args(case: dict[str, Any], seed: int) -> SimpleNamespace: "eps": 1.0e-6, "seed": seed, } - if operator_spec == "det_gemm": - common.update(batch=1, seq=shape["M"], k_dim=shape["K"], n_dim=shape["N"]) - elif operator_spec == "attention": - common.update( - batch=shape["B"], - seq=shape["Sq"], - skv=shape["Skv"], - n_heads=shape["Hq"], - n_kv_heads=shape["Hkv"], - causal=1, - use_padding=0, - scale_mode="default", - ) - elif operator_spec in {"logp", "batch_invariant_logp"}: - common.update(batch=shape["B"], seq=shape["T"], vocab=shape["vocab"]) - else: - raise WorkloadError(f"unsupported representative operator_spec {operator_spec!r}") + try: + if operator_spec == "det_gemm": + common.update(batch=1, seq=shape["M"], k_dim=shape["K"], n_dim=shape["N"]) + elif operator_spec == "attention": + common.update( + batch=shape["B"], + seq=shape["Sq"], + skv=shape["Skv"], + n_heads=shape["Hq"], + n_kv_heads=shape["Hkv"], + causal=1, + use_padding=0, + scale_mode="default", + ) + elif operator_spec in {"logp", "batch_invariant_logp"}: + common.update(batch=shape["B"], seq=shape["T"], vocab=shape["vocab"]) + else: + raise WorkloadError(f"unsupported representative operator_spec {operator_spec!r}") + except KeyError as exc: + raise WorkloadError( + f"case {case.get('case_id')!r} {operator_spec!r} shape missing {exc.args[0]!r}" + ) from exc return SimpleNamespace(**common) @@ -146,8 +154,9 @@ def main(argv: list[str] | None = None) -> int: if profiles.intersection(case["profile_ids"]) and (not selected_ids or case["case_id"] in selected_ids) ] - if selected_ids - {case["case_id"] for case in cases}: - unknown = sorted(selected_ids - {case["case_id"] for case in cases}) + resolved_ids = {case["case_id"] for case in cases} + if selected_ids - resolved_ids: + unknown = sorted(selected_ids - resolved_ids) raise WorkloadError(f"unknown or profile-filtered case IDs: {unknown}") device = torch.device("cuda:0") log_stream = sys.stderr if args.emit_json == "-" else sys.stdout @@ -156,7 +165,14 @@ def main(argv: list[str] | None = None) -> int: run_case(case, seed=manifest.seed + i, device=device) for i, case in enumerate(cases) ] - except (RuntimeError, ValueError, WorkloadError) as exc: + except ( + RuntimeError, + ValueError, + WorkloadError, + KeyError, + OSError, + json.JSONDecodeError, + ) as exc: print(f"error: {exc}", file=sys.stderr) return 2 diff --git a/scripts/ws1_reference.py b/scripts/ws1_reference.py index 7e677cba..5e81e579 100755 --- a/scripts/ws1_reference.py +++ b/scripts/ws1_reference.py @@ -88,7 +88,7 @@ def main(argv: list[str] | None = None) -> int: if args.seed is not None and int(args.seed) != manifest.seed: raise WorkloadError(f"--seed {args.seed} does not match manifest seed {manifest.seed}") payload = workload.reference_payload(manifest, cell_id=args.cell_id, dtype=args.dtype) - except WorkloadError as exc: + except (WorkloadError, KeyError, OSError, json.JSONDecodeError) as exc: print(f"error: {exc}", file=sys.stderr) return 2 diff --git a/tests/test_forward_invariance.py b/tests/test_forward_invariance.py index d3f89ed8..ae7cbd87 100644 --- a/tests/test_forward_invariance.py +++ b/tests/test_forward_invariance.py @@ -13,6 +13,7 @@ from rl_engine.kernels.gtest.forward_invariance import ( ConfigSpec, ForwardInvarianceReport, + RuntimeObservation, TensorComparisonDetail, _validate_provenance, ) @@ -30,6 +31,9 @@ def assert_forward_batch_invariant(*args: Any, **kwargs: Any) -> ForwardInvarian kwargs.setdefault("candidate_id", "synthetic-test-candidate") kwargs.setdefault("device", "cpu:test-double") kwargs.setdefault("compute_capability", "synthetic") + kwargs.setdefault("observed_actual_backend", kwargs["provenance"].actual_backend) + kwargs.setdefault("observed_kernel_id", "synthetic-test-candidate") + kwargs.setdefault("observed_output_dtype", kwargs["provenance"].output_dtype) return _assert_forward_batch_invariant(*args, **kwargs) @@ -422,6 +426,47 @@ def test_provenance_failure_fails_report(self, contract, manifest): assert report.provenance_valid is False assert report.passed is False + @pytest.mark.parametrize( + ("field", "value"), + [ + ("observed_actual_backend", "triton"), + ("observed_kernel_id", "other-kernel"), + ("observed_output_dtype", "float32"), + ], + ) + def test_runtime_observation_mismatch_fails_closed(self, contract, manifest, field, value): + kwargs = { + "observed_actual_backend": "cuda", + "observed_kernel_id": "synthetic-test-candidate", + "observed_output_dtype": "bfloat16", + } + kwargs[field] = value + + def observed_op(config: ConfigSpec, **kwargs: Any): + return RuntimeObservation( + output=_make_identity_op()(config, **kwargs), + actual_backend="cuda", + kernel_id="synthetic-test-candidate", + output_dtype="bfloat16", + device="cpu:test-double", + ) + + report = _assert_forward_batch_invariant( + observed_op, + contract=contract, + manifest=manifest, + backend_profile="cuda_bf16", + provenance=_make_provenance(), + gold_fn=_make_identity_op(), + include_logprob_smoke=False, + candidate_id="synthetic-test-candidate", + device="cpu:test-double", + compute_capability="synthetic", + **kwargs, + ) + assert report.metadata_valid is False + assert report.passed is False + class TestConfigMatrix: def test_config_matrix_covers_c2_cells(self, manifest): diff --git a/tests/test_ws1_workload.py b/tests/test_ws1_workload.py index ef02f28c..30a9e60d 100644 --- a/tests/test_ws1_workload.py +++ b/tests/test_ws1_workload.py @@ -472,7 +472,7 @@ def test_architecture_shrink_rejected(): def test_missing_matrix_cell_rejected(): raw = json.loads(default_manifest_path().read_text(encoding="utf-8")) raw["primary_matrix"]["cells"] = raw["primary_matrix"]["cells"][:3] - with pytest.raises(WorkloadError, match="primary_matrix.cells"): + with pytest.raises(WorkloadError, match=r"primary_matrix\.cells"): validate_manifest(raw)