From 152d6df1cc6bb967c060c4b370067e5b5e8b88d7 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 17:55:55 +0900 Subject: [PATCH 1/4] update(perf): bound DRY penalty step cost Replace quadratic backward matching with a linear reverse-Z pass and scatter DRY and rebuild frequency/presence updates only across touched token IDs. Preserve backend-specific dense dtype behavior and the existing incremental f32 path. Add dense-reference parity matrices, breaker-boundary and linear comparison-count checks, bf16 incremental regression coverage, and an ignored timing report. Build and tests have not run yet because the orchestrated CUDA profiling window is still active. Refs #2259 --- src/lib/mlxcel-core/src/sampling.rs | 572 +++++++++++++++++++++++++++- 1 file changed, 557 insertions(+), 15 deletions(-) diff --git a/src/lib/mlxcel-core/src/sampling.rs b/src/lib/mlxcel-core/src/sampling.rs index b2612746d..2c8527a88 100644 --- a/src/lib/mlxcel-core/src/sampling.rs +++ b/src/lib/mlxcel-core/src/sampling.rs @@ -1850,6 +1850,37 @@ pub(crate) fn apply_repetition_penalty_sorted( /// Apply OpenAI-style frequency and presence penalties to logits. /// /// Used by: standard generation, speculative decoding +fn apply_frequency_presence_sparse( + logits: &MlxArray, + idx: &[i32], + val: &[f32], +) -> UniquePtr { + debug_assert_eq!(idx.len(), val.len()); + + let shape = ffi::array_shape(logits); + let vocab_size = *shape.last().unwrap(); + let rows: i32 = shape[..shape.len() - 1].iter().product(); + if idx.is_empty() { + return ffi::copy(logits); + } + + let k = idx.len() as i32; + let mut row_idx = Vec::with_capacity(rows as usize * idx.len()); + let mut row_val = Vec::with_capacity(rows as usize * val.len()); + for _ in 0..rows { + row_idx.extend_from_slice(idx); + row_val.extend_from_slice(val); + } + + let indices = ffi::from_slice_i32(&row_idx, &[rows, k]); + let values = ffi::from_slice_f32(&row_val, &[rows, k]); + let flat = ffi::reshape(logits, &[rows, vocab_size]); + let selected = ffi::take_along_axis(&flat, &indices, -1); + let penalized = ffi::subtract(&selected, &values); + let scattered = ffi::put_along_axis(&flat, &indices, &penalized, -1); + ffi::reshape(&scattered, &shape) +} + pub(crate) fn apply_frequency_presence_penalty( logits: &MlxArray, token_history: &[i32], @@ -1865,6 +1896,43 @@ pub(crate) fn apply_frequency_presence_penalty( return ffi::copy(logits); } + let shape = ffi::array_shape(logits); + let vocab_size = *shape.last().unwrap() as usize; + let mut idx = Vec::with_capacity(token_counts.len()); + let mut val = Vec::with_capacity(token_counts.len()); + for (&token_id, &count) in &token_counts { + if token_id >= 0 && (token_id as usize) < vocab_size { + idx.push(token_id); + val.push(frequency_penalty * count as f32 + presence_penalty); + } + } + + // Reproduce the dense reference's mixed-dtype result before scattering. + // The CUDA MLX patch preserves bf16 for bf16/f32 binary arithmetic, while + // unpatched backends and f16 promote to f32. Subtracting a scalar +0.0 + // follows that platform rule without allocating a vocabulary-sized host + // buffer and leaves every untouched value unchanged. + let zero = ffi::full_f32(&[1], 0.0, dtype::FLOAT32); + let promoted = ffi::subtract(logits, &zero); + apply_frequency_presence_sparse(&promoted, &idx, &val) +} + +#[cfg(test)] +fn apply_frequency_presence_penalty_reference( + logits: &MlxArray, + token_history: &[i32], + frequency_penalty: f32, + presence_penalty: f32, +) -> UniquePtr { + let mut token_counts: HashMap = HashMap::new(); + for &tok in token_history { + *token_counts.entry(tok).or_insert(0) += 1; + } + + if token_counts.is_empty() { + return ffi::copy(logits); + } + let shape = ffi::array_shape(logits); let vocab_size = *shape.last().unwrap() as usize; @@ -1880,6 +1948,25 @@ pub(crate) fn apply_frequency_presence_penalty( ffi::subtract(logits, &penalty_broadcast) } +#[cfg(test)] +fn apply_frequency_presence_incremental_reference( + logits: &MlxArray, + idx: &[i32], + val: &[f32], +) -> UniquePtr { + if idx.is_empty() { + ffi::copy(logits) + } else { + let promoted = ffi::astype(logits, dtype::FLOAT32); + let k = idx.len() as i32; + let indices = ffi::from_slice_i32(idx, &[1, k]); + let selected = ffi::take_along_axis(&promoted, &indices, -1); + let values = ffi::from_slice_f32(val, &[1, k]); + let penalized = ffi::subtract(&selected, &values); + ffi::put_along_axis(&promoted, &indices, &penalized, -1) + } +} + /// Whether DRY matching breaks at `window[pos]` (#1485). /// /// Two breaker sources compose: @@ -1915,16 +2002,204 @@ fn dry_breaks_at(config: &SamplingConfig, window: &[i32], pos: usize) -> bool { }) } +#[cfg(test)] +thread_local! { + static DRY_STEP_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +fn record_dry_step_cost(cost: usize) { + DRY_STEP_COST.with(|counter| counter.set(cost)); +} + +#[cfg(test)] +fn last_dry_step_cost() -> usize { + DRY_STEP_COST.with(std::cell::Cell::get) +} + +/// Compute DRY penalties from a selected history window without MLX calls. +/// +/// Reversing the window turns every suffix match against the current suffix +/// into a prefix match, so one Z-function pass provides all match lengths. +/// A second forward pass records the nearest breaker strictly before each +/// candidate position. Both passes are linear in the window length. +fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { + #[cfg(test)] + let mut step_cost = 0usize; + #[cfg(test)] + record_dry_step_cost(0); + + let window_len = window.len(); + if window_len < 2 || config.dry_base < 1.0 { + return Vec::new(); + } + + let reversed: Vec = window.iter().rev().copied().collect(); + let mut z = vec![0usize; window_len]; + z[0] = window_len; + let mut left = 0usize; + let mut right = 0usize; + for i in 1..window_len { + if i <= right { + z[i] = (right - i + 1).min(z[i - left]); + } + while i + z[i] < window_len { + #[cfg(test)] + { + step_cost += 1; + } + if reversed[z[i]] != reversed[i + z[i]] { + break; + } + z[i] += 1; + } + if z[i] > 0 { + let candidate_right = i + z[i] - 1; + if candidate_right > right { + left = i; + right = candidate_right; + } + } + } + + let no_breaker = usize::MAX; + let mut breaker_before = vec![no_breaker; window_len]; + let mut most_recent_breaker = no_breaker; + for (pos, slot) in breaker_before.iter_mut().enumerate() { + *slot = most_recent_breaker; + #[cfg(test)] + { + step_cost += 1; + } + if dry_breaks_at(config, window, pos) { + most_recent_breaker = pos; + } + } + + let last_pos = window_len - 1; + let last_token = window[last_pos]; + let mut penalties: HashMap = HashMap::new(); + for (pos, &token) in window[..last_pos].iter().enumerate() { + #[cfg(test)] + { + step_cost += 1; + } + if token != last_token { + continue; + } + + let suffix_match_len = z[last_pos - pos]; + let breaker_limited_len = if breaker_before[pos] == no_breaker { + pos + 1 + } else { + pos - breaker_before[pos] + }; + let match_len = suffix_match_len.min(breaker_limited_len); + + if match_len >= config.dry_allowed_length { + let next_token = window[pos + 1]; + let mut exponent = (match_len - config.dry_allowed_length) as f32; + if config.dry_base > 1.0 { + let max_exponent = 88.722_84_f32 / config.dry_base.ln(); + if exponent > max_exponent { + exponent = max_exponent; + } + } + let penalty = config.dry_multiplier * config.dry_base.powf(exponent); + let entry = penalties.entry(next_token).or_insert(0.0); + if penalty > *entry { + *entry = penalty; + } + } + } + + #[cfg(test)] + record_dry_step_cost(step_cost); + penalties.into_iter().collect() +} + +fn apply_dry_penalties_sparse(logits: &MlxArray, penalties: &[(i32, f32)]) -> UniquePtr { + let shape = ffi::array_shape(logits); + let vocab_size = *shape.last().unwrap(); + let rows: i32 = shape[..shape.len() - 1].iter().product(); + let mut idx = Vec::with_capacity(penalties.len()); + let mut val = Vec::with_capacity(penalties.len()); + for &(token_id, penalty) in penalties { + if token_id >= 0 && token_id < vocab_size { + idx.push(token_id); + val.push(-penalty); + } + } + + // Match the old dense `add(logits, f32_penalty_buffer)` result dtype on + // every backend. In particular, the CUDA MLX patch keeps bf16 here rather + // than promoting it to f32. A scalar +0.0 obtains the same binary-op dtype + // and untouched values without a vocabulary-sized host allocation. + let zero = ffi::full_f32(&[1], 0.0, dtype::FLOAT32); + let promoted = ffi::add(logits, &zero); + if idx.is_empty() { + return promoted; + } + + let k = idx.len() as i32; + let mut row_idx = Vec::with_capacity(rows as usize * idx.len()); + let mut row_val = Vec::with_capacity(rows as usize * val.len()); + for _ in 0..rows { + row_idx.extend_from_slice(&idx); + row_val.extend_from_slice(&val); + } + + let indices = ffi::from_slice_i32(&row_idx, &[rows, k]); + let values = ffi::from_slice_f32(&row_val, &[rows, k]); + let flat = ffi::reshape(&promoted, &[rows, vocab_size]); + let selected = ffi::take_along_axis(&flat, &indices, -1); + let penalized = ffi::add(&selected, &values); + let scattered = ffi::put_along_axis(&flat, &indices, &penalized, -1); + ffi::reshape(&scattered, &shape) +} + /// Apply DRY (Don't Repeat Yourself) penalty to logits. /// -/// This runs on CPU as sequential pattern matching, which keeps the matching -/// invariant explicit and mirrors the upstream llama.cpp style algorithm. +/// The host matching pass is linear in the selected history window and the +/// device update touches only token ids that receive a penalty. /// /// Used by: standard generation, speculative decoding pub(crate) fn apply_dry_penalty( logits: &MlxArray, token_history: &[i32], config: &SamplingConfig, +) -> UniquePtr { + let history_len = token_history.len(); + if history_len < 2 || config.dry_base < 1.0 { + return ffi::copy(logits); + } + + let window = if config.dry_penalty_last_n == crate::generate::DRY_FULL_HISTORY { + token_history + } else if config.dry_penalty_last_n == 0 { + &token_history[history_len..] + } else { + let start = history_len.saturating_sub(config.dry_penalty_last_n); + &token_history[start..] + }; + + if window.len() < 2 { + return ffi::copy(logits); + } + + let penalties = dry_penalty_map(window, config); + if penalties.is_empty() { + return ffi::copy(logits); + } + apply_dry_penalties_sparse(logits, &penalties) +} + +/// Test-only copy of the pre-#2259 DRY implementation. +#[cfg(test)] +fn apply_dry_penalty_reference( + logits: &MlxArray, + token_history: &[i32], + config: &SamplingConfig, ) -> UniquePtr { let history_len = token_history.len(); if history_len < 2 { @@ -2232,12 +2507,12 @@ impl SamplerState { /// Frequency/presence penalty using the reusable sparse scratch buffers /// (touched tokens only); never allocates a full-vocabulary vector. /// - /// Byte-identical to [`apply_frequency_presence_penalty`]: the rebuild path - /// computes `subtract(logits, penalty_f32)`, which promotes the whole array - /// to f32 (f16/bf16 -> f32 is lossless). This path promotes first, then - /// applies `logits[id] - penalty[id]` to exactly the touched ids via - /// take/put. Untouched ids keep their promoted value, which equals the - /// rebuild path's `logits[id] - 0.0` for every finite logit. + /// Preserves the pre-#2259 incremental semantics: promote the logits to + /// f32 first, then apply `logits[id] - penalty[id]` to exactly the touched + /// ids. This intentionally differs from the dense rebuild path on CUDA + /// bf16, where patched MLX keeps mixed bf16/f32 subtraction in bf16; the + /// two paths already had that dtype/rounding distinction before this + /// refactor. An all-out-of-range count map still returns an unchanged copy. fn apply_frequency_presence( &mut self, logits: &MlxArray, @@ -2266,16 +2541,15 @@ impl SamplerState { } let result = if idx.is_empty() { - // No in-range tokens: matches the rebuild path's empty early return. + // Preserve the pre-#2259 incremental path: an all-out-of-range + // count map returns an unchanged copy without dtype promotion. ffi::copy(logits) } else { + // The incremental path explicitly promoted before its sparse + // scatter, including on CUDA bf16 where the dense rebuild path + // follows MLX's patched bf16/f32 binary-op rule instead. let promoted = ffi::astype(logits, dtype::FLOAT32); - let k = idx.len() as i32; - let indices = ffi::from_slice_i32(&idx, &[1, k]); - let selected = ffi::take_along_axis(&promoted, &indices, -1); - let values = ffi::from_slice_f32(&val, &[1, k]); - let penalized = ffi::subtract(&selected, &values); - ffi::put_along_axis(&promoted, &indices, &penalized, -1) + apply_frequency_presence_sparse(&promoted, &idx, &val) }; self.sparse_idx = idx; @@ -2938,6 +3212,274 @@ mod tests { assert_eq!(logit_at(&result, 2), -1.0); } + fn assert_penalty_logits_equal(a: &MlxArray, b: &MlxArray, context: &str) { + assert_eq!(ffi::array_shape(a), ffi::array_shape(b), "{context}: shape"); + assert_eq!(ffi::array_dtype(a), ffi::array_dtype(b), "{context}: dtype"); + let a = ffi::astype(a, dtype::FLOAT32); + let b = ffi::astype(b, dtype::FLOAT32); + let a = array_to_vec_f32(&a); + let b = array_to_vec_f32(&b); + assert_eq!(a.len(), b.len(), "{context}: length"); + for (index, (&actual, &expected)) in a.iter().zip(&b).enumerate() { + assert_eq!( + actual, expected, + "{context}: element {index}: {actual} != {expected}" + ); + } + } + + fn lcg_token_history(seed: u64, len: usize, vocab_size: usize) -> Vec { + let mut state = seed; + let mut history = Vec::with_capacity(len); + for _ in 0..len { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + history.push(((state >> 32) as usize % vocab_size) as i32); + } + history + } + + #[test] + fn dry_reverse_z_matches_reference_lcg_cases() { + const CASES: usize = 512; + const LENGTHS: [usize; 10] = [0, 1, 2, 3, 7, 17, 64, 257, 1024, 2048]; + const WINDOWS: [usize; 6] = [crate::generate::DRY_FULL_HISTORY, 0, 1, 2, 17, 64]; + + for case in 0..CASES { + let vocab_size = [8usize, 64][(case / 19) % 2]; + let history_len = LENGTHS[(case / 23) % LENGTHS.len()]; + let periodic = (case / 7) % 2 == 0; + let history = if periodic { + (0..history_len) + .map(|index| ((index + case) % 3) as i32) + .collect() + } else { + lcg_token_history( + 0x2259_0000_u64.wrapping_add(case as u64), + history_len, + vocab_size, + ) + }; + let batch = [1usize, 2][(case / 3) % 2]; + let raw_logits = lcg_logits( + 0xD4A0_0000_u64.wrapping_add(case as u64), + batch * vocab_size, + ); + let f32_logits = ffi::from_slice_f32(&raw_logits, &[batch as i32, vocab_size as i32]); + let input_dtype = [dtype::FLOAT32, dtype::BFLOAT16][(case / 5) % 2]; + let logits = if input_dtype == dtype::FLOAT32 { + f32_logits + } else { + ffi::astype(&f32_logits, input_dtype) + }; + + let mut config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: [1.0, 1.75][(case / 13) % 2], + dry_allowed_length: 1 + (case / 11) % 4, + dry_penalty_last_n: WINDOWS[(case / 29) % WINDOWS.len()], + ..Default::default() + }; + let breaker = ((case * 5 + 1) % vocab_size) as i32; + match (case / 17) % 4 { + 1 => config.dry_sequence_breakers = vec![breaker], + 2 => { + config.dry_breaker_heads = + std::sync::Arc::new(HashMap::from([(breaker, vec![Vec::new()])])); + } + 3 => { + let tail = (breaker + 1) % vocab_size as i32; + config.dry_breaker_heads = + std::sync::Arc::new(HashMap::from([(breaker, vec![vec![tail]])])); + } + _ => {} + } + + let reference = apply_dry_penalty_reference(&logits, &history, &config); + let linear = apply_dry_penalty(&logits, &history, &config); + assert_penalty_logits_equal( + &linear, + &reference, + &format!( + "case={case} vocab={vocab_size} history={history_len} periodic={periodic} batch={batch} dtype={input_dtype} window={} allowed={} base={}", + config.dry_penalty_last_n, config.dry_allowed_length, config.dry_base + ), + ); + } + } + + #[test] + fn dry_reverse_z_preserves_breaker_boundaries() { + let mut config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: 1.75, + dry_allowed_length: 2, + dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, + dry_sequence_breakers: vec![2], + ..Default::default() + }; + let history = [1, 2, 9, 1, 2]; + let map = dry_penalty_map(&history, &config); + assert!( + map.iter().any(|&(token, _)| token == 9), + "a breaker at the occurrence itself must not cut the preceding match" + ); + + config.dry_sequence_breakers.clear(); + config.dry_breaker_heads = std::sync::Arc::new(HashMap::from([(2, vec![vec![7, 8]])])); + let reference = apply_dry_penalty_reference( + &ffi::from_slice_f32(&[0.0; 12], &[1, 12]), + &history, + &config, + ); + let linear = apply_dry_penalty( + &ffi::from_slice_f32(&[0.0; 12], &[1, 12]), + &history, + &config, + ); + assert_penalty_logits_equal( + &linear, + &reference, + "a breaker tail running past the window end must not break", + ); + } + + #[test] + fn dry_reverse_z_step_cost_is_linear() { + let config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: 1.75, + dry_allowed_length: 2, + dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, + ..Default::default() + }; + let history_4096: Vec = (0..4096).map(|index| index % 3).collect(); + let _ = dry_penalty_map(&history_4096, &config); + let cost_4096 = last_dry_step_cost(); + assert!( + cost_4096 <= 8 * 4096, + "period-3 window 4096 cost {cost_4096}, expected <= {}", + 8 * 4096 + ); + + let history_8192: Vec = (0..8192).map(|index| index % 3).collect(); + let _ = dry_penalty_map(&history_8192, &config); + let cost_8192 = last_dry_step_cost(); + assert!( + cost_8192 <= 2 * cost_4096 + 16, + "doubling the window changed cost from {cost_4096} to {cost_8192}" + ); + } + + #[test] + fn frequency_presence_sparse_matches_vocab_reference() { + let vocab_size = 64usize; + let history = lcg_token_history(0xF2E9_2259, 96, vocab_size); + for penalty_last_n in [1, 64] { + let window = penalty_window(&history, penalty_last_n); + for batch in [1usize, 2] { + let raw = lcg_logits(0xF2E9_0000 + batch as u64, batch * vocab_size); + let f32_logits = ffi::from_slice_f32(&raw, &[batch as i32, vocab_size as i32]); + for input_dtype in [dtype::FLOAT32, dtype::FLOAT16, dtype::BFLOAT16] { + let logits = if input_dtype == dtype::FLOAT32 { + ffi::copy(&f32_logits) + } else { + ffi::astype(&f32_logits, input_dtype) + }; + let reference = + apply_frequency_presence_penalty_reference(&logits, window, 0.625, 0.375); + let sparse = apply_frequency_presence_penalty(&logits, window, 0.625, 0.375); + assert_penalty_logits_equal( + &sparse, + &reference, + &format!( + "penalty_last_n={penalty_last_n} batch={batch} dtype={input_dtype}" + ), + ); + } + } + } + } + + #[test] + fn sampler_state_frequency_presence_preserves_bf16_reference() { + let (frequency_penalty, presence_penalty) = (0.625, 0.375); + let config = SamplingConfig { + frequency_penalty, + presence_penalty, + ..Default::default() + }; + let raw = lcg_logits(0x1AC0_2259, 64); + let bf16_logits = ffi::astype(&ffi::from_slice_f32(&raw, &[1, 64]), dtype::BFLOAT16); + + let mut state = SamplerState::for_config(&config); + state.sync(&[1, 1, 2, 5, 8, 13, 13]); + let mut idx = Vec::new(); + let mut val = Vec::new(); + for (&token_id, &count) in &state.counts { + idx.push(token_id); + val.push(frequency_penalty * count as f32 + presence_penalty); + } + let reference = apply_frequency_presence_incremental_reference(&bf16_logits, &idx, &val); + let sparse = + state.apply_frequency_presence(&bf16_logits, frequency_penalty, presence_penalty); + assert_penalty_logits_equal( + &sparse, + &reference, + "incremental bf16 path must retain its explicit f32 promotion", + ); + assert_eq!(ffi::array_dtype(&sparse), dtype::FLOAT32); + + let mut out_of_range = SamplerState::for_config(&config); + out_of_range.sync(&[-3, 64, 100]); + let untouched = out_of_range.apply_frequency_presence( + &bf16_logits, + frequency_penalty, + presence_penalty, + ); + assert_eq!(ffi::array_dtype(&untouched), dtype::BFLOAT16); + assert_penalty_logits_equal( + &untouched, + &bf16_logits, + "all-out-of-range incremental counts must keep the copy early return", + ); + } + + #[test] + #[ignore = "manual timing evidence for issue #2259"] + fn dry_step_cost_report() { + let config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: 1.75, + dry_allowed_length: 2, + dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, + ..Default::default() + }; + let logits = ffi::from_slice_f32(&[0.0; 4], &[1, 4]); + for window_len in [1024usize, 4096, 16384] { + let history: Vec = (0..window_len).map(|index| (index % 3) as i32).collect(); + let reference_steps = if window_len == 16384 { 1 } else { 3 }; + let reference_start = std::time::Instant::now(); + for _ in 0..reference_steps { + std::hint::black_box(apply_dry_penalty_reference(&logits, &history, &config)); + } + let reference_us = + reference_start.elapsed().as_secs_f64() * 1_000_000.0 / reference_steps as f64; + + let linear_steps = 20; + let linear_start = std::time::Instant::now(); + for _ in 0..linear_steps { + std::hint::black_box(apply_dry_penalty(&logits, &history, &config)); + } + let linear_us = + linear_start.elapsed().as_secs_f64() * 1_000_000.0 / linear_steps as f64; + println!( + "dry_step_cost window={window_len} reference_us_per_step={reference_us:.3} linear_us_per_step={linear_us:.3}" + ); + } + } + #[test] fn sample_token_optimized_respects_greedy_argmax_path() { let logits = ffi::from_slice_f32(&[0.1, 0.9, 1.2], &[1, 1, 3]); From ab1a9508b74b4a1466c90e7af7d24fbb69075ebd Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 18:37:50 +0900 Subject: [PATCH 2/4] fix(perf): bound DRY breaker matching cost Build a reversed-token matcher once per DRY step so exact IDs and multi-token breakers cost expected O(P + N) instead of multiplying breaker configuration size by the history window. Preserve scalar breaker semantics in direct per-position parity tests and cover large exact sets plus shared-tail failure chains with separate build and scan counters. Validation: cargo fmt --all. Build and tests are pending the root validation gate. Refs #2259 --- src/lib/mlxcel-core/src/sampling.rs | 386 +++++++++++++++++++++++++++- 1 file changed, 383 insertions(+), 3 deletions(-) diff --git a/src/lib/mlxcel-core/src/sampling.rs b/src/lib/mlxcel-core/src/sampling.rs index 2c8527a88..59fb6b02c 100644 --- a/src/lib/mlxcel-core/src/sampling.rs +++ b/src/lib/mlxcel-core/src/sampling.rs @@ -1980,6 +1980,7 @@ fn apply_frequency_presence_incremental_reference( /// only form). A head with a non-empty tail breaks only when the window /// tokens after `pos` spell the tail out in full, which is how a /// multi-character breaker split across token boundaries is recognized. +#[cfg(test)] fn dry_breaks_at(config: &SamplingConfig, window: &[i32], pos: usize) -> bool { if config.dry_sequence_breakers.contains(&window[pos]) { return true; @@ -2002,9 +2003,184 @@ fn dry_breaks_at(config: &SamplingConfig, window: &[i32], pos: usize) -> bool { }) } +#[derive(Default)] +struct DryBreakerNode { + transitions: HashMap, + failure: usize, + terminal: bool, +} + +struct DryBreakerMatches { + positions: Vec, + #[cfg(test)] + build_steps: usize, + #[cfg(test)] + scan_steps: usize, +} + +/// Find every DRY breaker start in `reversed_window` in additive +/// `O(window + breaker-pattern tokens)` expected time. +/// +/// Exact IDs are singleton patterns. A forward breaker `[head] + tail` is +/// inserted as `reverse(tail) + [head]`, so an Aho-Corasick terminal reached +/// at reversed index `r` always maps to original start `n - 1 - r`, regardless +/// of pattern length. Failure-terminal propagation therefore marks a start +/// once without enumerating every matching suffix pattern. +fn dry_breaker_matches(reversed_window: &[i32], config: &SamplingConfig) -> DryBreakerMatches { + #[cfg(test)] + let mut build_steps = 0usize; + #[cfg(test)] + let mut scan_steps = 0usize; + + if config.dry_sequence_breakers.is_empty() && config.dry_breaker_heads.is_empty() { + return DryBreakerMatches { + positions: vec![false; reversed_window.len()], + #[cfg(test)] + build_steps, + #[cfg(test)] + scan_steps, + }; + } + + let mut nodes = vec![DryBreakerNode::default()]; + + for &token in &config.dry_sequence_breakers { + #[cfg(test)] + { + build_steps += 1; + } + let next = nodes[0].transitions.get(&token).copied(); + let state = match next { + Some(state) => state, + None => { + let state = nodes.len(); + nodes.push(DryBreakerNode::default()); + nodes[0].transitions.insert(token, state); + #[cfg(test)] + { + build_steps += 1; + } + state + } + }; + nodes[state].terminal = true; + } + + for (&head, tails) in config.dry_breaker_heads.iter() { + for tail in tails { + let mut state = 0usize; + for token in tail.iter().rev().copied().chain(std::iter::once(head)) { + #[cfg(test)] + { + build_steps += 1; + } + let next = nodes[state].transitions.get(&token).copied(); + state = match next { + Some(next) => next, + None => { + let next = nodes.len(); + nodes.push(DryBreakerNode::default()); + nodes[state].transitions.insert(token, next); + #[cfg(test)] + { + build_steps += 1; + } + next + } + }; + } + nodes[state].terminal = true; + } + } + + // Breadth-first failure-link construction. Root children fail to root; + // every deeper node inherits the terminal bit of its failure target. Across + // trie edges, failure hops telescope over failure depth, so construction is + // linear in the number of supplied pattern tokens in expected HashMap time. + let mut queue: Vec = nodes[0].transitions.values().copied().collect(); + #[cfg(test)] + { + build_steps += queue.len(); + } + let mut cursor = 0usize; + while cursor < queue.len() { + let state = queue[cursor]; + cursor += 1; + let transitions: Vec<(i32, usize)> = nodes[state] + .transitions + .iter() + .map(|(&token, &next)| (token, next)) + .collect(); + #[cfg(test)] + { + build_steps += transitions.len(); + } + for (token, next) in transitions { + let mut fallback = nodes[state].failure; + let failure = loop { + #[cfg(test)] + { + build_steps += 1; + } + if let Some(&target) = nodes[fallback].transitions.get(&token) { + break target; + } + if fallback == 0 { + break 0; + } + fallback = nodes[fallback].failure; + #[cfg(test)] + { + build_steps += 1; + } + }; + nodes[next].failure = failure; + let failure_terminal = nodes[failure].terminal; + nodes[next].terminal |= failure_terminal; + queue.push(next); + } + } + + let mut positions = vec![false; reversed_window.len()]; + let mut state = 0usize; + for (reversed_pos, &token) in reversed_window.iter().enumerate() { + loop { + #[cfg(test)] + { + scan_steps += 1; + } + if let Some(&next) = nodes[state].transitions.get(&token) { + state = next; + break; + } + if state == 0 { + break; + } + state = nodes[state].failure; + #[cfg(test)] + { + scan_steps += 1; + } + } + if nodes[state].terminal { + positions[reversed_window.len() - 1 - reversed_pos] = true; + } + } + + DryBreakerMatches { + positions, + #[cfg(test)] + build_steps, + #[cfg(test)] + scan_steps, + } +} + #[cfg(test)] thread_local! { static DRY_STEP_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; + static DRY_BREAKER_BUILD_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; + static DRY_BREAKER_SCAN_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; } #[cfg(test)] @@ -2017,17 +2193,34 @@ fn last_dry_step_cost() -> usize { DRY_STEP_COST.with(std::cell::Cell::get) } +#[cfg(test)] +fn record_dry_breaker_cost(build: usize, scan: usize) { + DRY_BREAKER_BUILD_COST.with(|counter| counter.set(build)); + DRY_BREAKER_SCAN_COST.with(|counter| counter.set(scan)); +} + +#[cfg(test)] +fn last_dry_breaker_cost() -> (usize, usize) { + let build = DRY_BREAKER_BUILD_COST.with(std::cell::Cell::get); + let scan = DRY_BREAKER_SCAN_COST.with(std::cell::Cell::get); + (build, scan) +} + /// Compute DRY penalties from a selected history window without MLX calls. /// /// Reversing the window turns every suffix match against the current suffix /// into a prefix match, so one Z-function pass provides all match lengths. -/// A second forward pass records the nearest breaker strictly before each -/// candidate position. Both passes are linear in the window length. +/// A reversed-token automaton marks breaker starts before one forward pass +/// records the nearest breaker strictly before each candidate position. The +/// per-step work is additive in the window and configured breaker-pattern +/// tokens; it never multiplies the two request-controlled sizes. fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { #[cfg(test)] let mut step_cost = 0usize; #[cfg(test)] record_dry_step_cost(0); + #[cfg(test)] + record_dry_breaker_cost(0, 0); let window_len = window.len(); if window_len < 2 || config.dry_base < 1.0 { @@ -2062,6 +2255,13 @@ fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { } } + let breaker_matches = dry_breaker_matches(&reversed, config); + #[cfg(test)] + { + step_cost += breaker_matches.scan_steps; + record_dry_breaker_cost(breaker_matches.build_steps, breaker_matches.scan_steps); + } + let no_breaker = usize::MAX; let mut breaker_before = vec![no_breaker; window_len]; let mut most_recent_breaker = no_breaker; @@ -2071,7 +2271,7 @@ fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { { step_cost += 1; } - if dry_breaks_at(config, window, pos) { + if breaker_matches.positions[pos] { most_recent_breaker = pos; } } @@ -3296,6 +3496,23 @@ mod tests { _ => {} } + let window = if config.dry_penalty_last_n == crate::generate::DRY_FULL_HISTORY { + history.as_slice() + } else if config.dry_penalty_last_n == 0 { + &history[history.len()..] + } else { + let start = history.len().saturating_sub(config.dry_penalty_last_n); + &history[start..] + }; + let breaker_matches = breaker_positions(window, &config); + for (pos, matched) in breaker_matches.iter().copied().enumerate() { + assert_eq!( + matched, + dry_breaks_at(&config, window, pos), + "breaker mismatch at case={case} pos={pos} window={window:?}" + ); + } + let reference = apply_dry_penalty_reference(&logits, &history, &config); let linear = apply_dry_penalty(&logits, &history, &config); assert_penalty_logits_equal( @@ -3345,6 +3562,169 @@ mod tests { ); } + fn breaker_positions(window: &[i32], config: &SamplingConfig) -> Vec { + let reversed: Vec = window.iter().rev().copied().collect(); + dry_breaker_matches(&reversed, config).positions + } + + #[test] + fn dry_breaker_automaton_preserves_overlap_and_boundary_semantics() { + let config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: 1.75, + dry_allowed_length: 2, + dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, + dry_sequence_breakers: vec![-7, 2, 2], + dry_breaker_heads: std::sync::Arc::new(HashMap::from([ + (2, vec![Vec::new(), Vec::new(), vec![3, 4]]), + (4, vec![vec![5, 6, 8]]), + (5, vec![vec![6], vec![6, 7], vec![7]]), + (9, vec![vec![10, 11]]), + ])), + ..Default::default() + }; + + let window = [-7, 2, 3, 4, 5, 6, 7, 5, 7, 9, 10, 11]; + let positions = breaker_positions(&window, &config); + for (pos, matched) in positions.iter().copied().enumerate() { + assert_eq!( + matched, + dry_breaks_at(&config, &window, pos), + "overlap breaker mismatch at position {pos}" + ); + } + for expected in [0usize, 1, 4, 7, 9] { + assert!(positions[expected], "missing breaker at {expected}"); + } + assert_eq!( + positions.iter().filter(|&&is_breaker| is_breaker).count(), + 5, + "duplicates and overlapping tails must mark each start once" + ); + + assert_eq!( + breaker_positions(&[9, 10, 11], &config), + vec![true, false, false], + "a tail ending exactly at the window end must match" + ); + assert_eq!( + breaker_positions(&[9, 10], &config), + vec![false, false], + "a tail running past the window end must not match" + ); + assert_eq!( + breaker_positions(&[5, 6, 8], &config), + vec![true, false, false], + "a terminal failure suffix must mark the same original start" + ); + + let logits = ffi::from_slice_f32(&[0.0; 16], &[1, 16]); + for history in [ + vec![9, 10, 11, 4, 9, 10, 11], + vec![9, 10, 4, 9, 10], + vec![1, -7, 2, 4, 1, -7, 2], + vec![5, 6, 8, 4, 5, 6, 8], + ] { + let reference = apply_dry_penalty_reference(&logits, &history, &config); + let automaton = apply_dry_penalty(&logits, &history, &config); + assert_penalty_logits_equal( + &automaton, + &reference, + &format!("breaker automaton history {history:?}"), + ); + } + } + + #[test] + fn dry_breaker_automaton_large_exact_list_is_additive() { + const WINDOW: usize = 4096; + const EXACT: usize = 2048; + let mut exact: Vec = (10_000..10_000 + EXACT as i32).collect(); + exact.extend([-7, 0, 1, 2]); + let config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: 1.75, + dry_allowed_length: 2, + dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, + dry_sequence_breakers: exact, + ..Default::default() + }; + let mut history: Vec = (0..WINDOW - 1).map(|index| (index % 3) as i32).collect(); + history.push(63); + + let _ = dry_penalty_map(&history, &config); + let (build_steps, scan_steps) = last_dry_breaker_cost(); + assert!( + build_steps <= 4 * config.dry_sequence_breakers.len() + 16, + "exact-list build took {build_steps} steps for {} tokens", + config.dry_sequence_breakers.len() + ); + assert!( + scan_steps <= 3 * WINDOW + 16, + "exact-list scan took {scan_steps} steps for window {WINDOW}" + ); + assert!( + last_dry_step_cost() <= 8 * WINDOW, + "window work must stay linear independently of the exact-list size" + ); + + let logits = ffi::from_slice_f32(&[0.0; 64], &[1, 64]); + let reference = apply_dry_penalty_reference(&logits, &history, &config); + let automaton = apply_dry_penalty(&logits, &history, &config); + assert_penalty_logits_equal( + &automaton, + &reference, + "large exact list with a rare last token", + ); + } + + #[test] + fn dry_breaker_automaton_long_shared_tails_are_additive() { + const WINDOW: usize = 4096; + const MAX_TAIL: usize = 128; + let mut breaker_heads = HashMap::new(); + let mut inserted_tokens = 0usize; + for len in 1..=MAX_TAIL { + let tail = vec![1; len]; + inserted_tokens += tail.len() + 1; + breaker_heads.insert(10_000 + len as i32, vec![tail]); + } + let config = SamplingConfig { + dry_multiplier: 0.75, + dry_base: 1.75, + dry_allowed_length: 2, + dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, + dry_breaker_heads: std::sync::Arc::new(breaker_heads), + ..Default::default() + }; + let mut history = vec![1; WINDOW - 1]; + history.push(63); + + let _ = dry_penalty_map(&history, &config); + let (build_steps, scan_steps) = last_dry_breaker_cost(); + assert!( + build_steps <= 8 * inserted_tokens + 32, + "shared-tail build took {build_steps} steps for {inserted_tokens} inserted tokens" + ); + assert!( + scan_steps <= 3 * WINDOW + 16, + "shared-tail scan took {scan_steps} steps for window {WINDOW}" + ); + assert!( + last_dry_step_cost() <= 8 * WINDOW, + "window work must not multiply by the shared-tail family" + ); + + let logits = ffi::from_slice_f32(&[0.0; 64], &[1, 64]); + let reference = apply_dry_penalty_reference(&logits, &history, &config); + let automaton = apply_dry_penalty(&logits, &history, &config); + assert_penalty_logits_equal( + &automaton, + &reference, + "long shared tails with a rare last token", + ); + } + #[test] fn dry_reverse_z_step_cost_is_linear() { let config = SamplingConfig { From 327b6bf13a31d0f98e4cce717443d03b4be2cb73 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 19:21:30 +0900 Subject: [PATCH 3/4] docs(perf): record DRY step cost evidence Publish the quiet GB10 host matching and lazy graph-construction timings for issue #2259 with byte-preserved logs, validation output, checksums, and source, artifact, driver, and host-gate provenance. The record distinguishes these measurements from GPU end-to-end latency. Validation: sha256sum --check SHA256SUMS and git diff --cached --check. Refs #2259 --- .../.gitattributes | 4 + .../2259-quiet-timing.sh | 34 ++ .../README.md | 33 ++ .../SHA256SUMS | 13 + .../clippy.log | 104 ++++++ .../core-build.log | 105 ++++++ .../dry-step-cost.log | 9 + .../engine-tests.log | 173 ++++++++++ .../environment.txt | 34 ++ .../fmt.log | 0 .../hostgate.log | 2 + .../outer-invocation.txt | 4 + .../provenance.txt | 22 ++ .../sampling-tests.log | 321 ++++++++++++++++++ .../dry-penalty-step-cost-gb10-2026-10-11.md | 53 +++ 15 files changed, 911 insertions(+) create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/2259-quiet-timing.sh create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/clippy.log create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/core-build.log create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/dry-step-cost.log create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/engine-tests.log create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/environment.txt create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/fmt.log create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/hostgate.log create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/outer-invocation.txt create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/sampling-tests.log create mode 100644 docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes new file mode 100644 index 000000000..324f58529 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes @@ -0,0 +1,4 @@ +dry-step-cost.log whitespace=-blank-at-eof +engine-tests.log whitespace=-blank-at-eof +environment.txt whitespace=-trailing-space +sampling-tests.log whitespace=-blank-at-eof diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/2259-quiet-timing.sh b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/2259-quiet-timing.sh new file mode 100644 index 000000000..63615d550 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/2259-quiet-timing.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +cd /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259 +OUT=/tmp/mlxcel-auto-plan/2259-timing +mkdir -p "$OUT" +test ! -e "$OUT/dry-step-cost.log" +BIN=target/test-fast/deps/mlxcel_core-5feb86ac05c84f53 +{ + date -u --iso-8601=seconds + hostname + uname -a + git rev-parse HEAD + git status --short + printf "configured_mlx_pin=" + scripts/ci/mlx_pinned_commit.sh + for mlx_source_dir in target/test-fast/build/mlxcel-core-*/out/build/_deps/mlx-src; do + if [[ -d "$mlx_source_dir/.git" ]]; then + printf "fetched_mlx_source=%s commit=" "$mlx_source_dir" + git -C "$mlx_source_dir" rev-parse HEAD + fi + done + sha256sum "$BIN" "$0" + nvidia-smi + gh run list --status in_progress --limit 30 --json databaseId,name,status,headBranch +} > "$OUT/environment.txt" 2>&1 +python3 -B - <<'GATE' > "$OUT/hostgate.log" 2>&1 +import sys +sys.path.insert(0, 'docs/benchmark_results/data/sdpa-plan-bucket-gb10-2026-09-12/harness') +import hostgate +print('gate_wait_s=', hostgate.wait_quiet(log=sys.stderr), flush=True) +print('ci_job_running=', hostgate.ci_job_running(), flush=True) +GATE +"$BIN" sampling::tests::dry_step_cost_report --ignored --nocapture --test-threads=1 > "$OUT/dry-step-cost.log" 2>&1 +date -u --iso-8601=seconds >> "$OUT/environment.txt" diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md new file mode 100644 index 000000000..9bbd179ae --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md @@ -0,0 +1,33 @@ +# Raw data for DRY penalty step cost on GB10 (2026-10-11) + +This directory preserves timing and validation evidence for issue #2259. + +## Timing run + +`dry-step-cost.log` is the byte-preserved output of the ignored `sampling::tests::dry_step_cost_report` test. The test compares the exact pre-#2259 reference retained under `cfg(test)` with the new production path on period-3 windows of 1,024, 4,096, and 16,384 tokens, using one row of four f32 logits and no breakers. Reference iterations are 3, 3, and 1; the new path runs 20 iterations at each size. The returned MLX arrays remain lazy and are not evaluated, so the results measure host matching and MLX graph construction rather than GPU end-to-end latency. + +`2259-quiet-timing.sh` is the byte-preserved inner driver. `outer-invocation.txt` records the exact outer `gpu-lock` command and the SHA-256 of the batch driver that issued it. The inner driver records the clean measurement checkout, source commit, MLX pin and fetched checkouts, host and GPU, start/end UTC, the measured test artifact hash, and its own hash in `environment.txt`. + +The reused host gate has SHA-256 `69e6d3f468c42ba7e478578e3bde9efb9c5a30165866d8e9da19440a6eb9413f`. Its default 12 samples at five-second intervals require 55 continuous seconds of quiet because the first sample is immediate. `gate_wait_s=0` means no busy interval reset the quiet counter; it does not mean the sustained quiet check took zero seconds. `hostgate.log` also records that no CI job was active at the end of the gate. + +## Validation logs + +The raw final logs correspond to source commit `ecb89a308aecc468dd1886d9c9f4982f556f3c93`: + +- `core-build.log`: CUDA core build, completed in 26.74 seconds. +- `clippy.log`: CUDA library/test clippy with warnings denied for Rust, completed in 9.08 seconds. +- `fmt.log`: formatting check. +- `sampling-tests.log`: 210 passed, 1 ignored, 0 failed. +- `engine-tests.log`: 63 passed, 0 failed. + +The native build logs contain existing compiler diagnostics from the pinned dependency build; the Rust clippy command still exited successfully with `-D warnings`. + +## Replay + +After squash merge, rebuild and run the ignored test from the merge commit on a quiet CUDA host with `gpu-lock run --tag issue-2259-quiet-timing -- cargo test --profile test-fast --features cuda -p mlxcel-core --lib sampling::tests::dry_step_cost_report -- --ignored --nocapture --test-threads=1`. That direct cargo form may compile while holding the lock if the cache is cold. For a controlled measurement, first build the test artifact outside the timing window, then adapt the preserved inner driver to invoke that exact artifact while retaining the same provenance capture and gate. Compare production and test code against the measured source and keep any post-measurement difference explicit. + +Validate the preserved files from this directory with: + +```bash +sha256sum --check SHA256SUMS +``` diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS new file mode 100644 index 000000000..b0f56b915 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS @@ -0,0 +1,13 @@ +e8e91485fcfd340dc8949fca525b0ec0b55eb2aae7f9cb0317ef8e1d3a124b40 .gitattributes +06b48baeb28c191f550564075a65e7e1f6ee6c9dc594437a63a467319c1ab988 2259-quiet-timing.sh +83265d7d112c40c2e6b73e8906bc32251e3defd7d56608bbd7d6d29286c78233 README.md +d01b1a6bcaad819dbd8658555cf25cf2709a26f12d64bd34c69987f69d50c9af clippy.log +192b41e9e4aa31324c5a661774ae74437d6ca6e04fc8f83fd67026f8f17b613d core-build.log +9169c635ff03036e4d3046b1e276f4f4c2be21af51cf7a088625bbd470457c9c dry-step-cost.log +c825e252273a90be4ec8b44c7547d41a04eb657a0e5ceae7f2faf52a6abf3d9b engine-tests.log +8e1104d2d3082f2230107434f3305fcfd92f6b353238695485dad9f53386ea47 environment.txt +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 fmt.log +890d1fcf0b4cab5cca9f5fd990ed2cc34f8e5c7a46097ed471b30bd9e15bd3d1 hostgate.log +ecf915044da3d513f8eff236dd9b75f2abb04c1a2f3738b7cfa089ddec684e04 outer-invocation.txt +20b46b97ebf8315a881818d1e23d6f9e075e9d32910078399baecdf0c69a419a provenance.txt +d241484673791105deec40767622a4a953ae6dfcfdbc94cf4594ea6629bdd421 sampling-tests.log diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/clippy.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/clippy.log new file mode 100644 index 000000000..11e1b3ddd --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/clippy.log @@ -0,0 +1,104 @@ +warning: mlxcel-core@0.7.0: In file included from cpp/mlx_cxx_bridge.cpp:7: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-0cdeace19df9bd92/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-0cdeace19df9bd92/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In function 'mlx_cxx::{anonymous}::get_compiled_q_path_proportional(int, int, int, float)::&)>': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3362:24: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 3362 | ref_rms); +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: At global scope: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3792:24: warning: 'mlx_cxx::{anonymous}::ADD3_LN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3792 | static const char* ADD3_LN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3027:24: warning: 'mlx_cxx::{anonymous}::MOE_FC1_RELU2_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3027 | static const char* MOE_FC1_RELU2_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2847:24: warning: 'mlx_cxx::{anonymous}::MOE_DOWN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2847 | static const char* MOE_DOWN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2708:24: warning: 'mlx_cxx::{anonymous}::MOE_GATEUP_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2708 | static const char* MOE_GATEUP_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2129:24: warning: 'mlx_cxx::{anonymous}::MAMBA1_SCAN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2129 | static const char* MAMBA1_SCAN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:752:24: warning: 'mlx_cxx::{anonymous}::SSM_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 752 | static const char* SSM_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:452:24: warning: 'mlx_cxx::{anonymous}::XIELU_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 452 | static const char* XIELU_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:159:24: warning: 'mlx_cxx::{anonymous}::BITLINEAR_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 159 | static const char* BITLINEAR_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp: In function 'void mlx_cxx::nemotron_decode_step(uint64_t, const MlxArray&, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, std::unique_ptr&, rust::cxxbridge1::Slice >, rust::cxxbridge1::Slice >)': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: In file included from ../mlx-cpp/turbo/kv_inplace_write.cpp:23: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-0cdeace19df9bd92/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-0cdeace19df9bd92/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ + Checking mlxcel-core v0.7.0 (/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/src/lib/mlxcel-core) + Finished `test-fast` profile [optimized] target(s) in 9.08s diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/core-build.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/core-build.log new file mode 100644 index 000000000..bd99bf70c --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/core-build.log @@ -0,0 +1,105 @@ +warning: mlxcel-core@0.7.0: In file included from cpp/mlx_cxx_bridge.cpp:7: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In function 'mlx_cxx::{anonymous}::get_compiled_q_path_proportional(int, int, int, float)::&)>': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3362:24: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 3362 | ref_rms); +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: At global scope: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3792:24: warning: 'mlx_cxx::{anonymous}::ADD3_LN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3792 | static const char* ADD3_LN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3027:24: warning: 'mlx_cxx::{anonymous}::MOE_FC1_RELU2_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3027 | static const char* MOE_FC1_RELU2_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2847:24: warning: 'mlx_cxx::{anonymous}::MOE_DOWN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2847 | static const char* MOE_DOWN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2708:24: warning: 'mlx_cxx::{anonymous}::MOE_GATEUP_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2708 | static const char* MOE_GATEUP_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2129:24: warning: 'mlx_cxx::{anonymous}::MAMBA1_SCAN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2129 | static const char* MAMBA1_SCAN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:752:24: warning: 'mlx_cxx::{anonymous}::SSM_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 752 | static const char* SSM_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:452:24: warning: 'mlx_cxx::{anonymous}::XIELU_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 452 | static const char* XIELU_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:159:24: warning: 'mlx_cxx::{anonymous}::BITLINEAR_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 159 | static const char* BITLINEAR_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp: In function 'void mlx_cxx::nemotron_decode_step(uint64_t, const MlxArray&, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, std::unique_ptr&, rust::cxxbridge1::Slice >, rust::cxxbridge1::Slice >)': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: In file included from ../mlx-cpp/turbo/kv_inplace_write.cpp:23: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ + Compiling mlxcel-core v0.7.0 (/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/src/lib/mlxcel-core) + Finished `test-fast` profile [optimized] target(s) in 26.74s + Executable unittests src/lib.rs (target/test-fast/deps/mlxcel_core-5feb86ac05c84f53) diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/dry-step-cost.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/dry-step-cost.log new file mode 100644 index 000000000..a2ce64d94 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/dry-step-cost.log @@ -0,0 +1,9 @@ + +running 1 test +test sampling::tests::dry_step_cost_report ... dry_step_cost window=1024 reference_us_per_step=619.751 linear_us_per_step=23.607 +dry_step_cost window=4096 reference_us_per_step=4889.370 linear_us_per_step=29.997 +dry_step_cost window=16384 reference_us_per_step=77199.293 linear_us_per_step=111.433 +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 2082 filtered out; finished in 0.38s + diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/engine-tests.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/engine-tests.log new file mode 100644 index 000000000..89993f2e5 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/engine-tests.log @@ -0,0 +1,173 @@ +warning: mlxcel-core@0.7.0: In file included from cpp/mlx_cxx_bridge.cpp:7: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In function 'mlx_cxx::{anonymous}::get_compiled_q_path_proportional(int, int, int, float)::&)>': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3362:24: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 3362 | ref_rms); +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: At global scope: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3792:24: warning: 'mlx_cxx::{anonymous}::ADD3_LN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3792 | static const char* ADD3_LN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3027:24: warning: 'mlx_cxx::{anonymous}::MOE_FC1_RELU2_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3027 | static const char* MOE_FC1_RELU2_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2847:24: warning: 'mlx_cxx::{anonymous}::MOE_DOWN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2847 | static const char* MOE_DOWN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2708:24: warning: 'mlx_cxx::{anonymous}::MOE_GATEUP_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2708 | static const char* MOE_GATEUP_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2129:24: warning: 'mlx_cxx::{anonymous}::MAMBA1_SCAN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2129 | static const char* MAMBA1_SCAN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:752:24: warning: 'mlx_cxx::{anonymous}::SSM_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 752 | static const char* SSM_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:452:24: warning: 'mlx_cxx::{anonymous}::XIELU_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 452 | static const char* XIELU_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:159:24: warning: 'mlx_cxx::{anonymous}::BITLINEAR_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 159 | static const char* BITLINEAR_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp: In function 'void mlx_cxx::nemotron_decode_step(uint64_t, const MlxArray&, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, std::unique_ptr&, rust::cxxbridge1::Slice >, rust::cxxbridge1::Slice >)': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: In file included from ../mlx-cpp/turbo/kv_inplace_write.cpp:23: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ + Finished `test-fast` profile [optimized] target(s) in 0.09s + Running unittests src/lib.rs (target/test-fast/deps/mlxcel_core-5feb86ac05c84f53) + +running 63 tests +test engine::direct_decode_rows_tests::a_penalty_request_takes_the_per_row_pipeline ... ok +test engine::direct_decode_rows_tests::draw_row_rejects_rows_that_need_the_host_token ... ok +test engine::direct_decode_rows_tests::eos_length_and_callback_finishes_match_the_synchronous_loop ... Compiling CUDA kernels (first run on this host; cached for later runs)... +ok +test engine::direct_decode_rows_tests::model_owned_families_pipeline_penalty_requests_with_the_same_teardowns ... ok +test engine::direct_decode_rows_tests::penalized_streams_match_the_synchronous_loop_token_for_token ... ok +test engine::direct_decode_rows_tests::the_per_row_pipeline_takes_only_what_the_fused_one_turns_away ... ok +test engine::direct_decode_tests::a_pipelined_client_is_reusable ... ok +test engine::direct_decode_tests::a_zero_budget_emits_nothing ... ok +test engine::direct_decode_tests::callback_stop_matches_the_synchronous_loop ... ok +test engine::direct_decode_tests::eos_finish_matches_the_synchronous_loop ... ok +test engine::direct_decode_tests::history_penalties_are_not_fused_eligible ... ok +test engine::direct_decode_tests::length_finish_matches_the_synchronous_loop_with_no_extra_forward ... ok +test engine::direct_decode_tests::lookahead_feedback_input_matches_a_host_input ... ok +test engine::direct_decode_tests::model_owned_families_pipeline_with_an_exact_or_a_discarding_teardown ... ok +test engine::direct_decode_tests::stop_id_finish_matches_the_synchronous_loop ... ok +test engine::direct_decode_tests::token_bias_is_folded_into_the_pipelined_draw ... ok +test engine::eval_fault_tests::discard_failures_release_the_sequence ... ok +test engine::eval_fault_tests::failed_draws_unwind_before_synchronous_fallback ... ok +test engine::eval_fault_tests::failed_in_loop_submits_preserve_seeded_adaptive_stream ... ok +test engine::eval_fault_tests::failed_in_loop_submits_preserve_seeded_fused_stream ... ok +test engine::eval_fault_tests::failed_in_loop_submits_preserve_seeded_mirostat_stream ... ok +test engine::eval_fault_tests::failed_in_loop_submits_preserve_seeded_penalty_stream ... ok +test engine::eval_fault_tests::failed_readbacks_unwind_to_the_committed_prefix ... ok +test engine::eval_fault_tests::failed_submit_commits_pending_eos_and_callback_stop ... ok +test engine::eval_fault_tests::failed_submit_commits_seeded_pending_callback_stop ... ok +test engine::eval_fault_tests::failed_submit_commits_seeded_pending_eos ... ok +test engine::eval_fault_tests::readback_helpers_return_errors ... ok +test engine::eval_fault_tests::rng_snapshots_restore_only_the_owning_threads_key_sequence ... ok +test engine::eval_fault_tests::sampler_reads_report_eval_errors_and_failed_draws_restore_feedback ... ok +test engine::speculative_plain_tests::a_drafter_that_does_not_pipeline_stays_synchronous ... ok +test engine::speculative_plain_tests::a_proposal_during_a_pipelined_run_is_verified_next_round ... ok +test engine::speculative_plain_tests::a_short_scoring_forward_is_an_error_not_a_panic ... ok +test engine::speculative_plain_tests::a_short_verify_forward_is_an_error_not_a_panic ... ok +test engine::speculative_plain_tests::an_eos_inside_a_verify_block_leaves_the_offset_where_the_step_does ... ok +test engine::speculative_plain_tests::failed_plain_readback_unwinds_to_the_committed_prefix ... ok +test engine::speculative_plain_tests::logits_vocab_accepts_only_a_single_row_of_enough_positions ... ok +test engine::speculative_plain_tests::pipelined_plain_rounds_match_the_synchronous_rounds ... ok +test engine::speculative_plain_tests::stop_and_callback_inside_a_pipelined_run_match_the_synchronous_rounds ... ok +test engine::tests::a_batch_of_one_runs_the_per_row_chain ... ok +test engine::tests::a_failed_row_never_touches_its_neighbours ... ok +test engine::tests::direct_engine_announces_the_whole_prompt_to_every_prefill_piece ... ok +test engine::tests::direct_engine_honors_the_callback_and_withholds_eos ... ok +test engine::tests::direct_engine_injects_the_resolved_kv_modes_into_the_model ... ok +test engine::tests::direct_engine_matches_a_hand_driven_sequence_and_closes_it ... ok +test engine::tests::direct_engine_token_bias_composition ... ok +test engine::tests::finish_rows_fails_every_row_on_a_token_count_mismatch ... ok +test engine::tests::fused_and_per_row_paths_agree_for_greedy_rows ... ok +test engine::tests::fused_readback_token_count_mismatch_fails_every_row ... ok +test engine::tests::mask_override_and_matcher_run_in_the_per_row_chain ... ok +test engine::tests::open_allocates_and_close_releases ... ok +test engine::tests::prefill_cohort_covers_every_row ... ok +test engine::tests::prefill_pieces_match_a_single_pass_for_every_partition ... ok +test engine::tests::prefill_returns_last_real_logits_and_trims_padding ... ok +test engine::tests::shared_fused_params_needs_every_row ... ok +test engine::tests::speculative_step_is_undone_by_unwind_appends ... ok +test engine::tests::step_names_the_missing_row ... ok +test engine::tests::step_of_many_runs_the_batched_forward_in_row_order ... ok +test engine::tests::step_of_one_runs_the_single_row_forward ... ok +test engine::tests::step_refuses_a_row_count_that_differs_from_the_batch ... ok +test engine::tests::submit_then_finish_rows_is_one_pipelined_step ... ok +test engine::verify_round_tests::a_failed_round_unwinds_every_append ... ok +test engine::verify_round_tests::a_round_commits_the_accepted_prefix_and_unwinds_the_rest ... ok +test engine::verify_round_tests::the_predicate_refuses_a_model_a_trim_cannot_roll_back ... ok + +test result: ok. 63 passed; 0 failed; 0 ignored; 0 measured; 2020 filtered out; finished in 1.50s + diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/environment.txt b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/environment.txt new file mode 100644 index 000000000..c16827068 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/environment.txt @@ -0,0 +1,34 @@ +2026-10-11T10:14:59+00:00 +spark-102 +Linux spark-102 7.0.0-1019-nvidia #19~24.04.2-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 6 18:35:11 UTC 2 aarch64 aarch64 aarch64 GNU/Linux +ecb89a308aecc468dd1886d9c9f4982f556f3c93 +configured_mlx_pin=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +fetched_mlx_source=target/test-fast/build/mlxcel-core-0cdeace19df9bd92/out/build/_deps/mlx-src commit=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +fetched_mlx_source=target/test-fast/build/mlxcel-core-3f6a9569cabe31aa/out/build/_deps/mlx-src commit=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +fetched_mlx_source=target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/_deps/mlx-src commit=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +fetched_mlx_source=target/test-fast/build/mlxcel-core-951f0822c2faa7cb/out/build/_deps/mlx-src commit=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +fetched_mlx_source=target/test-fast/build/mlxcel-core-d7f780f67d2a58c2/out/build/_deps/mlx-src commit=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +128d6b6c4e06c8a0a2c473d268c318110c643a583f8e233e1b3e35adf7aad9c8 target/test-fast/deps/mlxcel_core-5feb86ac05c84f53 +06b48baeb28c191f550564075a65e7e1f6ee6c9dc594437a63a467319c1ab988 /tmp/mlxcel-auto-plan/2259-quiet-timing.sh +Sun Oct 11 19:14:59 2026 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 580.178.04 Driver Version: 580.178.04 CUDA Version: 13.0 | ++-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA GB10 On | 0000000F:01:00.0 Off | N/A | +| N/A 53C P8 5W / N/A | Not Supported | 0% Default | +| | | N/A | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +[] +2026-10-11T10:15:55+00:00 diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/fmt.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/fmt.log new file mode 100644 index 000000000..e69de29bb diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/hostgate.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/hostgate.log new file mode 100644 index 000000000..af5db66fc --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/hostgate.log @@ -0,0 +1,2 @@ +gate_wait_s= 0 +ci_job_running= False diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/outer-invocation.txt b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/outer-invocation.txt new file mode 100644 index 000000000..6ea41009c --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/outer-invocation.txt @@ -0,0 +1,4 @@ +outer_driver=/tmp/mlxcel-auto-plan/batch2-quiet-measurements.sh +outer_driver_sha256=af9da7d10881230ecf0aa4cc5c7127921bb238a2a1e1cab7b832f85c773baed0 +outer_invocation=gpu-lock run --tag issue-2259-quiet-timing -- bash /tmp/mlxcel-auto-plan/2259-quiet-timing.sh +session_invocation=bash /tmp/mlxcel-auto-plan/batch2-quiet-measurements.sh > /tmp/mlxcel-auto-plan/batch2-quiet-measurements.log 2>&1 diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt new file mode 100644 index 000000000..30a53c933 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt @@ -0,0 +1,22 @@ +measurement_checkout=ecb89a308aecc468dd1886d9c9f4982f556f3c93 +measurement_checkout_clean=true +base_commit=73700aafa6a792f07830578c0c26137cea3a038b +test_artifact=target/test-fast/deps/mlxcel_core-5feb86ac05c84f53 +test_artifact_sha256=128d6b6c4e06c8a0a2c473d268c318110c643a583f8e233e1b3e35adf7aad9c8 +inner_driver=/tmp/mlxcel-auto-plan/2259-quiet-timing.sh +inner_driver_sha256=06b48baeb28c191f550564075a65e7e1f6ee6c9dc594437a63a467319c1ab988 +outer_driver=/tmp/mlxcel-auto-plan/batch2-quiet-measurements.sh +outer_driver_sha256=af9da7d10881230ecf0aa4cc5c7127921bb238a2a1e1cab7b832f85c773baed0 +outer_invocation=gpu-lock run --tag issue-2259-quiet-timing -- bash /tmp/mlxcel-auto-plan/2259-quiet-timing.sh +hostgate_source=docs/benchmark_results/data/sdpa-plan-bucket-gb10-2026-09-12/harness/hostgate.py +hostgate_sha256=69e6d3f468c42ba7e478578e3bde9efb9c5a30165866d8e9da19440a6eb9413f +hostgate_quiet_semantics=12 samples at 5-second intervals; first sample immediate; 55 continuous quiet seconds +hostgate_result=gate_wait_s=0 means no busy reset; ci_job_running=False after the sustained gate +configured_mlx_pin=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +fetched_mlx_commits=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +benchmark_started_utc=2026-10-11T10:14:59+00:00 +benchmark_completed_utc=2026-10-11T10:15:55+00:00 +timing_semantics=host matching and lazy MLX graph construction; returned arrays were not evaluated; not GPU end-to-end latency +benchmark_log_sha256=9169c635ff03036e4d3046b1e276f4f4c2be21af51cf7a088625bbd470457c9c +environment_log_sha256=8e1104d2d3082f2230107434f3305fcfd92f6b353238695485dad9f53386ea47 +hostgate_log_sha256=890d1fcf0b4cab5cca9f5fd990ed2cc34f8e5c7a46097ed471b30bd9e15bd3d1 diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/sampling-tests.log b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/sampling-tests.log new file mode 100644 index 000000000..a04a39589 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/sampling-tests.log @@ -0,0 +1,321 @@ +warning: mlxcel-core@0.7.0: In file included from cpp/mlx_cxx_bridge.cpp:7: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In function 'mlx_cxx::{anonymous}::get_compiled_q_path_proportional(int, int, int, float)::&)>': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3851:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3851 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3850:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3850 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2190:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2190 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2189:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2189 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3362:24: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 3362 | ref_rms); +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: At global scope: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3792:24: warning: 'mlx_cxx::{anonymous}::ADD3_LN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3792 | static const char* ADD3_LN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3027:24: warning: 'mlx_cxx::{anonymous}::MOE_FC1_RELU2_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3027 | static const char* MOE_FC1_RELU2_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2847:24: warning: 'mlx_cxx::{anonymous}::MOE_DOWN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2847 | static const char* MOE_DOWN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2708:24: warning: 'mlx_cxx::{anonymous}::MOE_GATEUP_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2708 | static const char* MOE_GATEUP_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2129:24: warning: 'mlx_cxx::{anonymous}::MAMBA1_SCAN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2129 | static const char* MAMBA1_SCAN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:752:24: warning: 'mlx_cxx::{anonymous}::SSM_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 752 | static const char* SSM_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:452:24: warning: 'mlx_cxx::{anonymous}::XIELU_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 452 | static const char* XIELU_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:159:24: warning: 'mlx_cxx::{anonymous}::BITLINEAR_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 159 | static const char* BITLINEAR_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp: In function 'void mlx_cxx::nemotron_decode_step(uint64_t, const MlxArray&, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, std::unique_ptr&, rust::cxxbridge1::Slice >, rust::cxxbridge1::Slice >)': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: In file included from ../mlx-cpp/turbo/kv_inplace_write.cpp:23: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2259/target/test-fast/build/mlxcel-core-7abde4bba84d02cd/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ + Finished `test-fast` profile [optimized] target(s) in 0.15s + Running unittests src/lib.rs (target/test-fast/deps/mlxcel_core-5feb86ac05c84f53) + +running 211 tests +test drafter::masked_embedder::tests::argmax_never_picks_masked_token_with_argmax_sampling ... ok +test generate::tests::sampling_config_with_temperature_only_changes_temperature ... ok +test sampling::tests::adaptive_p_disabled_target_stays_on_the_fused_path ... ok +test sampling::tests::adaptive_p_state_accept_updates_ema_only_on_matching_token ... ok +test sampling::tests::adaptive_p_transform_prefers_tokens_near_the_target ... Compiling CUDA kernels (first run on this host; cached for later runs)... +ok +test sampling::tests::apply_dry_penalty_penalizes_followup_token_after_suffix_match ... ok +test sampling::tests::apply_frequency_presence_penalty_accumulates_by_token_count ... ok +test sampling::tests::apply_repetition_penalty_modifies_selected_logits ... ok +test sampling::tests::apply_row_filters_disabled_returns_input_pointer_unchanged ... ok +test sampling::tests::apply_row_filters_greedy_skips_active_filter ... ok +test sampling::tests::apply_row_filters_treats_negative_sentinel_as_disabled ... ok +test sampling::tests::apply_token_bias_empty_noop ... ok +test sampling::tests::apply_token_bias_multiple_entries ... ok +test sampling::tests::apply_token_bias_neg_inf_forces_zero_prob ... ok +test sampling::tests::apply_token_bias_negative_index_ignored ... ok +test sampling::tests::apply_token_bias_out_of_range_ignored ... ok +test sampling::tests::apply_token_bias_positive_adds ... ok +test sampling::tests::apply_token_bias_rows_biases_each_row_independently ... ok +test sampling::tests::apply_token_bias_rows_ignores_out_of_range_ids ... ok +test sampling::tests::apply_token_bias_rows_is_a_noop_without_bias ... ok +test sampling::tests::apply_token_bias_rows_matches_single_row_helper ... ok +test sampling::tests::apply_token_bias_rows_rejects_a_row_count_mismatch - should panic ... ok +test sampling::tests::apply_xtc_filter_allowlist_tokens_survive_removal ... ok +test sampling::tests::apply_xtc_filter_is_noop_with_fewer_than_two_candidates ... ok +test sampling::tests::apply_xtc_filter_is_noop_with_zero_candidates ... ok +test sampling::tests::apply_xtc_filter_keeps_least_probable_above_threshold_token ... ok +test sampling::tests::apply_xtc_step_gate_at_one_always_fires ... ok +test sampling::tests::apply_xtc_step_gate_at_zero_never_fires ... ok +test sampling::tests::apply_xtc_step_mid_probability_is_reproducible_for_the_same_seed ... ok +test sampling::tests::batched_fused_sample_accepts_2d_logits ... ok +test sampling::tests::batched_fused_sample_greedy_matches_per_row ... ok +test sampling::tests::batched_fused_sample_honors_p_less ... ok +test sampling::tests::batched_fused_sample_honors_top_n_sigma ... ok +test sampling::tests::batched_fused_sample_honors_typical_p ... ok +test sampling::tests::batched_fused_sample_single_row_no_regression ... ok +test sampling::tests::batched_sample_greedy_selects_argmax_per_sequence ... ok +test sampling::tests::batched_sample_single_sequence_matches_unbatched ... ok +test sampling::tests::compute_logprobs_returns_none_when_disabled ... ok +test sampling::tests::compute_logprobs_returns_selected_token_logprob ... ok +test sampling::tests::compute_logprobs_returns_top_k_alternatives_sorted_descending ... ok +test sampling::tests::compute_logprobs_selected_token_bf16_matches_f32 ... ok +test sampling::tests::compute_logprobs_selected_token_f16_matches_f32 ... ok +test sampling::tests::compute_logprobs_top_k_bf16_no_panic_matches_f32 ... ok +test sampling::tests::compute_logprobs_top_k_capped_at_vocab_size ... ok +test sampling::tests::compute_logprobs_top_k_f16_matches_f32 ... ok +test sampling::tests::compute_logprobs_top_k_f32_values_correct ... ok +test sampling::tests::compute_logprobs_top_k_zero_fast_path_f16_matches_full_softmax ... ok +test sampling::tests::compute_logprobs_top_k_zero_fast_path_matches_full_softmax ... ok +test sampling::tests::config_supports_fused_batch_false_for_penalties ... ok +test sampling::tests::config_supports_fused_batch_false_for_token_bias ... ok +test sampling::tests::config_supports_fused_batch_false_for_xtc ... ok +test sampling::tests::config_supports_fused_batch_true_for_plain_configs ... ok +test sampling::tests::config_supports_fused_batch_true_for_top_n_sigma ... ok +test sampling::tests::cpp_xtc_gate_decides_the_removal ... ok +test sampling::tests::cpp_xtc_greedy_draw_is_the_argmax_of_the_filtered_row ... ok +test sampling::tests::cpp_xtc_matches_the_rust_reference_filter ... ok +test sampling::tests::dry_breaker_automaton_large_exact_list_is_additive ... ok +test sampling::tests::dry_breaker_automaton_long_shared_tails_are_additive ... ok +test sampling::tests::dry_breaker_automaton_preserves_overlap_and_boundary_semantics ... ok +test sampling::tests::dry_breaker_head_with_empty_tail_breaks_matching ... ok +test sampling::tests::dry_breaker_head_with_tail_requires_the_full_sequence ... ok +test sampling::tests::dry_full_history_sentinel_scans_everything ... ok +test sampling::tests::dry_penalizes_at_exactly_the_allowed_length ... ok +test sampling::tests::dry_penalty_last_n_zero_disables_dry ... ok +test sampling::tests::dry_reverse_z_matches_reference_lcg_cases ... ok +test sampling::tests::dry_reverse_z_preserves_breaker_boundaries ... ok +test sampling::tests::dry_reverse_z_step_cost_is_linear ... ok +test sampling::tests::dry_step_cost_report ... ignored, manual timing evidence for issue #2259 +test sampling::tests::dynatemp_keeps_temperature_zero_config_off_the_greedy_path ... ok +test sampling::tests::dynatemp_matches_host_reference_on_peaked_row ... ok +test sampling::tests::dynatemp_single_candidate_passes_through ... ok +test sampling::tests::dynatemp_uniform_row_scales_by_max_temp ... ok +test sampling::tests::effective_token_distribution_zeroes_top_n_sigma_masked_tokens ... ok +test sampling::tests::effective_token_distribution_zeroes_typical_p_masked_tokens ... ok +test sampling::tests::frequency_presence_sparse_matches_vocab_reference ... ok +test sampling::tests::fused_sample_params_match_detects_each_difference ... ok +test sampling::tests::fused_sample_params_matches_compares_p_less ... ok +test sampling::tests::fused_sample_params_matches_compares_top_n_sigma ... ok +test sampling::tests::fused_sample_params_matches_compares_typical_p ... ok +test sampling::tests::fused_sample_params_normalizes_inert_top_n_sigma ... ok +test sampling::tests::fused_sample_params_normalizes_inert_typical_p ... ok +test sampling::tests::fused_sample_probs_equals_softmax_of_filtered_over_t ... ok +test sampling::tests::greedy_path_unchanged ... ok +test sampling::tests::inert_new_fields_keep_the_fused_path_and_identical_bytes ... ok +test sampling::tests::min_p_min_keep_forces_floor_in_probability_order ... ok +test sampling::tests::min_p_support_is_temperature_invariant ... ok +test sampling::tests::mirostat_ignores_penalties_and_truncation_filters ... ok +test sampling::tests::mirostat_temperature_zero_draws_the_argmax ... ok +test sampling::tests::mirostat_v1_peaked_row_truncates_to_the_argmax ... ok +test sampling::tests::mirostat_v2_tiny_mu_selects_the_argmax_and_updates_mu ... ok +test sampling::tests::p_less_filter_argmax_always_survives ... ok +test sampling::tests::p_less_filter_higher_temp_keeps_more ... ok +test sampling::tests::p_less_filter_ignores_neg_inf_entries ... ok +test sampling::tests::p_less_filter_matches_host_reference ... ok +test sampling::tests::p_less_filter_peaked_keeps_top_only ... ok +test sampling::tests::p_less_filter_rows_independent ... ok +test sampling::tests::p_less_skipped_on_greedy_and_disabled_returns_input_pointer ... ok +test sampling::tests::penalty_last_n_zero_disables_the_stage_and_stays_fused_eligible ... ok +test sampling::tests::penalty_window_sentinels ... ok +test sampling::tests::post_sampling_probs_reports_linear_probabilities_and_drops_zeros ... ok +test sampling::tests::repetition_penalty_windowed_ignores_tokens_outside_the_window ... ok +test sampling::tests::row_supports_fused_batch_gate_off_for_incompatible_config ... ok +test sampling::tests::row_supports_fused_batch_gate_off_for_per_row_obligations ... ok +test sampling::tests::row_supports_fused_batch_gate_on_for_plain_row ... ok +test sampling::tests::sample_token_optimized_respects_greedy_argmax_path ... ok +test sampling::tests::sample_token_optimized_with_state_dry_only_allocates_no_state ... ok +test sampling::tests::sample_token_optimized_with_state_greedy_parity_over_sequence ... ok +test sampling::tests::sample_token_optimized_with_state_no_penalty_allocates_no_state ... ok +test sampling::tests::sampler_state_frequency_presence_matches_rebuild_f16 ... ok +test sampling::tests::sampler_state_frequency_presence_matches_rebuild_over_sequence ... ok +test sampling::tests::sampler_state_frequency_presence_preserves_bf16_reference ... ok +test sampling::tests::sampler_state_repetition_matches_rebuild_f16 ... ok +test sampling::tests::sampler_state_repetition_matches_rebuild_over_sequence ... ok +test sampling::tests::sampler_state_skips_aggregates_outside_the_full_history_window ... ok +test sampling::tests::sampler_state_sync_handles_append_shrink_and_divergence ... ok +test sampling::tests::suppress_tokens_empty_slice_is_noop ... ok +test sampling::tests::suppress_tokens_forces_neg_inf_and_drives_probability_to_zero ... ok +test sampling::tests::temperature_one_support_unchanged ... ok +test sampling::tests::top_n_sigma_filter_f16_large_vocab_does_not_overflow ... ok +test sampling::tests::top_n_sigma_filter_ignores_neg_inf_entries ... ok +test sampling::tests::top_n_sigma_filter_keeps_within_n_std ... ok +test sampling::tests::top_n_sigma_filter_nan_entry_does_not_mask_the_row ... ok +test sampling::tests::top_n_sigma_filter_rows_independent ... ok +test sampling::tests::top_n_sigma_skipped_on_greedy ... ok +test sampling::tests::top_p_filter_batched_equals_per_row ... ok +test sampling::tests::top_p_filter_single_row_all_pass_at_one ... ok +test sampling::tests::top_p_filter_single_row_nucleus_boundary ... ok +test sampling::tests::top_p_min_keep_floor_never_resurrects_masked_entries ... ok +test sampling::tests::top_p_min_keep_forces_floor_in_probability_order ... ok +test sampling::tests::top_p_support_is_temperature_invariant ... ok +test sampling::tests::typical_p_filter_can_drop_argmax ... ok +test sampling::tests::typical_p_filter_ignores_neg_inf_entries ... ok +test sampling::tests::typical_p_filter_matches_host_reference ... ok +test sampling::tests::typical_p_filter_nan_entry_does_not_mask_the_row ... ok +test sampling::tests::typical_p_filter_preserves_dtype ... ok +test sampling::tests::typical_p_filter_smaller_keeps_fewer ... ok +test sampling::tests::typical_p_min_keep_forces_floor_in_typicality_order ... ok +test sampling::tests::typical_p_one_disables_via_hook ... ok +test sampling::tests::typical_p_runs_on_the_renormalized_top_k_survivors ... ok +test sampling::tests::typical_p_skipped_on_greedy ... ok +test sampling::tests::typical_p_top_k_premask_skips_oversized_k ... ok +test sampling::tests::uniform_fused_batch_params_still_rejects_token_bias ... ok +test sampling::tests::windowed_penalties_match_full_history_when_the_window_covers_it ... ok +test sampling::tests::xtc_min_keep_skips_removal_when_too_few_would_survive ... ok +test sampling::tests::xtc_runs_on_renormalised_filtered_row ... ok +test sampling_dispatch::tests::draining_an_empty_queue_is_a_no_op ... ok +test sampling_dispatch::tests::the_production_round_cap_is_the_documented_value ... ok +test sampling_dispatch::tests::the_reset_clears_both_cap_overflow_counters ... ok +test sampling_dispatch::tests::the_threadgroup_size_is_a_power_of_two ... ok +test sampling_fixed_key_tests::gumbel_kernel_and_graph_categorical_draw_one_distribution ... ok +test sampling_fixed_key_tests::gumbel_kernel_matches_the_keyed_graph_argmax ... ok +test sampling_fixed_key_tests::rejection_kernel_and_graph_chain_draw_one_distribution ... ok +test sampling_fixed_key_tests::rejection_kernel_draws_stay_inside_the_filtered_support ... ok +test sampling_fixed_key_tests::rejection_kernel_draws_the_keyed_token_under_min_p ... ok +test sampling_gumbel_tests::filtered_configs_never_reach_the_gumbel_kernel ... ok +test sampling_gumbel_tests::fused_sample_routes_the_no_filter_path_to_the_gumbel_kernel ... ok +test sampling_gumbel_tests::greedy_sampling_is_byte_identical_to_argmax ... ok +test sampling_gumbel_tests::gumbel_noise_stays_inside_its_finite_range ... ok +test sampling_gumbel_tests::gumbel_sample_index_does_not_depend_on_the_launch_split ... ok +test sampling_gumbel_tests::gumbel_sample_is_reproducible_for_a_fixed_seed ... ok +test sampling_gumbel_tests::gumbel_sample_matches_softmax_on_bimodal_logits ... ok +test sampling_gumbel_tests::gumbel_sample_matches_softmax_on_flat_logits ... ok +test sampling_gumbel_tests::gumbel_sample_matches_softmax_on_peaked_logits ... ok +test sampling_gumbel_tests::gumbel_sample_matches_softmax_over_a_large_vocabulary ... ok +test sampling_gumbel_tests::gumbel_sample_matches_softmax_with_masked_entries ... ok +test sampling_gumbel_tests::gumbel_sample_never_selects_masked_tokens ... ok +test sampling_gumbel_tests::gumbel_sample_temperature_scaling_matches_reference ... ok +test sampling_gumbel_tests::split_heuristic_stays_within_its_documented_range ... ok +test sampling_rejection_tests::a_failed_stashed_launch_is_dropped_without_throwing_or_consuming_its_error ... ok +test sampling_rejection_tests::a_fixed_seed_reproduces_the_stream ... ok +test sampling_rejection_tests::a_large_vocabulary_sends_the_joint_config_back_to_the_stock_chain ... ok +test sampling_rejection_tests::a_near_uniform_152k_row_converges_inside_the_round_cap ... ok +test sampling_rejection_tests::a_starved_round_cap_falls_back_and_the_event_is_counted ... ok +test sampling_rejection_tests::all_three_filters_compose_into_one_threshold ... ok +test sampling_rejection_tests::cap_overflow_is_detected_without_waiting_on_the_production_path ... ok +test sampling_rejection_tests::fused_sample_routes_only_the_configurations_that_measured_faster ... ok +test sampling_rejection_tests::greedy_and_no_filter_paths_report_that_they_bypass_the_rejection_kernel ... ok +test sampling_rejection_tests::joint_top_k_top_p_support_is_a_superset_of_the_stock_chain ... ok +test sampling_rejection_tests::masked_entries_are_never_sampled_and_do_not_shift_the_support ... ok +test sampling_rejection_tests::min_p_filters_the_stock_chain ... ok +test sampling_rejection_tests::min_p_needs_no_rejection_round ... ok +test sampling_rejection_tests::min_p_support_and_frequencies_match_the_stock_filter ... ok +test sampling_rejection_tests::per_row_parameters_reach_the_kernel_from_one_launch ... ok +test sampling_rejection_tests::rejection_kernel_filter_on_untempered_draw_on_tempered ... ok +test sampling_rejection_tests::subnormal_scale_probabilities_do_not_break_the_shrinking_interval ... ok +test sampling_rejection_tests::the_forced_entry_point_is_the_one_that_synchronizes ... ok +test sampling_rejection_tests::the_joint_vocab_cap_is_the_measured_crossover ... ok +test sampling_rejection_tests::the_production_sampling_call_never_synchronizes ... ok +test sampling_rejection_tests::the_reference_arm_stays_on_the_argpartition_chain ... ok +test sampling_rejection_tests::the_reported_distribution_matches_what_the_sampler_draws ... ok +test sampling_rejection_tests::the_routing_policy_matches_the_measured_matrix ... ok +test sampling_rejection_tests::the_two_filter_orderings_really_do_differ_for_top_k_plus_top_p ... ok +test sampling_rejection_tests::top_k_frequencies_match_the_renormalised_truncated_distribution ... ok +test sampling_rejection_tests::top_k_holds_under_temperature_scaling ... ok +test sampling_rejection_tests::top_k_support_matches_the_argpartition_mask_including_ties ... ok +test sampling_rejection_tests::top_p_frequencies_match_the_renormalised_truncated_distribution ... ok +test sampling_rejection_tests::top_p_support_matches_the_stock_nucleus_filter ... ok +test sampling_row_step::fused_tests::fused_eligible_is_false_for_every_stage_the_dispatch_cannot_run ... ok +test sampling_row_step::fused_tests::fused_eligible_matches_the_pre_2169_gate ... ok +test sampling_row_step::tests::cli_loops_and_server_row_step_emit_identical_streams ... ok +test sampling_row_step::tests::cli_loops_carry_feedback_state_across_steps ... ok +test sampling_row_step::tests::resolve_returns_sampled_and_final_and_confirms_only_the_final_token ... ok +test sampling_row_step::tests::state_is_created_for_history_and_feedback_samplers_only ... ok +test sampling_row_step::tests::the_mask_runs_before_the_chain_and_its_error_aborts_the_draw ... ok +test sampling_token_bias::tests::byte_fragment_tags_survive_the_language_fallback ... ok +test sampling_token_bias::tests::empty_sources_compose_to_an_empty_map ... ok +test sampling_token_bias::tests::language_bias_applies_only_without_a_request_bias ... ok +test sampling_token_bias::tests::output_suppression_always_wins ... ok +test sampling_token_bias::tests::output_suppression_beats_non_finite_request_bias ... ok +test sampling_top_k_vocab_tests::top_k_at_or_above_vocab_is_no_top_k ... ok +test sampling_top_k_vocab_tests::top_k_at_or_above_vocab_is_no_top_k_with_xtc_active ... ok + +test result: ok. 210 passed; 0 failed; 1 ignored; 0 measured; 1872 filtered out; finished in 18.07s + diff --git a/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md b/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md new file mode 100644 index 000000000..c849d0424 --- /dev/null +++ b/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md @@ -0,0 +1,53 @@ +# DRY penalty step cost on GB10 (2026-10-11) + +Issue #2259 replaces the DRY sampler's repeated backward suffix scans with a reverse-Z pass, preprocesses request-supplied breakers with a reversed-token Aho-Corasick matcher, and scatters only touched token IDs. This record measures host matching and lazy MLX graph construction for the exact pre-change reference retained in the test module and the new production path on a period-3 stream. It does not measure GPU end-to-end latency because the returned arrays are not evaluated. + +## Environment and provenance + +- Host: NVIDIA GB10 on `spark-102` +- Backend: CUDA, pinned MLX `81ba1c6a0e50a9268b931579c2d4f1158b9aab5a` +- Source commit: `ecb89a308aecc468dd1886d9c9f4982f556f3c93` +- Base commit: `73700aafa6a792f07830578c0c26137cea3a038b` +- Profile: `test-fast` +- Measured test artifact: `target/test-fast/deps/mlxcel_core-5feb86ac05c84f53`, SHA-256 `128d6b6c4e06c8a0a2c473d268c318110c643a583f8e233e1b3e35adf7aad9c8` +- Inner driver SHA-256: `06b48baeb28c191f550564075a65e7e1f6ee6c9dc594437a63a467319c1ab988` +- Outer driver SHA-256: `af9da7d10881230ecf0aa4cc5c7127921bb238a2a1e1cab7b832f85c773baed0` +- Host gate: `hostgate.py` SHA-256 `69e6d3f468c42ba7e478578e3bde9efb9c5a30165866d8e9da19440a6eb9413f`; 12 quiet samples at five-second intervals enforce 55 continuous seconds +- Host-gate result: no busy-process reset (`gate_wait_s=0`) and no active CI job after the sustained quiet interval +- Benchmark start/end UTC: `2026-10-11T10:14:59+00:00` / `2026-10-11T10:15:55+00:00` + +The outer driver ran `gpu-lock run --tag issue-2259-quiet-timing -- bash /tmp/mlxcel-auto-plan/2259-quiet-timing.sh`. The raw package records that invocation, checkout state, test artifact provenance, host details, stdout/stderr, validation logs, and checksums. + +## Method + +`sampling::tests::dry_step_cost_report` constructs one row of four f32 logits and period-3 token histories of 1,024, 4,096, and 16,384 tokens, with no configured breakers. For each window it measures the exact pre-#2259 reference and the new production path. Reference repetitions are 3, 3, and 1; production repetitions are 20 for each window. Values below are elapsed host microseconds per call as printed by the test. Since MLX is lazy and the arrays are only passed to `black_box`, the measurement includes host matching and graph construction but excludes device evaluation. + +## Results + +| Window | Reference µs/step | New µs/step | Speedup | +|---:|---:|---:|---:| +| 1,024 | 619.751 | 23.607 | 26.25× | +| 4,096 | 4889.370 | 29.997 | 163.00× | +| 16,384 | 77199.293 | 111.433 | 692.79× | + +From 1,024 to 16,384 tokens, the reference cost grew 124.57× while the new path grew 4.72×. At 16,384 tokens the measured per-step speedup was 692.79×. + +## Correctness and work bounds + +The performance result is paired with exact logit parity rather than inferred from generated text. The CUDA sampling selection passed 210 tests with one ignored manual timing test, including 512 deterministic DRY reference cases and f32/f16/bf16 frequency/presence parity. The engine selection passed 63 tests. Instrumented tests bound the period-3 window pass below `8N`, the exact-breaker build and scan additively, and a shared-trunk failure-chain family below `8P` build and `3N` scan. + +## Raw evidence + +The package at `data/dry-penalty-step-cost-gb10-2026-10-11/` contains: + +- `dry-step-cost.log`: byte-preserved benchmark command output. +- `2259-quiet-timing.sh`: byte-preserved inner timing driver. +- `outer-invocation.txt`: exact `gpu-lock` invocation and outer driver hash. +- `hostgate.log`: byte-preserved host-gate result. +- `environment.txt`: byte-preserved host, GPU, checkout, MLX, timing, and artifact output. +- `provenance.txt`: derived source, artifact, driver, host-gate, and raw-log hash mapping. +- `core-build.log`, `clippy.log`, `fmt.log`, `sampling-tests.log`, `engine-tests.log`: byte-preserved final validation logs. +- `README.md`: artifact interpretation and replay instructions. +- `SHA256SUMS`: checksums for the package. + +After squash merge, rebuild the CUDA test artifact and reproduce from the merged commit using the driver command recorded in `README.md`; the production and test code should differ from the measured source only by the squash identity and evidence/report additions unless the PR changes after this record is captured. From 499f8626907bdd02fcaf184b3b067510eced3377 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 20:14:33 +0900 Subject: [PATCH 4/4] docs: finalize PR 2272 DRY cost evidence Add bilingual technical reports and preserve the original two-commit implementation patch used for GB10 timing. Record the measured and rebased source identities, prove the scoped core, native, and lockfile diff is empty, and document replay from the public measurement base. Validation: - `sha256sum --check SHA256SUMS` - Replayed the archived patch from `73700aaf` and matched tree `58b9a04161eaf88048699d0930a69dd59ccdd7b3` - `git diff --cached --check` Refs #2259, #2272 --- ...-bound-dry-sparse-penalties-20261011.en.md | 89 ++ ...-bound-dry-sparse-penalties-20261011.ko.md | 89 ++ .../.gitattributes | 1 + .../README.md | 16 +- .../SHA256SUMS | 8 +- .../implementation-73700aaf-to-ecb89a3.patch | 1118 +++++++++++++++++ .../provenance.txt | 12 + .../rebase-equivalence.txt | 19 + .../dry-penalty-step-cost-gb10-2026-10-11.md | 12 +- 9 files changed, 1357 insertions(+), 7 deletions(-) create mode 100644 TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.en.md create mode 100644 TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.ko.md create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/implementation-73700aaf-to-ecb89a3.patch create mode 100644 docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/rebase-equivalence.txt diff --git a/TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.en.md b/TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.en.md new file mode 100644 index 000000000..6db91b331 --- /dev/null +++ b/TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.en.md @@ -0,0 +1,89 @@ +# Technical Report: PR #2272 - Bound DRY and sparse penalty step costs + +**Date**: 2026-10-11 +**Status**: Completed and verified on CUDA (GB10) +**Languages**: Rust +**Risk Level**: Medium + +## Executive summary + +This change removes two request-amplified costs from sampling. DRY suffix matching now uses reverse Z, and breaker recognition uses a reversed-token Aho-Corasick matcher, so one step takes expected `O(N + P)` host work for selected history length `N` and configured breaker-pattern tokens `P` instead of quadratic suffix scans or `N × P` breaker checks. DRY and rebuilt frequency/presence updates also scatter only touched token IDs rather than allocating and uploading vocabulary-sized host buffers. + +The implementation preserves the exact dtype and element behavior of the prior paths. This required following the pinned CUDA MLX promotion table rather than the issue's original assumption: mixed bf16/f32 binary arithmetic can remain bf16 on CUDA. Scalar zero arithmetic resolves the same backend-specific dtype before sparse scatter, while the established incremental frequency/presence path retains its explicit f32 promotion. + +## 1. Problem statement + +The previous DRY implementation indexed token positions, then walked backward for every earlier occurrence of the current token. On a period-`p` stream with a full-history window of length `N`, one step performed roughly `N² / (2p)` comparisons. A server request can select full history, which includes the prompt, so this work increased with request-controlled history length. + +Breaker handling also needed an explicit bound. Calling the preserved scalar breaker predicate at every position would scan exact-ID vectors and multi-token tail collections repeatedly, creating expected work proportional to the history window multiplied by breaker configuration size. Separately, DRY and rebuilt frequency/presence penalties constructed vocabulary-sized host vectors even when only a few token IDs received penalties. + +## 2. Change summary + +### 2.1 Bounded DRY matching + +`dry_penalty_map` reverses the selected history and runs the Z algorithm once. For an earlier occurrence at `pos`, `Z[last_pos - pos]` gives the same common-suffix length as the former backward walk. A forward pass stores the most recent breaker strictly before each candidate, preserving the rule that a breaker at the candidate itself does not truncate its match. + +Exact breakers become singleton patterns. A forward multi-token breaker `[head] + tail` is inserted into an automaton as `reverse(tail) + [head]`. A terminal reached at reversed position `r` always maps to original position `N - 1 - r`; propagating terminal state through failure links therefore marks overlapping and suffix patterns without enumerating every match. Trie construction is expected `O(P)` and scanning is expected `O(N)` with the sparse hash transitions. + +### 2.2 Sparse device updates + +DRY penalties are reduced to one maximum penalty per next-token ID and scattered across every logits row with `take_along_axis`, arithmetic, and `put_along_axis`. The rebuilt frequency/presence path now builds only touched IDs and values and uses the same sparse helper. Neither path allocates a host vector proportional to vocabulary size. + +### 2.3 Exact dtype preservation + +The dense DRY reference used `add(logits, f32_buffer)`, and the rebuilt frequency/presence reference used `subtract(logits, f32_buffer)`. On the pinned CUDA MLX patch, bf16/f32 arithmetic can produce bf16; f16 and unpatched backends promote differently. Each rebuild path first performs the corresponding operation against a scalar f32 zero, obtaining the same platform result dtype and unchanged values before sparse scatter. + +The incremental `SamplerState` path behaved differently before this change: it explicitly cast logits to f32 when in-range IDs existed and returned an unchanged copy when all IDs were out of range. The shared sparse helper is fed that promoted array, and both branches retain their prior dtype behavior. + +## 3. Technical decisions + +### 3.1 Stateless per-step matching + +A persistent DRY index could reduce repeated work further, but it would need to rebuild on history divergence, rebase sliding windows, and revisit breaker tails whose status changes when more tokens arrive. The stateless reverse-Z and automaton passes preserve request semantics with bounded work and no sampler-state synchronization contract. + +### 3.2 Compiled matcher instead of repeated predicates + +The first linear DRY draft still invoked `dry_breaks_at` at every position, which left an `O(N × P)` configuration amplifier. The final matcher preprocesses exact IDs and tails once per step. Direct per-position comparison against the unchanged scalar predicate covers negative IDs, duplicates, empty tails, overlapping suffixes, exact window-end matches, and tails that run past the window. + +### 3.3 Backend-resolved promotion instead of forced f32 + +Forcing all sparse rebuild paths to f32 would have changed CUDA bf16 outputs. Scalar zero arithmetic keeps the old binary-operation contract without reconstructing a vocabulary-sized buffer. The pre-existing incremental f32 behavior remains explicit rather than being unified with rebuild semantics. + +## 4. Verification + +The measured source commit is `ecb89a308aecc468dd1886d9c9f4982f556f3c93` on base `73700aafa6a792f07830578c0c26137cea3a038b`. The CUDA GB10 gate completed a core build in 26.74 seconds, clippy in 9.08 seconds, and formatting with no Rust error. The sampling selection passed 210 tests with only `dry_step_cost_report` ignored, and the engine selection passed 63 tests. + +PR publication rebased the implementation onto `3ae626c2a4d67e7628a4050f01cc62b7419296fb`; the resulting checkout is `327b6bf13a31d0f98e4cce717443d03b4be2cb73`. The `sampling.rs` blob remains `59fb6b02c6b54913aa549085fafc090e0b516d55`, and a scoped diff between the measured and publication checkouts across `src/lib/mlxcel-core`, `src/lib/mlx-cpp`, and `Cargo.lock` is empty. The raw evidence package archives the original two-commit format patch from the public measurement base plus hashes and the empty-diff proof. + +The sampling coverage includes 512 deterministic parity cases across window sentinels, periodic and random histories up to 2,048 tokens, batch widths 1 and 2, f32 and bf16 logits, and exact, empty-tail, and nonempty-tail breakers. Frequency/presence rebuild parity covers window sizes 1 and 64, batch widths 1 and 2, and f32, f16, and bf16 logits. A separate regression confirms the incremental bf16 path still promotes to f32 and preserves its all-out-of-range early return. + +Instrumented complexity tests hold reverse-Z work below `8N`, show near-doubling when the window doubles from 4,096 to 8,192, bound a 2,048-entry exact breaker set to additive build and scan work, and exercise a 128-pattern shared-trunk comb that forces long failure walks while staying below `8P` construction and `3N` scanning. + +Independent implementation and security reviews reported no remaining findings. + +## 5. Timing evidence + +The manual test runs both the exact pre-change reference and the new implementation on a period-3 history with windows 1,024, 4,096, and 16,384, vocabulary 4, and no breakers. Reference iterations are 3, 3, and 1; the new path runs 20 iterations at every window. `black_box` receives lazy MLX arrays without evaluating them, so these figures measure host matching and MLX graph construction rather than GPU end-to-end latency. + +| Window | Reference µs/step | New µs/step | Speedup | +|---:|---:|---:|---:| +| 1,024 | 619.751 | 23.607 | 26.25× | +| 4,096 | 4889.370 | 29.997 | 163.00× | +| 16,384 | 77199.293 | 111.433 | 692.79× | + +Across the 16× window increase, reference cost grew 124.57× while the new path grew 4.72×; the 16,384-token speedup was 692.79×. + +Raw command output, the inner and outer drivers, environment, host-gate output, build/test logs, checksums, and binary/source provenance are published under `docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/`. + +## 6. Compatibility and limitations + +No request field, default, or public API changes. Full-history, zero-window, short-history, low-base, breaker, exponent-cap, and max-per-token semantics remain covered by reference parity. The XLA sampler and windowed repetition penalty remain outside this issue. + +The automaton is rebuilt each step, so total work includes configured pattern tokens as well as history length. Hash-map transitions make the stated bound expected rather than worst-case deterministic. The timing test measures one quiet GB10 host and is evidence for scaling and local cost, not a cross-platform throughput guarantee. + +## 7. Related records + +- Issue #2259: implementation requirements +- Issues #2229 and #2256: security-review context +- `docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md`: dated timing record +- `docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/`: raw evidence package diff --git a/TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.ko.md b/TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.ko.md new file mode 100644 index 000000000..b1c2f0b3d --- /dev/null +++ b/TECHNICAL_REPORTS/2272-bound-dry-sparse-penalties-20261011.ko.md @@ -0,0 +1,89 @@ +# 기술 보고서: PR #2272 - DRY 및 sparse penalty 단계 비용 제한 + +**날짜**: 2026-10-11 +**상태**: CUDA(GB10)에서 구현 및 검증 완료 +**언어**: Rust +**위험 수준**: 중간 + +## 요약 + +이 변경은 sampling에서 요청 크기에 따라 증폭되던 비용 두 가지를 제거합니다. DRY suffix matching은 reverse Z를 사용하고 breaker 판별은 reversed-token Aho-Corasick matcher를 사용합니다. 따라서 선택된 history 길이 `N`과 설정된 breaker pattern token 수 `P`에 대해 한 단계의 host 작업이 quadratic suffix scan이나 `N × P` breaker 검사 대신 expected `O(N + P)`가 됩니다. DRY와 rebuild frequency/presence update는 vocabulary 크기의 host buffer를 할당하고 upload하지 않고, 변경되는 token ID만 scatter합니다. + +구현은 기존 경로의 dtype과 element 값을 정확히 보존합니다. 이를 위해 이슈의 원래 가정 대신 고정된 CUDA MLX promotion table을 따랐습니다. CUDA에서는 bf16/f32 mixed binary arithmetic 결과가 bf16으로 남을 수 있습니다. Scalar zero 연산으로 sparse scatter 전에 같은 backend별 dtype을 결정하고, 기존 incremental frequency/presence 경로의 명시적 f32 promotion도 유지합니다. + +## 1. 문제 정의 + +기존 DRY 구현은 token position을 index한 뒤 현재 token의 이전 occurrence마다 뒤로 걸으며 비교했습니다. Period `p`, full-history window 길이 `N`인 stream에서는 한 단계가 약 `N² / (2p)`번 비교했습니다. Server request는 prompt를 포함한 full history를 선택할 수 있으므로 요청이 제어하는 history 길이에 따라 작업량이 증가했습니다. + +Breaker 처리에도 명시적인 상한이 필요했습니다. 각 위치에서 보존된 scalar breaker predicate를 호출하면 exact-ID vector와 multi-token tail collection을 반복해서 scan하므로 history window와 breaker 설정 크기를 곱한 작업이 생깁니다. 이와 별도로 DRY와 rebuild frequency/presence penalty는 소수의 token ID만 바뀌어도 vocabulary 크기의 host vector를 만들었습니다. + +## 2. 변경 요약 + +### 2.1 제한된 DRY matching + +`dry_penalty_map`은 선택된 history를 뒤집고 Z algorithm을 한 번 실행합니다. 이전 occurrence `pos`에 대해 `Z[last_pos - pos]`가 기존 backward walk와 같은 common-suffix 길이를 제공합니다. Forward pass는 각 candidate보다 엄격히 앞에 있는 가장 최근 breaker를 저장하므로 candidate 위치 자체의 breaker가 match를 자르지 않는 규칙을 보존합니다. + +Exact breaker는 singleton pattern이 됩니다. Forward multi-token breaker `[head] + tail`은 automaton에 `reverse(tail) + [head]`로 삽입합니다. Reversed position `r`에서 도달한 terminal은 언제나 원래 위치 `N - 1 - r`에 대응하므로 failure link를 따라 terminal state를 전파하면 모든 match를 나열하지 않고도 overlap 및 suffix pattern을 표시할 수 있습니다. Sparse hash transition을 사용할 때 trie 구성은 expected `O(P)`, scan은 expected `O(N)`입니다. + +### 2.2 Sparse device update + +DRY penalty는 next-token ID별 최대 penalty 하나로 축약한 뒤 `take_along_axis`, 산술 연산, `put_along_axis`로 모든 logits row에 scatter합니다. Rebuild frequency/presence 경로도 변경되는 ID와 값만 만들고 같은 sparse helper를 사용합니다. 어느 경로도 vocabulary 크기에 비례하는 host vector를 할당하지 않습니다. + +### 2.3 정확한 dtype 보존 + +Dense DRY reference는 `add(logits, f32_buffer)`를 사용했고 rebuild frequency/presence reference는 `subtract(logits, f32_buffer)`를 사용했습니다. 고정된 CUDA MLX patch에서 bf16/f32 arithmetic은 bf16을 만들 수 있고, f16 및 patch되지 않은 backend의 promotion은 다릅니다. 각 rebuild 경로는 scalar f32 zero에 대응 연산을 먼저 적용하여 sparse scatter 전에 같은 platform result dtype과 변경되지 않은 값을 얻습니다. + +Incremental `SamplerState` 경로는 이 변경 전부터 다르게 동작했습니다. 범위 안 ID가 있으면 logits를 명시적으로 f32로 cast하고 모든 ID가 범위 밖이면 변경되지 않은 copy를 반환했습니다. Shared sparse helper에는 promotion된 array를 전달하며 두 분기의 기존 dtype 동작을 모두 유지합니다. + +## 3. 기술적 선택과 그 이유 + +### 3.1 Stateless per-step matching + +Persistent DRY index는 반복 작업을 더 줄일 수 있지만 history divergence에서 rebuild하고 sliding window를 rebase하며 token이 추가될 때 결과가 달라지는 breaker tail을 다시 확인해야 합니다. Stateless reverse-Z와 automaton pass는 sampler-state synchronization contract를 추가하지 않고 요청 semantics와 제한된 작업량을 보존합니다. + +### 3.2 반복 predicate 대신 compiled matcher + +첫 linear DRY 초안은 여전히 모든 위치에서 `dry_breaks_at`을 호출하여 `O(N × P)` 설정 증폭을 남겼습니다. 최종 matcher는 단계마다 exact ID와 tail을 한 번 preprocess합니다. 변경하지 않은 scalar predicate와 위치별로 직접 비교하는 test가 negative ID, duplicate, empty tail, overlapping suffix, window 끝에 정확히 맞는 pattern, window 밖으로 넘어가는 tail을 검증합니다. + +### 3.3 강제 f32 대신 backend가 결정하는 promotion + +모든 sparse rebuild 경로를 f32로 강제하면 CUDA bf16 출력이 바뀝니다. Scalar zero arithmetic은 vocabulary 크기의 buffer를 다시 만들지 않고 기존 binary-operation contract를 유지합니다. 기존 incremental f32 동작은 rebuild semantics와 통합하지 않고 명시적으로 유지합니다. + +## 4. 검증 + +측정 source commit은 base `73700aafa6a792f07830578c0c26137cea3a038b` 위의 `ecb89a308aecc468dd1886d9c9f4982f556f3c93`입니다. CUDA GB10 gate에서 core build는 26.74초, clippy는 9.08초에 끝났고 formatting과 Rust 검사가 통과했습니다. Sampling selection은 수동 `dry_step_cost_report`만 ignore한 상태로 210개 test가 통과했고 engine selection은 63개 test가 통과했습니다. + +PR 공개 전 구현을 `3ae626c2a4d67e7628a4050f01cc62b7419296fb` 위로 rebase했으며 결과 checkout은 `327b6bf13a31d0f98e4cce717443d03b4be2cb73`입니다. `sampling.rs` blob은 `59fb6b02c6b54913aa549085fafc090e0b516d55`로 동일하고, 측정 checkout과 공개 checkout 사이에서 `src/lib/mlxcel-core`, `src/lib/mlx-cpp`, `Cargo.lock`을 비교한 diff는 비어 있습니다. Raw evidence package에는 공개된 측정 base에서 시작하는 원본 2-commit format patch와 hash 및 빈 diff 증명이 포함됩니다. + +Sampling coverage는 window sentinel, 길이 2,048까지의 periodic 및 random history, batch width 1과 2, f32 및 bf16 logits, exact/empty-tail/nonempty-tail breaker를 조합한 deterministic parity case 512개를 포함합니다. Frequency/presence rebuild parity는 window 1과 64, batch width 1과 2, f32/f16/bf16 logits를 포함합니다. 별도 regression은 incremental bf16 경로가 계속 f32로 promote되고 모든 ID가 범위 밖일 때의 early return을 보존함을 확인합니다. + +Instrumented complexity test는 reverse-Z 작업을 `8N` 아래로 제한하고 window를 4,096에서 8,192로 두 배 늘릴 때 작업량이 거의 두 배임을 확인합니다. 또한 exact breaker 2,048개의 build와 scan이 additive임을 검증하고, 긴 failure walk를 유발하는 128-pattern shared-trunk comb를 사용하여 구성이 `8P`, scan이 `3N` 아래임을 확인합니다. + +독립 implementation review와 security review에서 남은 finding은 없었습니다. + +## 5. Timing 증거 + +수동 test는 vocabulary 4, breaker 없음, period-3 history와 1,024, 4,096, 16,384 window에서 정확한 변경 전 reference와 새 구현을 모두 실행합니다. Reference iteration 수는 3, 3, 1이고 새 경로는 각 window에서 20회 실행합니다. `black_box`는 lazy MLX array를 평가하지 않고 받으므로 아래 수치는 GPU end-to-end latency가 아니라 host matching 및 MLX graph construction 시간입니다. + +| Window | Reference µs/step | 새 구현 µs/step | Speedup | +|---:|---:|---:|---:| +| 1,024 | 619.751 | 23.607 | 26.25× | +| 4,096 | 4889.370 | 29.997 | 163.00× | +| 16,384 | 77199.293 | 111.433 | 692.79× | + +Window가 16배 늘어나는 동안 reference 비용은 124.57배, 새 경로는 4.72배 증가했으며 16,384-token speedup은 692.79배였습니다. + +Raw command output, 내부 및 외부 driver, environment, host-gate output, build/test log, checksum, binary/source provenance는 `docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/`에 공개합니다. + +## 6. 호환성과 제한 사항 + +Request field, default, public API는 바뀌지 않습니다. Full-history, zero-window, short-history, low-base, breaker, exponent-cap, max-per-token semantics는 reference parity로 유지됩니다. XLA sampler와 windowed repetition penalty는 이 이슈의 범위 밖입니다. + +Automaton은 단계마다 다시 구성하므로 전체 작업은 history 길이뿐 아니라 설정된 pattern token도 포함합니다. Hash-map transition을 사용하므로 명시한 상한은 deterministic worst-case가 아니라 expected bound입니다. Timing test는 quiet GB10 host 한 대의 scaling과 local cost에 대한 증거이며 cross-platform throughput 보장은 아닙니다. + +## 7. 관련 기록 + +- Issue #2259: 구현 요구 사항 +- Issue #2229 및 #2256: security review 배경 +- `docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md`: 날짜가 기록된 timing 결과 +- `docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/`: raw evidence package diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes index 324f58529..abf7c7cba 100644 --- a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/.gitattributes @@ -1,4 +1,5 @@ dry-step-cost.log whitespace=-blank-at-eof engine-tests.log whitespace=-blank-at-eof environment.txt whitespace=-trailing-space +implementation-73700aaf-to-ecb89a3.patch whitespace=-trailing-space sampling-tests.log whitespace=-blank-at-eof diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md index 9bbd179ae..efb178ca8 100644 --- a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/README.md @@ -22,9 +22,23 @@ The raw final logs correspond to source commit `ecb89a308aecc468dd1886d9c9f4982f The native build logs contain existing compiler diagnostics from the pinned dependency build; the Rust clippy command still exited successfully with `-D warnings`. +## Rebase equivalence + +The benchmark was captured before PR #2272 rebased onto main. The measured source is `ecb89a308aecc468dd1886d9c9f4982f556f3c93`; the rebased publication checkout is `327b6bf13a31d0f98e4cce717443d03b4be2cb73` on `3ae626c2a4d67e7628a4050f01cc62b7419296fb`. Both checkouts have `sampling.rs` blob `59fb6b02c6b54913aa549085fafc090e0b516d55`, and their diff across `src/lib/mlxcel-core`, `src/lib/mlx-cpp`, and `Cargo.lock` is empty. `rebase-equivalence.txt` records the exact identities and empty-output hash. The byte-preserved original `environment.txt` is unchanged. + +`implementation-73700aaf-to-ecb89a3.patch` contains the original two implementation commits as a format patch. To reconstruct the measured implementation tree from the public base: + +```bash +cp implementation-73700aaf-to-ecb89a3.patch /tmp/issue-2259-implementation.patch +git switch --detach 73700aafa6a792f07830578c0c26137cea3a038b +git am /tmp/issue-2259-implementation.patch +git rev-parse HEAD^{tree} +# 58b9a04161eaf88048699d0930a69dd59ccdd7b3 +``` + ## Replay -After squash merge, rebuild and run the ignored test from the merge commit on a quiet CUDA host with `gpu-lock run --tag issue-2259-quiet-timing -- cargo test --profile test-fast --features cuda -p mlxcel-core --lib sampling::tests::dry_step_cost_report -- --ignored --nocapture --test-threads=1`. That direct cargo form may compile while holding the lock if the cache is cold. For a controlled measurement, first build the test artifact outside the timing window, then adapt the preserved inner driver to invoke that exact artifact while retaining the same provenance capture and gate. Compare production and test code against the measured source and keep any post-measurement difference explicit. +Rebuild the reconstructed measured tree or a source-equivalent merge checkout, then run the ignored test on a quiet CUDA host with `gpu-lock run --tag issue-2259-quiet-timing -- cargo test --profile test-fast --features cuda -p mlxcel-core --lib sampling::tests::dry_step_cost_report -- --ignored --nocapture --test-threads=1`. That direct cargo form may compile while holding the lock if the cache is cold. For a controlled measurement, first build the test artifact outside the timing window, then adapt the preserved inner driver to invoke that exact artifact while retaining the same provenance capture and gate. Keep the replay checkout and any source difference explicit; the published timing always refers to the original measured artifact. Validate the preserved files from this directory with: diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS index b0f56b915..a10797ceb 100644 --- a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/SHA256SUMS @@ -1,6 +1,6 @@ -e8e91485fcfd340dc8949fca525b0ec0b55eb2aae7f9cb0317ef8e1d3a124b40 .gitattributes +873c69ee621792d5c53c810053c7d2e23997cf2a77a0e4aafc89c28d1480b84c .gitattributes 06b48baeb28c191f550564075a65e7e1f6ee6c9dc594437a63a467319c1ab988 2259-quiet-timing.sh -83265d7d112c40c2e6b73e8906bc32251e3defd7d56608bbd7d6d29286c78233 README.md +dd8766e4d2eb3c4ce310a51cc24445ca04b527e25bcf40d8c6b47d672eccb609 README.md d01b1a6bcaad819dbd8658555cf25cf2709a26f12d64bd34c69987f69d50c9af clippy.log 192b41e9e4aa31324c5a661774ae74437d6ca6e04fc8f83fd67026f8f17b613d core-build.log 9169c635ff03036e4d3046b1e276f4f4c2be21af51cf7a088625bbd470457c9c dry-step-cost.log @@ -8,6 +8,8 @@ c825e252273a90be4ec8b44c7547d41a04eb657a0e5ceae7f2faf52a6abf3d9b engine-tests.l 8e1104d2d3082f2230107434f3305fcfd92f6b353238695485dad9f53386ea47 environment.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 fmt.log 890d1fcf0b4cab5cca9f5fd990ed2cc34f8e5c7a46097ed471b30bd9e15bd3d1 hostgate.log +d28554322892d1103464f426996cede079336c54f5103843b0b6c63e95b7ebd2 implementation-73700aaf-to-ecb89a3.patch ecf915044da3d513f8eff236dd9b75f2abb04c1a2f3738b7cfa089ddec684e04 outer-invocation.txt -20b46b97ebf8315a881818d1e23d6f9e075e9d32910078399baecdf0c69a419a provenance.txt +683e603484a114917f0db1cb6c1dc01a7db4db7fb466fdf96a5f7ba62519ef2e provenance.txt +099ff894e330da020d8f02f8df28625fa5483ac70264e0c96b43617620537b4e rebase-equivalence.txt d241484673791105deec40767622a4a953ae6dfcfdbc94cf4594ea6629bdd421 sampling-tests.log diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/implementation-73700aaf-to-ecb89a3.patch b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/implementation-73700aaf-to-ecb89a3.patch new file mode 100644 index 000000000..78c17b05f --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/implementation-73700aaf-to-ecb89a3.patch @@ -0,0 +1,1118 @@ +From a7e8bb910c1fb5dd762bc54f2b57aa560e660e39 Mon Sep 17 00:00:00 2001 +From: Jeongkyu Shin +Date: Sun, 11 Oct 2026 17:55:55 +0900 +Subject: [PATCH 1/2] update(perf): bound DRY penalty step cost + +Replace quadratic backward matching with a linear reverse-Z pass and scatter DRY and rebuild frequency/presence updates only across touched token IDs. Preserve backend-specific dense dtype behavior and the existing incremental f32 path. + +Add dense-reference parity matrices, breaker-boundary and linear comparison-count checks, bf16 incremental regression coverage, and an ignored timing report. Build and tests have not run yet because the orchestrated CUDA profiling window is still active. + +Refs #2259 +--- + src/lib/mlxcel-core/src/sampling.rs | 572 +++++++++++++++++++++++++++- + 1 file changed, 557 insertions(+), 15 deletions(-) + +diff --git a/src/lib/mlxcel-core/src/sampling.rs b/src/lib/mlxcel-core/src/sampling.rs +index b2612746dad0aef4573b5f14c3dfc88dd9e391e7..2c8527a884d3186e6da81065a6e48ea851046333 100644 +--- a/src/lib/mlxcel-core/src/sampling.rs ++++ b/src/lib/mlxcel-core/src/sampling.rs +@@ -1850,6 +1850,37 @@ pub(crate) fn apply_repetition_penalty_sorted( + /// Apply OpenAI-style frequency and presence penalties to logits. + /// + /// Used by: standard generation, speculative decoding ++fn apply_frequency_presence_sparse( ++ logits: &MlxArray, ++ idx: &[i32], ++ val: &[f32], ++) -> UniquePtr { ++ debug_assert_eq!(idx.len(), val.len()); ++ ++ let shape = ffi::array_shape(logits); ++ let vocab_size = *shape.last().unwrap(); ++ let rows: i32 = shape[..shape.len() - 1].iter().product(); ++ if idx.is_empty() { ++ return ffi::copy(logits); ++ } ++ ++ let k = idx.len() as i32; ++ let mut row_idx = Vec::with_capacity(rows as usize * idx.len()); ++ let mut row_val = Vec::with_capacity(rows as usize * val.len()); ++ for _ in 0..rows { ++ row_idx.extend_from_slice(idx); ++ row_val.extend_from_slice(val); ++ } ++ ++ let indices = ffi::from_slice_i32(&row_idx, &[rows, k]); ++ let values = ffi::from_slice_f32(&row_val, &[rows, k]); ++ let flat = ffi::reshape(logits, &[rows, vocab_size]); ++ let selected = ffi::take_along_axis(&flat, &indices, -1); ++ let penalized = ffi::subtract(&selected, &values); ++ let scattered = ffi::put_along_axis(&flat, &indices, &penalized, -1); ++ ffi::reshape(&scattered, &shape) ++} ++ + pub(crate) fn apply_frequency_presence_penalty( + logits: &MlxArray, + token_history: &[i32], +@@ -1865,6 +1896,43 @@ pub(crate) fn apply_frequency_presence_penalty( + return ffi::copy(logits); + } + ++ let shape = ffi::array_shape(logits); ++ let vocab_size = *shape.last().unwrap() as usize; ++ let mut idx = Vec::with_capacity(token_counts.len()); ++ let mut val = Vec::with_capacity(token_counts.len()); ++ for (&token_id, &count) in &token_counts { ++ if token_id >= 0 && (token_id as usize) < vocab_size { ++ idx.push(token_id); ++ val.push(frequency_penalty * count as f32 + presence_penalty); ++ } ++ } ++ ++ // Reproduce the dense reference's mixed-dtype result before scattering. ++ // The CUDA MLX patch preserves bf16 for bf16/f32 binary arithmetic, while ++ // unpatched backends and f16 promote to f32. Subtracting a scalar +0.0 ++ // follows that platform rule without allocating a vocabulary-sized host ++ // buffer and leaves every untouched value unchanged. ++ let zero = ffi::full_f32(&[1], 0.0, dtype::FLOAT32); ++ let promoted = ffi::subtract(logits, &zero); ++ apply_frequency_presence_sparse(&promoted, &idx, &val) ++} ++ ++#[cfg(test)] ++fn apply_frequency_presence_penalty_reference( ++ logits: &MlxArray, ++ token_history: &[i32], ++ frequency_penalty: f32, ++ presence_penalty: f32, ++) -> UniquePtr { ++ let mut token_counts: HashMap = HashMap::new(); ++ for &tok in token_history { ++ *token_counts.entry(tok).or_insert(0) += 1; ++ } ++ ++ if token_counts.is_empty() { ++ return ffi::copy(logits); ++ } ++ + let shape = ffi::array_shape(logits); + let vocab_size = *shape.last().unwrap() as usize; + +@@ -1880,6 +1948,25 @@ pub(crate) fn apply_frequency_presence_penalty( + ffi::subtract(logits, &penalty_broadcast) + } + ++#[cfg(test)] ++fn apply_frequency_presence_incremental_reference( ++ logits: &MlxArray, ++ idx: &[i32], ++ val: &[f32], ++) -> UniquePtr { ++ if idx.is_empty() { ++ ffi::copy(logits) ++ } else { ++ let promoted = ffi::astype(logits, dtype::FLOAT32); ++ let k = idx.len() as i32; ++ let indices = ffi::from_slice_i32(idx, &[1, k]); ++ let selected = ffi::take_along_axis(&promoted, &indices, -1); ++ let values = ffi::from_slice_f32(val, &[1, k]); ++ let penalized = ffi::subtract(&selected, &values); ++ ffi::put_along_axis(&promoted, &indices, &penalized, -1) ++ } ++} ++ + /// Whether DRY matching breaks at `window[pos]` (#1485). + /// + /// Two breaker sources compose: +@@ -1915,16 +2002,204 @@ fn dry_breaks_at(config: &SamplingConfig, window: &[i32], pos: usize) -> bool { + }) + } + ++#[cfg(test)] ++thread_local! { ++ static DRY_STEP_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; ++} ++ ++#[cfg(test)] ++fn record_dry_step_cost(cost: usize) { ++ DRY_STEP_COST.with(|counter| counter.set(cost)); ++} ++ ++#[cfg(test)] ++fn last_dry_step_cost() -> usize { ++ DRY_STEP_COST.with(std::cell::Cell::get) ++} ++ ++/// Compute DRY penalties from a selected history window without MLX calls. ++/// ++/// Reversing the window turns every suffix match against the current suffix ++/// into a prefix match, so one Z-function pass provides all match lengths. ++/// A second forward pass records the nearest breaker strictly before each ++/// candidate position. Both passes are linear in the window length. ++fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { ++ #[cfg(test)] ++ let mut step_cost = 0usize; ++ #[cfg(test)] ++ record_dry_step_cost(0); ++ ++ let window_len = window.len(); ++ if window_len < 2 || config.dry_base < 1.0 { ++ return Vec::new(); ++ } ++ ++ let reversed: Vec = window.iter().rev().copied().collect(); ++ let mut z = vec![0usize; window_len]; ++ z[0] = window_len; ++ let mut left = 0usize; ++ let mut right = 0usize; ++ for i in 1..window_len { ++ if i <= right { ++ z[i] = (right - i + 1).min(z[i - left]); ++ } ++ while i + z[i] < window_len { ++ #[cfg(test)] ++ { ++ step_cost += 1; ++ } ++ if reversed[z[i]] != reversed[i + z[i]] { ++ break; ++ } ++ z[i] += 1; ++ } ++ if z[i] > 0 { ++ let candidate_right = i + z[i] - 1; ++ if candidate_right > right { ++ left = i; ++ right = candidate_right; ++ } ++ } ++ } ++ ++ let no_breaker = usize::MAX; ++ let mut breaker_before = vec![no_breaker; window_len]; ++ let mut most_recent_breaker = no_breaker; ++ for (pos, slot) in breaker_before.iter_mut().enumerate() { ++ *slot = most_recent_breaker; ++ #[cfg(test)] ++ { ++ step_cost += 1; ++ } ++ if dry_breaks_at(config, window, pos) { ++ most_recent_breaker = pos; ++ } ++ } ++ ++ let last_pos = window_len - 1; ++ let last_token = window[last_pos]; ++ let mut penalties: HashMap = HashMap::new(); ++ for (pos, &token) in window[..last_pos].iter().enumerate() { ++ #[cfg(test)] ++ { ++ step_cost += 1; ++ } ++ if token != last_token { ++ continue; ++ } ++ ++ let suffix_match_len = z[last_pos - pos]; ++ let breaker_limited_len = if breaker_before[pos] == no_breaker { ++ pos + 1 ++ } else { ++ pos - breaker_before[pos] ++ }; ++ let match_len = suffix_match_len.min(breaker_limited_len); ++ ++ if match_len >= config.dry_allowed_length { ++ let next_token = window[pos + 1]; ++ let mut exponent = (match_len - config.dry_allowed_length) as f32; ++ if config.dry_base > 1.0 { ++ let max_exponent = 88.722_84_f32 / config.dry_base.ln(); ++ if exponent > max_exponent { ++ exponent = max_exponent; ++ } ++ } ++ let penalty = config.dry_multiplier * config.dry_base.powf(exponent); ++ let entry = penalties.entry(next_token).or_insert(0.0); ++ if penalty > *entry { ++ *entry = penalty; ++ } ++ } ++ } ++ ++ #[cfg(test)] ++ record_dry_step_cost(step_cost); ++ penalties.into_iter().collect() ++} ++ ++fn apply_dry_penalties_sparse(logits: &MlxArray, penalties: &[(i32, f32)]) -> UniquePtr { ++ let shape = ffi::array_shape(logits); ++ let vocab_size = *shape.last().unwrap(); ++ let rows: i32 = shape[..shape.len() - 1].iter().product(); ++ let mut idx = Vec::with_capacity(penalties.len()); ++ let mut val = Vec::with_capacity(penalties.len()); ++ for &(token_id, penalty) in penalties { ++ if token_id >= 0 && token_id < vocab_size { ++ idx.push(token_id); ++ val.push(-penalty); ++ } ++ } ++ ++ // Match the old dense `add(logits, f32_penalty_buffer)` result dtype on ++ // every backend. In particular, the CUDA MLX patch keeps bf16 here rather ++ // than promoting it to f32. A scalar +0.0 obtains the same binary-op dtype ++ // and untouched values without a vocabulary-sized host allocation. ++ let zero = ffi::full_f32(&[1], 0.0, dtype::FLOAT32); ++ let promoted = ffi::add(logits, &zero); ++ if idx.is_empty() { ++ return promoted; ++ } ++ ++ let k = idx.len() as i32; ++ let mut row_idx = Vec::with_capacity(rows as usize * idx.len()); ++ let mut row_val = Vec::with_capacity(rows as usize * val.len()); ++ for _ in 0..rows { ++ row_idx.extend_from_slice(&idx); ++ row_val.extend_from_slice(&val); ++ } ++ ++ let indices = ffi::from_slice_i32(&row_idx, &[rows, k]); ++ let values = ffi::from_slice_f32(&row_val, &[rows, k]); ++ let flat = ffi::reshape(&promoted, &[rows, vocab_size]); ++ let selected = ffi::take_along_axis(&flat, &indices, -1); ++ let penalized = ffi::add(&selected, &values); ++ let scattered = ffi::put_along_axis(&flat, &indices, &penalized, -1); ++ ffi::reshape(&scattered, &shape) ++} ++ + /// Apply DRY (Don't Repeat Yourself) penalty to logits. + /// +-/// This runs on CPU as sequential pattern matching, which keeps the matching +-/// invariant explicit and mirrors the upstream llama.cpp style algorithm. ++/// The host matching pass is linear in the selected history window and the ++/// device update touches only token ids that receive a penalty. + /// + /// Used by: standard generation, speculative decoding + pub(crate) fn apply_dry_penalty( + logits: &MlxArray, + token_history: &[i32], + config: &SamplingConfig, ++) -> UniquePtr { ++ let history_len = token_history.len(); ++ if history_len < 2 || config.dry_base < 1.0 { ++ return ffi::copy(logits); ++ } ++ ++ let window = if config.dry_penalty_last_n == crate::generate::DRY_FULL_HISTORY { ++ token_history ++ } else if config.dry_penalty_last_n == 0 { ++ &token_history[history_len..] ++ } else { ++ let start = history_len.saturating_sub(config.dry_penalty_last_n); ++ &token_history[start..] ++ }; ++ ++ if window.len() < 2 { ++ return ffi::copy(logits); ++ } ++ ++ let penalties = dry_penalty_map(window, config); ++ if penalties.is_empty() { ++ return ffi::copy(logits); ++ } ++ apply_dry_penalties_sparse(logits, &penalties) ++} ++ ++/// Test-only copy of the pre-#2259 DRY implementation. ++#[cfg(test)] ++fn apply_dry_penalty_reference( ++ logits: &MlxArray, ++ token_history: &[i32], ++ config: &SamplingConfig, + ) -> UniquePtr { + let history_len = token_history.len(); + if history_len < 2 { +@@ -2232,12 +2507,12 @@ impl SamplerState { + /// Frequency/presence penalty using the reusable sparse scratch buffers + /// (touched tokens only); never allocates a full-vocabulary vector. + /// +- /// Byte-identical to [`apply_frequency_presence_penalty`]: the rebuild path +- /// computes `subtract(logits, penalty_f32)`, which promotes the whole array +- /// to f32 (f16/bf16 -> f32 is lossless). This path promotes first, then +- /// applies `logits[id] - penalty[id]` to exactly the touched ids via +- /// take/put. Untouched ids keep their promoted value, which equals the +- /// rebuild path's `logits[id] - 0.0` for every finite logit. ++ /// Preserves the pre-#2259 incremental semantics: promote the logits to ++ /// f32 first, then apply `logits[id] - penalty[id]` to exactly the touched ++ /// ids. This intentionally differs from the dense rebuild path on CUDA ++ /// bf16, where patched MLX keeps mixed bf16/f32 subtraction in bf16; the ++ /// two paths already had that dtype/rounding distinction before this ++ /// refactor. An all-out-of-range count map still returns an unchanged copy. + fn apply_frequency_presence( + &mut self, + logits: &MlxArray, +@@ -2266,16 +2541,15 @@ impl SamplerState { + } + + let result = if idx.is_empty() { +- // No in-range tokens: matches the rebuild path's empty early return. ++ // Preserve the pre-#2259 incremental path: an all-out-of-range ++ // count map returns an unchanged copy without dtype promotion. + ffi::copy(logits) + } else { ++ // The incremental path explicitly promoted before its sparse ++ // scatter, including on CUDA bf16 where the dense rebuild path ++ // follows MLX's patched bf16/f32 binary-op rule instead. + let promoted = ffi::astype(logits, dtype::FLOAT32); +- let k = idx.len() as i32; +- let indices = ffi::from_slice_i32(&idx, &[1, k]); +- let selected = ffi::take_along_axis(&promoted, &indices, -1); +- let values = ffi::from_slice_f32(&val, &[1, k]); +- let penalized = ffi::subtract(&selected, &values); +- ffi::put_along_axis(&promoted, &indices, &penalized, -1) ++ apply_frequency_presence_sparse(&promoted, &idx, &val) + }; + + self.sparse_idx = idx; +@@ -2938,6 +3212,274 @@ mod tests { + assert_eq!(logit_at(&result, 2), -1.0); + } + ++ fn assert_penalty_logits_equal(a: &MlxArray, b: &MlxArray, context: &str) { ++ assert_eq!(ffi::array_shape(a), ffi::array_shape(b), "{context}: shape"); ++ assert_eq!(ffi::array_dtype(a), ffi::array_dtype(b), "{context}: dtype"); ++ let a = ffi::astype(a, dtype::FLOAT32); ++ let b = ffi::astype(b, dtype::FLOAT32); ++ let a = array_to_vec_f32(&a); ++ let b = array_to_vec_f32(&b); ++ assert_eq!(a.len(), b.len(), "{context}: length"); ++ for (index, (&actual, &expected)) in a.iter().zip(&b).enumerate() { ++ assert_eq!( ++ actual, expected, ++ "{context}: element {index}: {actual} != {expected}" ++ ); ++ } ++ } ++ ++ fn lcg_token_history(seed: u64, len: usize, vocab_size: usize) -> Vec { ++ let mut state = seed; ++ let mut history = Vec::with_capacity(len); ++ for _ in 0..len { ++ state = state ++ .wrapping_mul(6364136223846793005) ++ .wrapping_add(1442695040888963407); ++ history.push(((state >> 32) as usize % vocab_size) as i32); ++ } ++ history ++ } ++ ++ #[test] ++ fn dry_reverse_z_matches_reference_lcg_cases() { ++ const CASES: usize = 512; ++ const LENGTHS: [usize; 10] = [0, 1, 2, 3, 7, 17, 64, 257, 1024, 2048]; ++ const WINDOWS: [usize; 6] = [crate::generate::DRY_FULL_HISTORY, 0, 1, 2, 17, 64]; ++ ++ for case in 0..CASES { ++ let vocab_size = [8usize, 64][(case / 19) % 2]; ++ let history_len = LENGTHS[(case / 23) % LENGTHS.len()]; ++ let periodic = (case / 7) % 2 == 0; ++ let history = if periodic { ++ (0..history_len) ++ .map(|index| ((index + case) % 3) as i32) ++ .collect() ++ } else { ++ lcg_token_history( ++ 0x2259_0000_u64.wrapping_add(case as u64), ++ history_len, ++ vocab_size, ++ ) ++ }; ++ let batch = [1usize, 2][(case / 3) % 2]; ++ let raw_logits = lcg_logits( ++ 0xD4A0_0000_u64.wrapping_add(case as u64), ++ batch * vocab_size, ++ ); ++ let f32_logits = ffi::from_slice_f32(&raw_logits, &[batch as i32, vocab_size as i32]); ++ let input_dtype = [dtype::FLOAT32, dtype::BFLOAT16][(case / 5) % 2]; ++ let logits = if input_dtype == dtype::FLOAT32 { ++ f32_logits ++ } else { ++ ffi::astype(&f32_logits, input_dtype) ++ }; ++ ++ let mut config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: [1.0, 1.75][(case / 13) % 2], ++ dry_allowed_length: 1 + (case / 11) % 4, ++ dry_penalty_last_n: WINDOWS[(case / 29) % WINDOWS.len()], ++ ..Default::default() ++ }; ++ let breaker = ((case * 5 + 1) % vocab_size) as i32; ++ match (case / 17) % 4 { ++ 1 => config.dry_sequence_breakers = vec![breaker], ++ 2 => { ++ config.dry_breaker_heads = ++ std::sync::Arc::new(HashMap::from([(breaker, vec![Vec::new()])])); ++ } ++ 3 => { ++ let tail = (breaker + 1) % vocab_size as i32; ++ config.dry_breaker_heads = ++ std::sync::Arc::new(HashMap::from([(breaker, vec![vec![tail]])])); ++ } ++ _ => {} ++ } ++ ++ let reference = apply_dry_penalty_reference(&logits, &history, &config); ++ let linear = apply_dry_penalty(&logits, &history, &config); ++ assert_penalty_logits_equal( ++ &linear, ++ &reference, ++ &format!( ++ "case={case} vocab={vocab_size} history={history_len} periodic={periodic} batch={batch} dtype={input_dtype} window={} allowed={} base={}", ++ config.dry_penalty_last_n, config.dry_allowed_length, config.dry_base ++ ), ++ ); ++ } ++ } ++ ++ #[test] ++ fn dry_reverse_z_preserves_breaker_boundaries() { ++ let mut config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: 1.75, ++ dry_allowed_length: 2, ++ dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, ++ dry_sequence_breakers: vec![2], ++ ..Default::default() ++ }; ++ let history = [1, 2, 9, 1, 2]; ++ let map = dry_penalty_map(&history, &config); ++ assert!( ++ map.iter().any(|&(token, _)| token == 9), ++ "a breaker at the occurrence itself must not cut the preceding match" ++ ); ++ ++ config.dry_sequence_breakers.clear(); ++ config.dry_breaker_heads = std::sync::Arc::new(HashMap::from([(2, vec![vec![7, 8]])])); ++ let reference = apply_dry_penalty_reference( ++ &ffi::from_slice_f32(&[0.0; 12], &[1, 12]), ++ &history, ++ &config, ++ ); ++ let linear = apply_dry_penalty( ++ &ffi::from_slice_f32(&[0.0; 12], &[1, 12]), ++ &history, ++ &config, ++ ); ++ assert_penalty_logits_equal( ++ &linear, ++ &reference, ++ "a breaker tail running past the window end must not break", ++ ); ++ } ++ ++ #[test] ++ fn dry_reverse_z_step_cost_is_linear() { ++ let config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: 1.75, ++ dry_allowed_length: 2, ++ dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, ++ ..Default::default() ++ }; ++ let history_4096: Vec = (0..4096).map(|index| index % 3).collect(); ++ let _ = dry_penalty_map(&history_4096, &config); ++ let cost_4096 = last_dry_step_cost(); ++ assert!( ++ cost_4096 <= 8 * 4096, ++ "period-3 window 4096 cost {cost_4096}, expected <= {}", ++ 8 * 4096 ++ ); ++ ++ let history_8192: Vec = (0..8192).map(|index| index % 3).collect(); ++ let _ = dry_penalty_map(&history_8192, &config); ++ let cost_8192 = last_dry_step_cost(); ++ assert!( ++ cost_8192 <= 2 * cost_4096 + 16, ++ "doubling the window changed cost from {cost_4096} to {cost_8192}" ++ ); ++ } ++ ++ #[test] ++ fn frequency_presence_sparse_matches_vocab_reference() { ++ let vocab_size = 64usize; ++ let history = lcg_token_history(0xF2E9_2259, 96, vocab_size); ++ for penalty_last_n in [1, 64] { ++ let window = penalty_window(&history, penalty_last_n); ++ for batch in [1usize, 2] { ++ let raw = lcg_logits(0xF2E9_0000 + batch as u64, batch * vocab_size); ++ let f32_logits = ffi::from_slice_f32(&raw, &[batch as i32, vocab_size as i32]); ++ for input_dtype in [dtype::FLOAT32, dtype::FLOAT16, dtype::BFLOAT16] { ++ let logits = if input_dtype == dtype::FLOAT32 { ++ ffi::copy(&f32_logits) ++ } else { ++ ffi::astype(&f32_logits, input_dtype) ++ }; ++ let reference = ++ apply_frequency_presence_penalty_reference(&logits, window, 0.625, 0.375); ++ let sparse = apply_frequency_presence_penalty(&logits, window, 0.625, 0.375); ++ assert_penalty_logits_equal( ++ &sparse, ++ &reference, ++ &format!( ++ "penalty_last_n={penalty_last_n} batch={batch} dtype={input_dtype}" ++ ), ++ ); ++ } ++ } ++ } ++ } ++ ++ #[test] ++ fn sampler_state_frequency_presence_preserves_bf16_reference() { ++ let (frequency_penalty, presence_penalty) = (0.625, 0.375); ++ let config = SamplingConfig { ++ frequency_penalty, ++ presence_penalty, ++ ..Default::default() ++ }; ++ let raw = lcg_logits(0x1AC0_2259, 64); ++ let bf16_logits = ffi::astype(&ffi::from_slice_f32(&raw, &[1, 64]), dtype::BFLOAT16); ++ ++ let mut state = SamplerState::for_config(&config); ++ state.sync(&[1, 1, 2, 5, 8, 13, 13]); ++ let mut idx = Vec::new(); ++ let mut val = Vec::new(); ++ for (&token_id, &count) in &state.counts { ++ idx.push(token_id); ++ val.push(frequency_penalty * count as f32 + presence_penalty); ++ } ++ let reference = apply_frequency_presence_incremental_reference(&bf16_logits, &idx, &val); ++ let sparse = ++ state.apply_frequency_presence(&bf16_logits, frequency_penalty, presence_penalty); ++ assert_penalty_logits_equal( ++ &sparse, ++ &reference, ++ "incremental bf16 path must retain its explicit f32 promotion", ++ ); ++ assert_eq!(ffi::array_dtype(&sparse), dtype::FLOAT32); ++ ++ let mut out_of_range = SamplerState::for_config(&config); ++ out_of_range.sync(&[-3, 64, 100]); ++ let untouched = out_of_range.apply_frequency_presence( ++ &bf16_logits, ++ frequency_penalty, ++ presence_penalty, ++ ); ++ assert_eq!(ffi::array_dtype(&untouched), dtype::BFLOAT16); ++ assert_penalty_logits_equal( ++ &untouched, ++ &bf16_logits, ++ "all-out-of-range incremental counts must keep the copy early return", ++ ); ++ } ++ ++ #[test] ++ #[ignore = "manual timing evidence for issue #2259"] ++ fn dry_step_cost_report() { ++ let config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: 1.75, ++ dry_allowed_length: 2, ++ dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, ++ ..Default::default() ++ }; ++ let logits = ffi::from_slice_f32(&[0.0; 4], &[1, 4]); ++ for window_len in [1024usize, 4096, 16384] { ++ let history: Vec = (0..window_len).map(|index| (index % 3) as i32).collect(); ++ let reference_steps = if window_len == 16384 { 1 } else { 3 }; ++ let reference_start = std::time::Instant::now(); ++ for _ in 0..reference_steps { ++ std::hint::black_box(apply_dry_penalty_reference(&logits, &history, &config)); ++ } ++ let reference_us = ++ reference_start.elapsed().as_secs_f64() * 1_000_000.0 / reference_steps as f64; ++ ++ let linear_steps = 20; ++ let linear_start = std::time::Instant::now(); ++ for _ in 0..linear_steps { ++ std::hint::black_box(apply_dry_penalty(&logits, &history, &config)); ++ } ++ let linear_us = ++ linear_start.elapsed().as_secs_f64() * 1_000_000.0 / linear_steps as f64; ++ println!( ++ "dry_step_cost window={window_len} reference_us_per_step={reference_us:.3} linear_us_per_step={linear_us:.3}" ++ ); ++ } ++ } ++ + #[test] + fn sample_token_optimized_respects_greedy_argmax_path() { + let logits = ffi::from_slice_f32(&[0.1, 0.9, 1.2], &[1, 1, 3]); + +From ecb89a308aecc468dd1886d9c9f4982f556f3c93 Mon Sep 17 00:00:00 2001 +From: Jeongkyu Shin +Date: Sun, 11 Oct 2026 18:37:50 +0900 +Subject: [PATCH 2/2] fix(perf): bound DRY breaker matching cost + +Build a reversed-token matcher once per DRY step so exact IDs and multi-token breakers cost expected O(P + N) instead of multiplying breaker configuration size by the history window. + +Preserve scalar breaker semantics in direct per-position parity tests and cover large exact sets plus shared-tail failure chains with separate build and scan counters. + +Validation: cargo fmt --all. Build and tests are pending the root validation gate. + +Refs #2259 +--- + src/lib/mlxcel-core/src/sampling.rs | 386 +++++++++++++++++++++++++++- + 1 file changed, 383 insertions(+), 3 deletions(-) + +diff --git a/src/lib/mlxcel-core/src/sampling.rs b/src/lib/mlxcel-core/src/sampling.rs +index 2c8527a884d3186e6da81065a6e48ea851046333..59fb6b02c6b54913aa549085fafc090e0b516d55 100644 +--- a/src/lib/mlxcel-core/src/sampling.rs ++++ b/src/lib/mlxcel-core/src/sampling.rs +@@ -1980,6 +1980,7 @@ fn apply_frequency_presence_incremental_reference( + /// only form). A head with a non-empty tail breaks only when the window + /// tokens after `pos` spell the tail out in full, which is how a + /// multi-character breaker split across token boundaries is recognized. ++#[cfg(test)] + fn dry_breaks_at(config: &SamplingConfig, window: &[i32], pos: usize) -> bool { + if config.dry_sequence_breakers.contains(&window[pos]) { + return true; +@@ -2002,9 +2003,184 @@ fn dry_breaks_at(config: &SamplingConfig, window: &[i32], pos: usize) -> bool { + }) + } + ++#[derive(Default)] ++struct DryBreakerNode { ++ transitions: HashMap, ++ failure: usize, ++ terminal: bool, ++} ++ ++struct DryBreakerMatches { ++ positions: Vec, ++ #[cfg(test)] ++ build_steps: usize, ++ #[cfg(test)] ++ scan_steps: usize, ++} ++ ++/// Find every DRY breaker start in `reversed_window` in additive ++/// `O(window + breaker-pattern tokens)` expected time. ++/// ++/// Exact IDs are singleton patterns. A forward breaker `[head] + tail` is ++/// inserted as `reverse(tail) + [head]`, so an Aho-Corasick terminal reached ++/// at reversed index `r` always maps to original start `n - 1 - r`, regardless ++/// of pattern length. Failure-terminal propagation therefore marks a start ++/// once without enumerating every matching suffix pattern. ++fn dry_breaker_matches(reversed_window: &[i32], config: &SamplingConfig) -> DryBreakerMatches { ++ #[cfg(test)] ++ let mut build_steps = 0usize; ++ #[cfg(test)] ++ let mut scan_steps = 0usize; ++ ++ if config.dry_sequence_breakers.is_empty() && config.dry_breaker_heads.is_empty() { ++ return DryBreakerMatches { ++ positions: vec![false; reversed_window.len()], ++ #[cfg(test)] ++ build_steps, ++ #[cfg(test)] ++ scan_steps, ++ }; ++ } ++ ++ let mut nodes = vec![DryBreakerNode::default()]; ++ ++ for &token in &config.dry_sequence_breakers { ++ #[cfg(test)] ++ { ++ build_steps += 1; ++ } ++ let next = nodes[0].transitions.get(&token).copied(); ++ let state = match next { ++ Some(state) => state, ++ None => { ++ let state = nodes.len(); ++ nodes.push(DryBreakerNode::default()); ++ nodes[0].transitions.insert(token, state); ++ #[cfg(test)] ++ { ++ build_steps += 1; ++ } ++ state ++ } ++ }; ++ nodes[state].terminal = true; ++ } ++ ++ for (&head, tails) in config.dry_breaker_heads.iter() { ++ for tail in tails { ++ let mut state = 0usize; ++ for token in tail.iter().rev().copied().chain(std::iter::once(head)) { ++ #[cfg(test)] ++ { ++ build_steps += 1; ++ } ++ let next = nodes[state].transitions.get(&token).copied(); ++ state = match next { ++ Some(next) => next, ++ None => { ++ let next = nodes.len(); ++ nodes.push(DryBreakerNode::default()); ++ nodes[state].transitions.insert(token, next); ++ #[cfg(test)] ++ { ++ build_steps += 1; ++ } ++ next ++ } ++ }; ++ } ++ nodes[state].terminal = true; ++ } ++ } ++ ++ // Breadth-first failure-link construction. Root children fail to root; ++ // every deeper node inherits the terminal bit of its failure target. Across ++ // trie edges, failure hops telescope over failure depth, so construction is ++ // linear in the number of supplied pattern tokens in expected HashMap time. ++ let mut queue: Vec = nodes[0].transitions.values().copied().collect(); ++ #[cfg(test)] ++ { ++ build_steps += queue.len(); ++ } ++ let mut cursor = 0usize; ++ while cursor < queue.len() { ++ let state = queue[cursor]; ++ cursor += 1; ++ let transitions: Vec<(i32, usize)> = nodes[state] ++ .transitions ++ .iter() ++ .map(|(&token, &next)| (token, next)) ++ .collect(); ++ #[cfg(test)] ++ { ++ build_steps += transitions.len(); ++ } ++ for (token, next) in transitions { ++ let mut fallback = nodes[state].failure; ++ let failure = loop { ++ #[cfg(test)] ++ { ++ build_steps += 1; ++ } ++ if let Some(&target) = nodes[fallback].transitions.get(&token) { ++ break target; ++ } ++ if fallback == 0 { ++ break 0; ++ } ++ fallback = nodes[fallback].failure; ++ #[cfg(test)] ++ { ++ build_steps += 1; ++ } ++ }; ++ nodes[next].failure = failure; ++ let failure_terminal = nodes[failure].terminal; ++ nodes[next].terminal |= failure_terminal; ++ queue.push(next); ++ } ++ } ++ ++ let mut positions = vec![false; reversed_window.len()]; ++ let mut state = 0usize; ++ for (reversed_pos, &token) in reversed_window.iter().enumerate() { ++ loop { ++ #[cfg(test)] ++ { ++ scan_steps += 1; ++ } ++ if let Some(&next) = nodes[state].transitions.get(&token) { ++ state = next; ++ break; ++ } ++ if state == 0 { ++ break; ++ } ++ state = nodes[state].failure; ++ #[cfg(test)] ++ { ++ scan_steps += 1; ++ } ++ } ++ if nodes[state].terminal { ++ positions[reversed_window.len() - 1 - reversed_pos] = true; ++ } ++ } ++ ++ DryBreakerMatches { ++ positions, ++ #[cfg(test)] ++ build_steps, ++ #[cfg(test)] ++ scan_steps, ++ } ++} ++ + #[cfg(test)] + thread_local! { + static DRY_STEP_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; ++ static DRY_BREAKER_BUILD_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; ++ static DRY_BREAKER_SCAN_COST: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + #[cfg(test)] +@@ -2017,17 +2193,34 @@ fn last_dry_step_cost() -> usize { + DRY_STEP_COST.with(std::cell::Cell::get) + } + ++#[cfg(test)] ++fn record_dry_breaker_cost(build: usize, scan: usize) { ++ DRY_BREAKER_BUILD_COST.with(|counter| counter.set(build)); ++ DRY_BREAKER_SCAN_COST.with(|counter| counter.set(scan)); ++} ++ ++#[cfg(test)] ++fn last_dry_breaker_cost() -> (usize, usize) { ++ let build = DRY_BREAKER_BUILD_COST.with(std::cell::Cell::get); ++ let scan = DRY_BREAKER_SCAN_COST.with(std::cell::Cell::get); ++ (build, scan) ++} ++ + /// Compute DRY penalties from a selected history window without MLX calls. + /// + /// Reversing the window turns every suffix match against the current suffix + /// into a prefix match, so one Z-function pass provides all match lengths. +-/// A second forward pass records the nearest breaker strictly before each +-/// candidate position. Both passes are linear in the window length. ++/// A reversed-token automaton marks breaker starts before one forward pass ++/// records the nearest breaker strictly before each candidate position. The ++/// per-step work is additive in the window and configured breaker-pattern ++/// tokens; it never multiplies the two request-controlled sizes. + fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { + #[cfg(test)] + let mut step_cost = 0usize; + #[cfg(test)] + record_dry_step_cost(0); ++ #[cfg(test)] ++ record_dry_breaker_cost(0, 0); + + let window_len = window.len(); + if window_len < 2 || config.dry_base < 1.0 { +@@ -2062,6 +2255,13 @@ fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { + } + } + ++ let breaker_matches = dry_breaker_matches(&reversed, config); ++ #[cfg(test)] ++ { ++ step_cost += breaker_matches.scan_steps; ++ record_dry_breaker_cost(breaker_matches.build_steps, breaker_matches.scan_steps); ++ } ++ + let no_breaker = usize::MAX; + let mut breaker_before = vec![no_breaker; window_len]; + let mut most_recent_breaker = no_breaker; +@@ -2071,7 +2271,7 @@ fn dry_penalty_map(window: &[i32], config: &SamplingConfig) -> Vec<(i32, f32)> { + { + step_cost += 1; + } +- if dry_breaks_at(config, window, pos) { ++ if breaker_matches.positions[pos] { + most_recent_breaker = pos; + } + } +@@ -3296,6 +3496,23 @@ mod tests { + _ => {} + } + ++ let window = if config.dry_penalty_last_n == crate::generate::DRY_FULL_HISTORY { ++ history.as_slice() ++ } else if config.dry_penalty_last_n == 0 { ++ &history[history.len()..] ++ } else { ++ let start = history.len().saturating_sub(config.dry_penalty_last_n); ++ &history[start..] ++ }; ++ let breaker_matches = breaker_positions(window, &config); ++ for (pos, matched) in breaker_matches.iter().copied().enumerate() { ++ assert_eq!( ++ matched, ++ dry_breaks_at(&config, window, pos), ++ "breaker mismatch at case={case} pos={pos} window={window:?}" ++ ); ++ } ++ + let reference = apply_dry_penalty_reference(&logits, &history, &config); + let linear = apply_dry_penalty(&logits, &history, &config); + assert_penalty_logits_equal( +@@ -3345,6 +3562,169 @@ mod tests { + ); + } + ++ fn breaker_positions(window: &[i32], config: &SamplingConfig) -> Vec { ++ let reversed: Vec = window.iter().rev().copied().collect(); ++ dry_breaker_matches(&reversed, config).positions ++ } ++ ++ #[test] ++ fn dry_breaker_automaton_preserves_overlap_and_boundary_semantics() { ++ let config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: 1.75, ++ dry_allowed_length: 2, ++ dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, ++ dry_sequence_breakers: vec![-7, 2, 2], ++ dry_breaker_heads: std::sync::Arc::new(HashMap::from([ ++ (2, vec![Vec::new(), Vec::new(), vec![3, 4]]), ++ (4, vec![vec![5, 6, 8]]), ++ (5, vec![vec![6], vec![6, 7], vec![7]]), ++ (9, vec![vec![10, 11]]), ++ ])), ++ ..Default::default() ++ }; ++ ++ let window = [-7, 2, 3, 4, 5, 6, 7, 5, 7, 9, 10, 11]; ++ let positions = breaker_positions(&window, &config); ++ for (pos, matched) in positions.iter().copied().enumerate() { ++ assert_eq!( ++ matched, ++ dry_breaks_at(&config, &window, pos), ++ "overlap breaker mismatch at position {pos}" ++ ); ++ } ++ for expected in [0usize, 1, 4, 7, 9] { ++ assert!(positions[expected], "missing breaker at {expected}"); ++ } ++ assert_eq!( ++ positions.iter().filter(|&&is_breaker| is_breaker).count(), ++ 5, ++ "duplicates and overlapping tails must mark each start once" ++ ); ++ ++ assert_eq!( ++ breaker_positions(&[9, 10, 11], &config), ++ vec![true, false, false], ++ "a tail ending exactly at the window end must match" ++ ); ++ assert_eq!( ++ breaker_positions(&[9, 10], &config), ++ vec![false, false], ++ "a tail running past the window end must not match" ++ ); ++ assert_eq!( ++ breaker_positions(&[5, 6, 8], &config), ++ vec![true, false, false], ++ "a terminal failure suffix must mark the same original start" ++ ); ++ ++ let logits = ffi::from_slice_f32(&[0.0; 16], &[1, 16]); ++ for history in [ ++ vec![9, 10, 11, 4, 9, 10, 11], ++ vec![9, 10, 4, 9, 10], ++ vec![1, -7, 2, 4, 1, -7, 2], ++ vec![5, 6, 8, 4, 5, 6, 8], ++ ] { ++ let reference = apply_dry_penalty_reference(&logits, &history, &config); ++ let automaton = apply_dry_penalty(&logits, &history, &config); ++ assert_penalty_logits_equal( ++ &automaton, ++ &reference, ++ &format!("breaker automaton history {history:?}"), ++ ); ++ } ++ } ++ ++ #[test] ++ fn dry_breaker_automaton_large_exact_list_is_additive() { ++ const WINDOW: usize = 4096; ++ const EXACT: usize = 2048; ++ let mut exact: Vec = (10_000..10_000 + EXACT as i32).collect(); ++ exact.extend([-7, 0, 1, 2]); ++ let config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: 1.75, ++ dry_allowed_length: 2, ++ dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, ++ dry_sequence_breakers: exact, ++ ..Default::default() ++ }; ++ let mut history: Vec = (0..WINDOW - 1).map(|index| (index % 3) as i32).collect(); ++ history.push(63); ++ ++ let _ = dry_penalty_map(&history, &config); ++ let (build_steps, scan_steps) = last_dry_breaker_cost(); ++ assert!( ++ build_steps <= 4 * config.dry_sequence_breakers.len() + 16, ++ "exact-list build took {build_steps} steps for {} tokens", ++ config.dry_sequence_breakers.len() ++ ); ++ assert!( ++ scan_steps <= 3 * WINDOW + 16, ++ "exact-list scan took {scan_steps} steps for window {WINDOW}" ++ ); ++ assert!( ++ last_dry_step_cost() <= 8 * WINDOW, ++ "window work must stay linear independently of the exact-list size" ++ ); ++ ++ let logits = ffi::from_slice_f32(&[0.0; 64], &[1, 64]); ++ let reference = apply_dry_penalty_reference(&logits, &history, &config); ++ let automaton = apply_dry_penalty(&logits, &history, &config); ++ assert_penalty_logits_equal( ++ &automaton, ++ &reference, ++ "large exact list with a rare last token", ++ ); ++ } ++ ++ #[test] ++ fn dry_breaker_automaton_long_shared_tails_are_additive() { ++ const WINDOW: usize = 4096; ++ const MAX_TAIL: usize = 128; ++ let mut breaker_heads = HashMap::new(); ++ let mut inserted_tokens = 0usize; ++ for len in 1..=MAX_TAIL { ++ let tail = vec![1; len]; ++ inserted_tokens += tail.len() + 1; ++ breaker_heads.insert(10_000 + len as i32, vec![tail]); ++ } ++ let config = SamplingConfig { ++ dry_multiplier: 0.75, ++ dry_base: 1.75, ++ dry_allowed_length: 2, ++ dry_penalty_last_n: crate::generate::DRY_FULL_HISTORY, ++ dry_breaker_heads: std::sync::Arc::new(breaker_heads), ++ ..Default::default() ++ }; ++ let mut history = vec![1; WINDOW - 1]; ++ history.push(63); ++ ++ let _ = dry_penalty_map(&history, &config); ++ let (build_steps, scan_steps) = last_dry_breaker_cost(); ++ assert!( ++ build_steps <= 8 * inserted_tokens + 32, ++ "shared-tail build took {build_steps} steps for {inserted_tokens} inserted tokens" ++ ); ++ assert!( ++ scan_steps <= 3 * WINDOW + 16, ++ "shared-tail scan took {scan_steps} steps for window {WINDOW}" ++ ); ++ assert!( ++ last_dry_step_cost() <= 8 * WINDOW, ++ "window work must not multiply by the shared-tail family" ++ ); ++ ++ let logits = ffi::from_slice_f32(&[0.0; 64], &[1, 64]); ++ let reference = apply_dry_penalty_reference(&logits, &history, &config); ++ let automaton = apply_dry_penalty(&logits, &history, &config); ++ assert_penalty_logits_equal( ++ &automaton, ++ &reference, ++ "long shared tails with a rare last token", ++ ); ++ } ++ + #[test] + fn dry_reverse_z_step_cost_is_linear() { + let config = SamplingConfig { diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt index 30a53c933..7a3fd422a 100644 --- a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/provenance.txt @@ -1,6 +1,18 @@ measurement_checkout=ecb89a308aecc468dd1886d9c9f4982f556f3c93 measurement_checkout_clean=true base_commit=73700aafa6a792f07830578c0c26137cea3a038b +measurement_tree=58b9a04161eaf88048699d0930a69dd59ccdd7b3 +measurement_sampling_blob=59fb6b02c6b54913aa549085fafc090e0b516d55 +publication_base=3ae626c2a4d67e7628a4050f01cc62b7419296fb +publication_checkout=327b6bf13a31d0f98e4cce717443d03b4be2cb73 +publication_tree=77e800ea62c641c80d0d5341da8a0d974e162bcf +publication_sampling_blob=59fb6b02c6b54913aa549085fafc090e0b516d55 +equivalence_paths=src/lib/mlxcel-core src/lib/mlxcel-cpp Cargo.lock +equivalence_diff_bytes=0 +equivalence_diff_sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +implementation_patch=implementation-73700aaf-to-ecb89a3.patch +implementation_patch_sha256=d28554322892d1103464f426996cede079336c54f5103843b0b6c63e95b7ebd2 +implementation_patch_bytes=43470 test_artifact=target/test-fast/deps/mlxcel_core-5feb86ac05c84f53 test_artifact_sha256=128d6b6c4e06c8a0a2c473d268c318110c643a583f8e233e1b3e35adf7aad9c8 inner_driver=/tmp/mlxcel-auto-plan/2259-quiet-timing.sh diff --git a/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/rebase-equivalence.txt b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/rebase-equivalence.txt new file mode 100644 index 000000000..0ccb52142 --- /dev/null +++ b/docs/benchmark_results/data/dry-penalty-step-cost-gb10-2026-10-11/rebase-equivalence.txt @@ -0,0 +1,19 @@ +measurement_base=73700aafa6a792f07830578c0c26137cea3a038b +measurement_commit_1=a7e8bb910c1fb5dd762bc54f2b57aa560e660e39 +measurement_commit_2=ecb89a308aecc468dd1886d9c9f4982f556f3c93 +measurement_tree=58b9a04161eaf88048699d0930a69dd59ccdd7b3 +measurement_sampling_blob=59fb6b02c6b54913aa549085fafc090e0b516d55 +publication_base=3ae626c2a4d67e7628a4050f01cc62b7419296fb +publication_checkout=327b6bf13a31d0f98e4cce717443d03b4be2cb73 +publication_tree=77e800ea62c641c80d0d5341da8a0d974e162bcf +publication_sampling_blob=59fb6b02c6b54913aa549085fafc090e0b516d55 +comparison_paths=src/lib/mlxcel-core src/lib/mlxcel-cpp Cargo.lock +comparison_command=git diff --exit-code ecb89a308aecc468dd1886d9c9f4982f556f3c93 327b6bf13a31d0f98e4cce717443d03b4be2cb73 -- src/lib/mlxcel-core src/lib/mlxcel-cpp Cargo.lock +comparison_exit=0 +comparison_stdout_bytes=0 +comparison_stdout_sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +implementation_patch=implementation-73700aaf-to-ecb89a3.patch +implementation_patch_sha256=d28554322892d1103464f426996cede079336c54f5103843b0b6c63e95b7ebd2 +implementation_patch_bytes=43470 +implementation_patch_commits=a7e8bb910c1fb5dd762bc54f2b57aa560e660e39,ecb89a308aecc468dd1886d9c9f4982f556f3c93 +result=measured and publication sampling blobs are identical; the scoped core, native dependency, and lockfile diff is empty diff --git a/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md b/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md index c849d0424..e2bd5c9ac 100644 --- a/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md +++ b/docs/benchmark_results/dry-penalty-step-cost-gb10-2026-10-11.md @@ -6,8 +6,10 @@ Issue #2259 replaces the DRY sampler's repeated backward suffix scans with a rev - Host: NVIDIA GB10 on `spark-102` - Backend: CUDA, pinned MLX `81ba1c6a0e50a9268b931579c2d4f1158b9aab5a` -- Source commit: `ecb89a308aecc468dd1886d9c9f4982f556f3c93` -- Base commit: `73700aafa6a792f07830578c0c26137cea3a038b` +- Measured source commit: `ecb89a308aecc468dd1886d9c9f4982f556f3c93` +- Measurement base commit: `73700aafa6a792f07830578c0c26137cea3a038b` +- Rebased publication checkout: `327b6bf13a31d0f98e4cce717443d03b4be2cb73` on `3ae626c2a4d67e7628a4050f01cc62b7419296fb` +- Source equivalence: both checkouts use `sampling.rs` blob `59fb6b02c6b54913aa549085fafc090e0b516d55`; their scoped diff across `src/lib/mlxcel-core`, `src/lib/mlx-cpp`, and `Cargo.lock` is empty - Profile: `test-fast` - Measured test artifact: `target/test-fast/deps/mlxcel_core-5feb86ac05c84f53`, SHA-256 `128d6b6c4e06c8a0a2c473d268c318110c643a583f8e233e1b3e35adf7aad9c8` - Inner driver SHA-256: `06b48baeb28c191f550564075a65e7e1f6ee6c9dc594437a63a467319c1ab988` @@ -18,6 +20,8 @@ Issue #2259 replaces the DRY sampler's repeated backward suffix scans with a rev The outer driver ran `gpu-lock run --tag issue-2259-quiet-timing -- bash /tmp/mlxcel-auto-plan/2259-quiet-timing.sh`. The raw package records that invocation, checkout state, test artifact provenance, host details, stdout/stderr, validation logs, and checksums. +The benchmark preceded the PR branch rebase. `implementation-73700aaf-to-ecb89a3.patch` preserves the original two implementation commits from the public measurement base, while `rebase-equivalence.txt` records the measured and publication tree and source identities plus the empty scoped diff. The original `environment.txt` remains byte-for-byte unchanged. + ## Method `sampling::tests::dry_step_cost_report` constructs one row of four f32 logits and period-3 token histories of 1,024, 4,096, and 16,384 tokens, with no configured breakers. For each window it measures the exact pre-#2259 reference and the new production path. Reference repetitions are 3, 3, and 1; production repetitions are 20 for each window. Values below are elapsed host microseconds per call as printed by the test. Since MLX is lazy and the arrays are only passed to `black_box`, the measurement includes host matching and graph construction but excludes device evaluation. @@ -46,8 +50,10 @@ The package at `data/dry-penalty-step-cost-gb10-2026-10-11/` contains: - `hostgate.log`: byte-preserved host-gate result. - `environment.txt`: byte-preserved host, GPU, checkout, MLX, timing, and artifact output. - `provenance.txt`: derived source, artifact, driver, host-gate, and raw-log hash mapping. +- `implementation-73700aaf-to-ecb89a3.patch`: exact two-commit format patch from the public measurement base to the measured source. +- `rebase-equivalence.txt`: measured-source, publication-checkout, source-blob, and empty scoped-diff proof. - `core-build.log`, `clippy.log`, `fmt.log`, `sampling-tests.log`, `engine-tests.log`: byte-preserved final validation logs. - `README.md`: artifact interpretation and replay instructions. - `SHA256SUMS`: checksums for the package. -After squash merge, rebuild the CUDA test artifact and reproduce from the merged commit using the driver command recorded in `README.md`; the production and test code should differ from the measured source only by the squash identity and evidence/report additions unless the PR changes after this record is captured. +For an exact replay of the measured implementation, start at public commit `73700aafa6a792f07830578c0c26137cea3a038b`, apply the archived two-commit format patch with `git am`, and verify tree `58b9a04161eaf88048699d0930a69dd59ccdd7b3`. After squash merge, a fresh build from the merge commit is also valid when the scoped source-equivalence check remains empty; the raw measurement still belongs to the original measured commit and artifact.