From 502c2b8472f4acce10807685e07779b1ed2ced06 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 22:49:32 +0000 Subject: [PATCH 1/3] fix: plumb the estimated flag end to end (fixes #29) OnBoundaryCallback gains a 6th param `estimated: bool`; WordBoundary carries the field; tts_set_on_boundary3 now reports the real value. - floravox: forwards floravox-core WordTiming.estimated (true on unpatched voices, false on duration-tensor-measured ones) - sherpa-onnx: true (150-wpm estimates) - cloud: false (provider timings track the delivered audio) - system/default estimator: true - v1/v2 C-ABI callbacks unchanged; v3 (tts_set_on_boundary3) was the hardcoded-0 lie this removes --- examples/edge-bare-envelope.rs | 2 +- examples/floravox-stream-demo.rs | 2 +- examples/word-boundary-demo.rs | 53 ++++++++++++++++---------------- src/boundaries.rs | 2 ++ src/cloud_engine.rs | 14 +++++++-- src/engine.rs | 8 +++-- src/floravox_engine.rs | 10 +++++- src/lib.rs | 22 +++++++++---- src/sherpaonnx_engine.rs | 23 +++++++++++--- src/system_engine.rs | 2 +- src/types.rs | 4 +++ tests/sherpaonnx_live.rs | 6 ++-- 12 files changed, 101 insertions(+), 47 deletions(-) diff --git a/examples/edge-bare-envelope.rs b/examples/edge-bare-envelope.rs index e7631f2..8f4d396 100644 --- a/examples/edge-bare-envelope.rs +++ b/examples/edge-bare-envelope.rs @@ -24,7 +24,7 @@ fn main() { 1.0, 1.0, Some(&mut |chunk: &[u8]| bytes += chunk.len()), - Some(&mut |_w, _s, _e, _o, _l| words += 1), + Some(&mut |_w, _s, _e, _o, _l, _est| words += 1), None, ) .unwrap_or_else(|e| panic!("{text}: speak failed: {e}")); diff --git a/examples/floravox-stream-demo.rs b/examples/floravox-stream-demo.rs index e6eb1ac..e1a7063 100644 --- a/examples/floravox-stream-demo.rs +++ b/examples/floravox-stream-demo.rs @@ -51,7 +51,7 @@ fn main() -> Result<(), Box> { chunk.len() as u64 / 2 / 16, )); }), - Some(&mut |_w, _s, _e, _o, _l| { + Some(&mut |_w, _s, _e, _o, _l, _est| { nb.fetch_add(1, Ordering::SeqCst); }), None, diff --git a/examples/word-boundary-demo.rs b/examples/word-boundary-demo.rs index 413b822..0980ab0 100644 --- a/examples/word-boundary-demo.rs +++ b/examples/word-boundary-demo.rs @@ -117,32 +117,33 @@ fn demonstrate_word_boundaries( println!("🎵 Starting speech synthesis with word boundary events...\n"); // Set up word boundary callback - let mut boundary_callback = move |word: &str, start: f32, end: f32, _offset: i32, _len: i32| { - let mut word_list = words_clone.lock().unwrap(); - word_list.push((word.to_string(), start, end)); - - // Simulate word highlighting in a UI - let current_word = word; - let highlighted = word_list - .iter() - .map(|(w, _, _)| { - if w.to_lowercase() == current_word.to_lowercase() { - format!("[{}]", w) - } else { - w.clone() - } - }) - .collect::>() - .join(" "); - - println!( - "📍 Word: \"{}\" | Time: {:.3}s - {:.3}s", - current_word, start, end - ); - println!(" Highlighted: {}", highlighted); - - *count_clone.lock().unwrap() += 1; - }; + let mut boundary_callback = + move |word: &str, start: f32, end: f32, _offset: i32, _len: i32, _estimated: bool| { + let mut word_list = words_clone.lock().unwrap(); + word_list.push((word.to_string(), start, end)); + + // Simulate word highlighting in a UI + let current_word = word; + let highlighted = word_list + .iter() + .map(|(w, _, _)| { + if w.to_lowercase() == current_word.to_lowercase() { + format!("[{}]", w) + } else { + w.clone() + } + }) + .collect::>() + .join(" "); + + println!( + "📍 Word: \"{}\" | Time: {:.3}s - {:.3}s", + current_word, start, end + ); + println!(" Highlighted: {}", highlighted); + + *count_clone.lock().unwrap() += 1; + }; // Start speaking with word boundary events engine.speak( diff --git a/src/boundaries.rs b/src/boundaries.rs index a02dd4c..1c69262 100644 --- a/src/boundaries.rs +++ b/src/boundaries.rs @@ -223,11 +223,13 @@ mod tests { text: "hello".into(), offset: 0, duration: 400, + estimated: false, }, WordBoundary { text: "world".into(), offset: 400, duration: 400, + estimated: false, }, ]; let plan = EstimatePlan::from_estimates(&est, "hello world"); diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 713a3d2..798994b 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1400,6 +1400,7 @@ fn parse_google_timepoints( text: word.clone(), offset: tp.time_ms, duration, + estimated: false, }); } boundaries @@ -2163,6 +2164,7 @@ impl TtsEngine for CloudEngine { / 1000.0, final_offset, final_len, + false, ); } } @@ -2400,7 +2402,7 @@ impl TtsEngine for CloudEngine { search_from = char_offset as usize + word.len(); } let char_len = word.chars().count() as i32; - cb(&word, start, end, char_offset, char_len); + cb(&word, start, end, char_offset, char_len, false); } } } @@ -2445,6 +2447,7 @@ impl TtsEngine for CloudEngine { (b.offset + b.duration) as f32 / 1000.0, -1, -1, + false, ); } } else { @@ -2467,6 +2470,7 @@ impl TtsEngine for CloudEngine { (b.offset + b.duration) as f32 / 1000.0, char_offset, char_len, + false, ); } } @@ -2488,7 +2492,7 @@ impl TtsEngine for CloudEngine { } StreamEvt::Boundary(word, start, end, offset, len) => { if let Some(bcb) = on_boundary.as_mut() { - bcb(word, start, end, offset, len); + bcb(word, start, end, offset, len, false); } } }; @@ -4466,6 +4470,7 @@ mod tests { text: "Hi".into(), offset: 0, duration: 0, + estimated: false, }]; compute_durations(&mut v); assert_eq!(v[0].duration, 500); @@ -4478,16 +4483,19 @@ mod tests { text: "a".into(), offset: 0, duration: 0, + estimated: false, }, WordBoundary { text: "b".into(), offset: 300, duration: 0, + estimated: false, }, WordBoundary { text: "c".into(), offset: 700, duration: 0, + estimated: false, }, ]; compute_durations(&mut v); @@ -4503,11 +4511,13 @@ mod tests { text: "a".into(), offset: 0, duration: 250, + estimated: false, }, WordBoundary { text: "b".into(), offset: 250, duration: 0, + estimated: false, }, ]; compute_durations(&mut v); diff --git a/src/engine.rs b/src/engine.rs index 632e464..48fca37 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -7,9 +7,12 @@ use std::fmt; pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); /// Callback for word boundary events. -/// Signature: (word, start_sec, end_sec, char_offset, char_len) +/// Signature: (word, start_sec, end_sec, char_offset, char_len, estimated) /// char_offset/char_len are -1 when the engine doesn't report them. -pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32, i32, i32); +/// `estimated` is true for proportional estimates (unpatched voices, +/// sherpa-onnx's wpm model) and false for measured timings (floravox +/// duration tensor, cloud provider timings). +pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32, i32, i32, bool); /// Callback for SSML mark/bookmark events. /// Signature: (name, start_sec, end_sec, char_offset) @@ -318,6 +321,7 @@ pub fn estimate_word_boundaries_with_wpm(text: &str, words_per_minute: f64) -> V text: (*word).to_string(), offset: current_ms, duration, + estimated: true, }); current_ms += duration; } diff --git a/src/floravox_engine.rs b/src/floravox_engine.rs index 0c3f038..7673e27 100644 --- a/src/floravox_engine.rs +++ b/src/floravox_engine.rs @@ -344,12 +344,20 @@ impl FloravoxEngine { #[allow(clippy::cast_precision_loss)] let (s, e) = (w.ms_start as f32 / 1000.0, w.ms_end as f32 / 1000.0); #[allow(clippy::cast_possible_wrap)] - cb(&w.text, s, e, w.char_offset as i32, w.char_len as i32); + cb( + &w.text, + s, + e, + w.char_offset as i32, + w.char_len as i32, + w.estimated, + ); } boundaries.push(WordBoundary { text: w.text.clone(), offset: w.ms_start, duration: w.ms_end.saturating_sub(w.ms_start), + estimated: w.estimated, }); }; loop { diff --git a/src/lib.rs b/src/lib.rs index 9aa46e0..74ce5e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,7 +84,7 @@ pub type CVisemeCb = Option; pub type CVoidCb = Option; pub type CErrorCb = Option; type BoxedAudioCb = Box; -type BoxedBoundaryCb = Box; +type BoxedBoundaryCb = Box; type BoxedMarkCb = Box; pub struct tts_ctx { @@ -420,7 +420,12 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) let mut on_boundary_closure: Option = match (boundary.cb, boundary2.cb) { (None, None) => None, _ => Some(Box::new( - move |word: &str, start: f32, end: f32, char_offset: i32, char_len: i32| { + move |word: &str, + start: f32, + end: f32, + char_offset: i32, + char_len: i32, + estimated: bool| { if let Some(cb) = boundary.cb { if let Ok(c_word) = CString::new(word) { cb(c_word.as_ptr(), start, end, boundary.userdata); @@ -446,7 +451,7 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) char_len, start, end, - 0, + i32::from(estimated), boundary3.userdata, ); } @@ -498,7 +503,7 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) .map(|f| &mut **f as &mut dyn FnMut(&[u8])), on_boundary_closure .as_mut() - .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32, i32)), + .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32, i32, bool)), on_mark_closure .as_mut() .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32)), @@ -576,7 +581,12 @@ pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 let mut on_boundary_closure: Option = match (boundary.cb, boundary2.cb) { (None, None) => None, _ => Some(Box::new( - move |word: &str, start: f32, end: f32, char_offset: i32, char_len: i32| { + move |word: &str, + start: f32, + end: f32, + char_offset: i32, + char_len: i32, + estimated: bool| { if let Some(cb) = boundary.cb { if let Ok(c_word) = CString::new(word) { cb(c_word.as_ptr(), start, end, boundary.userdata); @@ -632,7 +642,7 @@ pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 .map(|f| &mut **f as &mut dyn FnMut(&[u8])), on_boundary_closure .as_mut() - .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32, i32)), + .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32, i32, bool)), on_mark_closure .as_mut() .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32)), diff --git a/src/sherpaonnx_engine.rs b/src/sherpaonnx_engine.rs index a947548..943a45d 100644 --- a/src/sherpaonnx_engine.rs +++ b/src/sherpaonnx_engine.rs @@ -21,7 +21,7 @@ static CANCEL_REQUESTED: AtomicBool = AtomicBool::new(false); // (same technique as VISEME_CB in the cloud engine) and clear them after. // Synthesis per engine instance is serialised by the tts_instance mutex. type AudioCbPtr = *mut dyn FnMut(&[u8]); -type BoundaryCbPtr = *mut dyn FnMut(&str, f32, f32, i32, i32); +type BoundaryCbPtr = *mut dyn FnMut(&str, f32, f32, i32, i32, bool); thread_local! { static STREAM_AUDIO_CB: std::cell::RefCell> = @@ -595,7 +595,7 @@ impl TtsEngine for SherpaOnnxEngine { // SAFETY: as above. unsafe { std::mem::transmute::< - *mut (dyn FnMut(&str, f32, f32, i32, i32) + '_), + *mut (dyn FnMut(&str, f32, f32, i32, i32, bool) + '_), BoundaryCbPtr, >(std::ptr::from_mut(&mut **cb)) } @@ -647,6 +647,7 @@ impl TtsEngine for SherpaOnnxEngine { ev.end_s, ev.char_offset, ev.char_len, + true, // wpm estimates ); } } @@ -671,7 +672,14 @@ impl TtsEngine for SherpaOnnxEngine { if let (Some(f), Some(cb)) = (firer.as_ref(), on_boundary.as_mut()) { if let Ok(mut f) = f.lock() { f.flush(&mut |ev| { - cb(&ev.word, ev.start_s, ev.end_s, ev.char_offset, ev.char_len); + cb( + &ev.word, + ev.start_s, + ev.end_s, + ev.char_offset, + ev.char_len, + true, + ); }); } } @@ -696,7 +704,14 @@ impl TtsEngine for SherpaOnnxEngine { let plan = EstimatePlan::build(text); for i in 0..plan.len() { let ev = plan.event(i).expect("in range"); - cb(&ev.word, ev.start_s, ev.end_s, ev.char_offset, ev.char_len); + cb( + &ev.word, + ev.start_s, + ev.end_s, + ev.char_offset, + ev.char_len, + true, + ); } } } diff --git a/src/system_engine.rs b/src/system_engine.rs index fdeb216..7aa8bc8 100644 --- a/src/system_engine.rs +++ b/src/system_engine.rs @@ -78,7 +78,7 @@ impl TtsEngine for SystemEngine { #[allow(clippy::cast_precision_loss)] let end = (b.offset + b.duration) as f32 / 1000.0; let char_len = b.text.chars().count() as i32; - cb(&b.text, start, end, char_offset, char_len); + cb(&b.text, start, end, char_offset, char_len, false); } } diff --git a/src/types.rs b/src/types.rs index a7b3883..964e3a6 100644 --- a/src/types.rs +++ b/src/types.rs @@ -240,6 +240,10 @@ pub struct WordBoundary { pub offset: u64, /// Duration of the word in milliseconds. pub duration: u64, + /// True when the timings are proportional estimates (unpatched + /// voices, sherpa-onnx's 150-wpm model); false when measured (floravox + /// duration tensor, cloud provider timings that track the audio). + pub estimated: bool, } /// Describes a registered engine for introspection. diff --git a/tests/sherpaonnx_live.rs b/tests/sherpaonnx_live.rs index c336f64..1b88a21 100644 --- a/tests/sherpaonnx_live.rs +++ b/tests/sherpaonnx_live.rs @@ -283,7 +283,7 @@ fn sherpa_streams_audio_per_sentence_batch() { *last_a.lock().unwrap() = t_start.elapsed(); seq_a.lock().unwrap().push(A); }), - Some(&mut move |_w, _s, _e, _o, _l| seq_b.lock().unwrap().push(B)), + Some(&mut move |_w, _s, _e, _o, _l, _est| seq_b.lock().unwrap().push(B)), None, ) .expect("speak"); @@ -316,7 +316,7 @@ fn vits_piper_word_boundaries_fire_per_word() { let engine = engine_for(&id); let sink = Arc::new(Mutex::new(BoundarySink::new())); let sink_cb = sink.clone(); - let mut bound_cb = move |word: &str, start: f32, end: f32, _: i32, _: i32| { + let mut bound_cb = move |word: &str, start: f32, end: f32, _: i32, _: i32, _: bool| { sink_cb .lock() .unwrap() @@ -454,7 +454,7 @@ fn matcha_word_boundaries_fire() { let engine = engine_for(&id); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; engine From c441400aa26066a823200299ddb5dff33ee6ff64 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 22:57:21 +0000 Subject: [PATCH 2/3] fix: boundary3 fan-out in the sync path (was missing; flag unused) --- src/lib.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 74ce5e0..3df4c29 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -558,6 +558,7 @@ pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 let audio = { *ctx_ref.on_audio.lock().unwrap() }; let boundary = { *ctx_ref.on_boundary.lock().unwrap() }; let boundary2 = { *ctx_ref.on_boundary2.lock().unwrap() }; + let boundary3 = { *ctx_ref.on_boundary3.lock().unwrap() }; let mark = { *ctx_ref.on_mark.lock().unwrap() }; let mut on_mark_closure: Option = mark.cb.map(|cb| { @@ -604,6 +605,19 @@ pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 ); } } + if let Some(cb) = boundary3.cb { + if let Ok(c_word) = CString::new(word) { + cb( + c_word.as_ptr(), + char_offset, + char_len, + start, + end, + i32::from(estimated), + boundary3.userdata, + ); + } + } }, )), }; From 922085db87ba5a0475629b41d0d91ae8bf36bd10 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 23:05:34 +0000 Subject: [PATCH 3/3] fix: avsynth/sapi boundary callbacks carry the estimated flag --- src/avsynth_engine.rs | 1 + src/sapi_engine.rs | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/avsynth_engine.rs b/src/avsynth_engine.rs index c68f8ed..0f9a3e3 100644 --- a/src/avsynth_engine.rs +++ b/src/avsynth_engine.rs @@ -119,6 +119,7 @@ impl TtsEngine for AvSynthEngine { (b.offset + b.duration) as f32 / 1000.0, -1, -1, + true, // wpm estimates ); } } diff --git a/src/sapi_engine.rs b/src/sapi_engine.rs index ef069d2..0c89417 100644 --- a/src/sapi_engine.rs +++ b/src/sapi_engine.rs @@ -180,7 +180,7 @@ impl SapiEngine { #[allow(clippy::cast_precision_loss)] let end = (b.offset + b.duration) as f32 / 1000.0; let char_len = b.text.chars().count() as i32; - cb(&b.text, start, end, char_offset, char_len); + cb(&b.text, start, end, char_offset, char_len, true); // estimates } } return Ok(()); @@ -248,7 +248,8 @@ impl SapiEngine { let start_sec = audio_offset_ms as f32 / 1000.0; #[allow(clippy::cast_precision_loss)] let end_sec = (audio_offset_ms + 1) as f32 / 1000.0; - cb(&word, start_sec, end_sec, pos as i32, len as i32); + cb(&word, start_sec, end_sec, pos as i32, len as i32, true); + // bookmark spans } } }