From a0eb9f7eea536cc3598e14f634cac1e1be4c28e4 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 20:42:49 +0000 Subject: [PATCH 1/5] feat: mark events through the trait and C ABI (OnMarkCallback, tts_set_on_mark) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine trait speak/speak_sync/speak_with_options take Option (name, start_s, end_s, char_offset); floravox forwards its measured MarkReached events (char_offset from the tag position — requires floravox-core 0.8.1); other engines accept and ignore for now - C ABI: tts_set_on_mark(name, char_offset, start_s, end_s, userdata) and tts_set_on_boundary3 (…, estimated, userdata) for the measured-vs-estimated flag; both wired through tts_speak and tts_speak_sync; header updated - floravox engine: deps to 0.8.1 (MarkReached carries char_offset, is_char_table auto-CharFrontend, Windows USERPROFILE HOME resolution, kokoro leading-silence trim) - all engines/examples/tests updated for the new signature --- Cargo.toml | 6 +- examples/advanced-features-demo.rs | 16 ++-- examples/edge-bare-envelope.rs | 1 + examples/edge-speechmarkdown.rs | 1 + examples/edge-style.rs | 1 + examples/floravox-stream-demo.rs | 2 +- examples/quick-start.rs | 14 ++-- examples/streaming-audio-demo.rs | 1 + examples/word-boundary-demo.rs | 1 + include/tts_wrapper.h | 35 +++++++++ src/cloud_engine.rs | 13 +++- src/engine.rs | 15 ++++ src/floravox_engine.rs | 100 +++++++++++++++++++++---- src/lib.rs | 116 +++++++++++++++++++++++++++++ src/sherpaonnx_engine.rs | 17 ++++- src/system_engine.rs | 13 +++- tests/sherpaonnx_live.rs | 46 +++++++++--- tests/trait_methods.rs | 2 +- 18 files changed, 354 insertions(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b531cce..399fe61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,9 +106,9 @@ icu_displaynames = { version = "0.11", optional = true, features = ["compiled_da icu_locid = { version = "1.0.0-beta1", optional = true } # floravox: event-driven SSML synthesis (piper/MMS VITS, Matcha, # Kokoro) with measured word timings (vs sherpa-onnx's estimated ones). -floravox-core = { version = "0.7.1", optional = true } -floravox-ssml = { version = "0.7.2", optional = true } -floravox-g2p = { version = "0.7.1", optional = true, default-features = false } +floravox-core = { version = "0.8.1", optional = true } +floravox-ssml = { version = "0.8.1", optional = true } +floravox-g2p = { version = "0.8.1", optional = true, default-features = false } # Published lexicon bundles (gruut-derived, MIT) + per-language # Phonetisaurus models, fetched and cached by lang code. voicegarden-lexicons = { version = "0.2.0", optional = true } diff --git a/examples/advanced-features-demo.rs b/examples/advanced-features-demo.rs index 02f0fa8..c9d63c7 100644 --- a/examples/advanced-features-demo.rs +++ b/examples/advanced-features-demo.rs @@ -56,6 +56,7 @@ fn main() -> Result<(), Box> { 1.0, None, None, + None, ) { Ok(_) => println!(" ✅ Voice test successful"), Err(e) => println!(" ⚠️ Voice test failed: {}", e), @@ -92,27 +93,27 @@ fn main() -> Result<(), Box> { let test_text = "This demonstrates prosody control in text-to-speech synthesis."; println!("🐢 Slow speech (0.7x rate):"); - engine.speak(test_text, None, 0.7, 1.0, 1.0, None, None)?; + engine.speak(test_text, None, 0.7, 1.0, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🐇 Fast speech (1.5x rate):"); - engine.speak(test_text, None, 1.5, 1.0, 1.0, None, None)?; + engine.speak(test_text, None, 1.5, 1.0, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🔉 Low pitch (0.8x):"); - engine.speak(test_text, None, 1.0, 0.8, 1.0, None, None)?; + engine.speak(test_text, None, 1.0, 0.8, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🔈 High pitch (1.2x):"); - engine.speak(test_text, None, 1.0, 1.2, 1.0, None, None)?; + engine.speak(test_text, None, 1.0, 1.2, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🔉 Low volume (0.5x):"); - engine.speak(test_text, None, 1.0, 1.0, 0.5, None, None)?; + engine.speak(test_text, None, 1.0, 1.0, 0.5, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🔊 High volume (1.5x):"); - engine.speak(test_text, None, 1.0, 1.0, 1.5, None, None)?; + engine.speak(test_text, None, 1.0, 1.0, 1.5, None, None, None)?; println!("✅ Prosody control demo complete\n"); } @@ -172,6 +173,7 @@ fn main() -> Result<(), Box> { 1.0, None, None, + None, ) { Ok(_) => println!(" ✅ Synthesis successful"), Err(e) => println!(" ❌ Synthesis failed: {}", e), @@ -223,7 +225,7 @@ fn main() -> Result<(), Box> { "#; println!("Testing SSML input..."); - match engine.speak(ssml_example, None, 1.0, 1.0, 1.0, None, None) { + match engine.speak(ssml_example, None, 1.0, 1.0, 1.0, None, None, None) { Ok(_) => println!("✅ SSML synthesis successful"), Err(e) => println!("❌ SSML synthesis failed: {}", e), } diff --git a/examples/edge-bare-envelope.rs b/examples/edge-bare-envelope.rs index a9990ef..e7631f2 100644 --- a/examples/edge-bare-envelope.rs +++ b/examples/edge-bare-envelope.rs @@ -25,6 +25,7 @@ fn main() { 1.0, Some(&mut |chunk: &[u8]| bytes += chunk.len()), Some(&mut |_w, _s, _e, _o, _l| words += 1), + None, ) .unwrap_or_else(|e| panic!("{text}: speak failed: {e}")); println!("{text:?}: {bytes} PCM bytes, {words} word boundaries"); diff --git a/examples/edge-speechmarkdown.rs b/examples/edge-speechmarkdown.rs index bc5a0d7..c1a77f1 100644 --- a/examples/edge-speechmarkdown.rs +++ b/examples/edge-speechmarkdown.rs @@ -21,6 +21,7 @@ fn main() { 1.0, Some(&mut |chunk: &[u8]| bytes += chunk.len()), None, + None, ) .unwrap_or_else(|e| panic!("{text}: {e}")); println!("{text:?}: {bytes} bytes"); diff --git a/examples/edge-style.rs b/examples/edge-style.rs index 82d8ec1..6ac5fc1 100644 --- a/examples/edge-style.rs +++ b/examples/edge-style.rs @@ -15,6 +15,7 @@ fn main() { 1.0, Some(&mut |c: &[u8]| bytes += c.len()), None, + None, ) .unwrap_or_else(|e| panic!("{text}: {e}")); println!("{text:?}: {bytes} bytes"); diff --git a/examples/floravox-stream-demo.rs b/examples/floravox-stream-demo.rs index 7a583df..e6eb1ac 100644 --- a/examples/floravox-stream-demo.rs +++ b/examples/floravox-stream-demo.rs @@ -54,8 +54,8 @@ fn main() -> Result<(), Box> { Some(&mut |_w, _s, _e, _o, _l| { nb.fetch_add(1, Ordering::SeqCst); }), + None, )?; - let total = t0.elapsed().as_millis() as u64; let fa = first_audio.load(Ordering::SeqCst); println!("first audio after {fa} ms"); diff --git a/examples/quick-start.rs b/examples/quick-start.rs index fb76e9d..c5fce05 100644 --- a/examples/quick-start.rs +++ b/examples/quick-start.rs @@ -36,6 +36,7 @@ fn main() -> Result<(), Box> { 1.0, // Normal volume None, // No audio callback (direct playback) None, // No word boundary callback + None, // No mark callback )?; std::thread::sleep(std::time::Duration::from_millis(500)); @@ -64,6 +65,7 @@ fn main() -> Result<(), Box> { 1.2, // Louder volume None, None, + None, )?; std::thread::sleep(std::time::Duration::from_millis(500)); @@ -86,23 +88,23 @@ fn main() -> Result<(), Box> { let test_sentence = "This demonstrates rate and pitch control in text-to-speech."; println!("🐢 Slow speech:"); - engine.speak(test_sentence, None, 0.7, 1.0, 1.0, None, None)?; + engine.speak(test_sentence, None, 0.7, 1.0, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🎯 Normal speech:"); - engine.speak(test_sentence, None, 1.0, 1.0, 1.0, None, None)?; + engine.speak(test_sentence, None, 1.0, 1.0, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🐇 Fast speech:"); - engine.speak(test_sentence, None, 1.5, 1.0, 1.0, None, None)?; + engine.speak(test_sentence, None, 1.5, 1.0, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🔈 Low pitch:"); - engine.speak(test_sentence, None, 1.0, 0.8, 1.0, None, None)?; + engine.speak(test_sentence, None, 1.0, 0.8, 1.0, None, None, None)?; std::thread::sleep(std::time::Duration::from_millis(100)); println!("🔊 High pitch:"); - engine.speak(test_sentence, None, 1.0, 1.2, 1.0, None, None)?; + engine.speak(test_sentence, None, 1.0, 1.2, 1.0, None, None, None)?; // Example 4: Saving to file println!("\n\nExample 4: Saving Audio to File"); @@ -131,7 +133,7 @@ fn main() -> Result<(), Box> { println!("let engine = create_engine(\"system\", \"\")?;"); println!(""); println!("// Simple speech"); - println!("engine.speak(\"Hello!\", None, 1.0, 1.0, 1.0, None, None)?;"); + println!("engine.speak(\"Hello!\", None, 1.0, 1.0, 1.0, None, None, None)?;"); println!(""); println!("// Save to file"); println!("let audio = engine.synth_to_bytes(\"Hello file!\", None, 1.0, 1.0, 1.0)?;"); diff --git a/examples/streaming-audio-demo.rs b/examples/streaming-audio-demo.rs index a091c98..f40a21c 100644 --- a/examples/streaming-audio-demo.rs +++ b/examples/streaming-audio-demo.rs @@ -145,6 +145,7 @@ fn stream_to_file( } }), None, // no boundary callback + None, // no mark callback )?; println!( diff --git a/examples/word-boundary-demo.rs b/examples/word-boundary-demo.rs index e2bc46f..413b822 100644 --- a/examples/word-boundary-demo.rs +++ b/examples/word-boundary-demo.rs @@ -153,6 +153,7 @@ fn demonstrate_word_boundaries( 1.0, // volume None, // no audio callback for this demo Some(&mut boundary_callback), + None, // no mark callback for this demo )?; // Give time for all events to complete diff --git a/include/tts_wrapper.h b/include/tts_wrapper.h index 958aac8..8a0c0a0 100644 --- a/include/tts_wrapper.h +++ b/include/tts_wrapper.h @@ -43,6 +43,21 @@ typedef void (*CBoundaryCb)(const char*, float, float, void*); typedef void (*CBoundaryCb2)(const char*, int32_t, int32_t, float, float, void*); +/** + * Mark/bookmark callback: cb(name, char_offset, start_s, end_s, userdata). + * char_offset is -1 when unknown; start/end are the measured (or + * estimated) audio position the mark fires at. + */ +typedef void (*CMarkCb)(const char*, int32_t, float, float, void*); + +/** + * Boundary callback with the estimated flag: cb(word, char_offset, + * char_len, start_s, end_s, estimated, userdata). `estimated` is 1 when + * the timings are proportional estimates (unpatched voice), 0 when + * measured from the model's duration tensor. + */ +typedef void (*CBoundaryCb3)(const char*, int32_t, int32_t, float, float, int32_t, void*); + typedef void (*CVisemeCb)(int32_t, float, void*); typedef void (*CVoidCb)(void*); @@ -235,6 +250,26 @@ void tts_set_on_boundary(struct tts_ctx *ctx, CBoundaryCb cb, void *userdata); */ void tts_set_on_boundary2(struct tts_ctx *ctx, CBoundaryCb2 cb, void *userdata); +/** + * Set the mark/bookmark callback: cb(name, char_offset, start_s, end_s, userdata). + * Fires for ``/`` SSML tags at their measured audio + * position on engines that report them (floravox). + * + * # Safety + * `ctx` must be valid. + */ +void tts_set_on_mark(struct tts_ctx *ctx, CMarkCb cb, void *userdata); + +/** + * Boundary callback with the estimated flag: + * cb(word, char_offset, char_len, start_s, end_s, estimated, userdata). + * `estimated` != 0 means proportional estimates, not measured timings. + * + * # Safety + * `ctx` must be valid. + */ +void tts_set_on_boundary3(struct tts_ctx *ctx, CBoundaryCb3 cb, void *userdata); + /** * Viseme callback for lip-sync / facial animation. * cb(viseme_id, audio_offset_sec, userdata) diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 4cdb366..713a3d2 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1830,6 +1830,7 @@ impl TtsEngine for CloudEngine { volume: f32, mut on_audio: Option, mut on_boundary: Option, + _on_mark: Option, ) -> TtsResult<()> { let (original_text, is_ssml) = preprocess_speech_markdown(text, &self.config.provider_id); @@ -2525,8 +2526,18 @@ impl TtsEngine for CloudEngine { volume: f32, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()> { - self.speak(text, voice, rate, pitch, volume, on_audio, on_boundary) + self.speak( + text, + voice, + rate, + pitch, + volume, + on_audio, + on_boundary, + on_mark, + ) } fn stop(&self) -> TtsResult<()> { diff --git a/src/engine.rs b/src/engine.rs index a777605..632e464 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -11,6 +11,14 @@ pub type OnAudioCallback<'a> = &'a mut dyn FnMut(&[u8]); /// char_offset/char_len are -1 when the engine doesn't report them. pub type OnBoundaryCallback<'a> = &'a mut dyn FnMut(&str, f32, f32, i32, i32); +/// Callback for SSML mark/bookmark events. +/// Signature: (name, start_sec, end_sec, char_offset) +/// char_offset is -1 when the engine doesn't report it. Engines that +/// support native `` (floravox) fire this at the measured (or +/// estimated) audio position; consumers map it to their bookmark event +/// (SAPI `SPEI_TTS_BOOKMARK`, SSIP index marks). +pub type OnMarkCallback<'a> = &'a mut dyn FnMut(&str, f32, f32, i32); + /// Callback for speech-started events. pub type OnStartCallback<'a> = &'a mut dyn FnMut(); @@ -143,6 +151,7 @@ fn extract_name_attr(tag: &str) -> Option { pub trait TtsEngine: Send + Sync + fmt::Debug { /// Start speaking `text` asynchronously. #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] fn speak( &self, text: &str, @@ -152,6 +161,7 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { volume: f32, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()>; /// Speak with full [`SpeakOptions`], matching Swift's `speak(_:options:)`. @@ -161,6 +171,7 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { options: Option<&SpeakOptions>, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()> { let opts = options.cloned().unwrap_or_default(); self.speak( @@ -171,11 +182,13 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { opts.effective_volume(), on_audio, on_boundary, + on_mark, ) } /// Speak `text` synchronously, blocking until synthesis completes. #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] fn speak_sync( &self, text: &str, @@ -185,6 +198,7 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { volume: f32, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()>; /// Stop any in-progress speech. @@ -236,6 +250,7 @@ pub trait TtsEngine: Send + Sync + fmt::Debug { buf.extend_from_slice(chunk); }), None, + None, )?; Ok(buf) } diff --git a/src/floravox_engine.rs b/src/floravox_engine.rs index a25ddbb..0c3f038 100644 --- a/src/floravox_engine.rs +++ b/src/floravox_engine.rs @@ -103,8 +103,14 @@ impl Config { /// convention for directory credentials). fn expand_tilde(p: &str) -> PathBuf { if let Some(rest) = p.strip_prefix("~/") { - if let Some(home) = std::env::var_os("HOME") { - return PathBuf::from(home).join(rest); + // Unix HOME first, Windows USERPROFILE second ($HOME is almost + // never set there). + for key in ["HOME", "USERPROFILE"] { + if let Some(home) = std::env::var_os(key) { + if !home.is_empty() { + return PathBuf::from(home).join(rest); + } + } } } PathBuf::from(p) @@ -243,6 +249,7 @@ impl FloravoxEngine { voice: Option<&str>, doc_lang: Option<&str>, ) -> TtsResult>> { + #[cfg_attr(not(feature = "floravox-lexicons"), allow(unused_variables))] let effective_lang = self.lang.as_deref().or(doc_lang); let onnx = self.resolve_model(voice)?; let key = format!( @@ -267,10 +274,12 @@ impl FloravoxEngine { } let model: Box = floravox_core::load_voice(&onnx) .map_err(|e| TtsError(format!("loading {}: {e:#}", onnx.display())))?; + let auto_chars = model.config().is_char_table; let mut synth = Synthesizer::new(model, build_phonemizer(self, effective_lang)); // Document-level pre-passes, in order of specificity: - // chars (MMS-style character voices) > misaki (English) + // explicit chars credential > auto-detected character table + // (MMS-style voices) > misaki (English) if let Some(spec) = self.chars.as_deref() { let rom: Option<&'static str> = match spec { "" | "true" => None, @@ -280,6 +289,14 @@ impl FloravoxEngine { lowercase: true, romanize: rom, })); + } else if auto_chars { + // Character-table voice with no explicit frontend: CharFrontend + // is the only correct choice — phonemizing per-word would + // spell everything out. + synth = synth.with_document_phonemizer(Box::new(CharFrontend { + lowercase: true, + romanize: None, + })); } else if let Some(dialect) = self.misaki.as_deref() { let british = dialect.eq_ignore_ascii_case("gb"); synth = synth.with_document_phonemizer(Box::new(MisakiPrePass( @@ -294,18 +311,32 @@ impl FloravoxEngine { /// Shared pump: streams audio + events from a synthesis, feeding the /// callbacks. Returns collected `(pcm bytes, boundaries)`. + #[allow(clippy::too_many_arguments)] fn pump( &self, stream: StreamingSynthesis, volume: f32, mut on_audio: Option>, mut on_boundary: Option>, + mut on_mark: Option>, collect: bool, ) -> TtsResult<(Vec, Vec)> { let StreamingSynthesis { audio, events } = stream; let mut bytes = Vec::new(); let mut boundaries = Vec::new(); self.cancel.store(false, Ordering::SeqCst); + let fire_mark = + |name: &str, + ms: u64, + char_offset: i64, + on_mark: &mut Option>| { + if let Some(cb) = on_mark.as_mut() { + #[allow(clippy::cast_precision_loss)] + let s = ms as f32 / 1000.0; + #[allow(clippy::cast_possible_wrap)] + cb(name, s, s, char_offset as i32); + } + }; let fire_boundary = |w: &floravox_core::WordTiming, on_boundary: &mut Option>, boundaries: &mut Vec| { @@ -328,8 +359,19 @@ impl FloravoxEngine { // Drain pending events first so boundaries precede the audio // they time. while let Ok(ev) = events.try_recv() { - if let SynthesisEvent::WordBoundary(w) = ev { - fire_boundary(&w, &mut on_boundary, &mut boundaries); + match ev { + SynthesisEvent::WordBoundary(w) => { + fire_boundary(&w, &mut on_boundary, &mut boundaries); + } + SynthesisEvent::MarkReached { + name, + ms, + char_offset, + .. + } => { + fire_mark(&name, ms, char_offset, &mut on_mark); + } + _ => {} } } match audio.recv_timeout(POLL) { @@ -347,8 +389,19 @@ impl FloravoxEngine { Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { // Audio done; drain the remaining events. for ev in events { - if let SynthesisEvent::WordBoundary(w) = ev { - fire_boundary(&w, &mut on_boundary, &mut boundaries); + match ev { + SynthesisEvent::WordBoundary(w) => { + fire_boundary(&w, &mut on_boundary, &mut boundaries); + } + SynthesisEvent::MarkReached { + name, + ms, + char_offset, + .. + } => { + fire_mark(&name, ms, char_offset, &mut on_mark); + } + _ => {} } } return Ok((bytes, boundaries)); @@ -360,14 +413,18 @@ impl FloravoxEngine { /// Default models dir: `~/.rust-tts-wrapper/floravox`. fn default_models_dir() -> PathBuf { - std::env::var_os("HOME").map_or_else( - || PathBuf::from(".floravox"), - |h| PathBuf::from(h).join(".rust-tts-wrapper").join("floravox"), - ) + ["HOME", "USERPROFILE"] + .iter() + .find_map(|k| { + let h = std::env::var_os(k)?; + (!h.is_empty()).then(|| PathBuf::from(h).join(".rust-tts-wrapper").join("floravox")) + }) + .unwrap_or_else(|| PathBuf::from(".floravox")) } /// Build the phonemizer stack from the engine's g2p options. /// OOV chain: Phonetisaurus → ByT5 → letter spelling (first hit wins). +#[cfg_attr(not(feature = "floravox-lexicons"), allow(unused_variables))] fn build_phonemizer(engine: &FloravoxEngine, effective_lang: Option<&str>) -> Phon { // Resolve the lexicon stem: explicit `lexicon` config wins; with the // floravox-lexicons feature, the published bundle for the voice's @@ -677,6 +734,7 @@ fn bcp47_from(espeak_voice: &str, dataset: &str) -> (String, &'static str) { } impl TtsEngine for FloravoxEngine { + #[allow(clippy::too_many_arguments)] fn speak( &self, text: &str, @@ -686,16 +744,18 @@ impl TtsEngine for FloravoxEngine { volume: f32, on_audio: Option>, on_boundary: Option>, + on_mark: Option>, ) -> TtsResult<()> { let input = prepare_input(text, rate); let synth = self.synthesizer_for(voice, document_lang(text).as_deref())?; let stream = synth .synthesize_stream(&input) .map_err(|e| TtsError(format!("floravox synthesis: {e:#}")))?; - self.pump(stream, volume, on_audio, on_boundary, false) + self.pump(stream, volume, on_audio, on_boundary, on_mark, false) .map(|_| ()) } + #[allow(clippy::too_many_arguments)] fn speak_sync( &self, text: &str, @@ -705,8 +765,18 @@ impl TtsEngine for FloravoxEngine { volume: f32, on_audio: Option>, on_boundary: Option>, + on_mark: Option>, ) -> TtsResult<()> { - self.speak(text, voice, rate, pitch, volume, on_audio, on_boundary) + self.speak( + text, + voice, + rate, + pitch, + volume, + on_audio, + on_boundary, + on_mark, + ) } fn stop(&self) -> TtsResult<()> { @@ -750,7 +820,7 @@ impl TtsEngine for FloravoxEngine { let stream = synth .synthesize_stream(&input) .map_err(|e| TtsError(format!("floravox synthesis: {e:#}")))?; - self.pump(stream, volume, None, None, true) + self.pump(stream, volume, None, None, None, true) } } @@ -826,6 +896,7 @@ mod tests { } #[test] + #[cfg(feature = "speechmarkdown")] fn speechmarkdown_flows_into_floravox_ssml() { // SpeechMarkdown inline modifiers → generic-dialect SSML that // floravox parses natively (break, prosody rate, sub, say-as). @@ -840,6 +911,7 @@ mod tests { } #[test] + #[cfg(feature = "speechmarkdown")] fn whisper_maps_to_floravox_prosody() { let input = prepare_input("(be very quiet)[whisper]", 1.0); assert!( diff --git a/src/lib.rs b/src/lib.rs index ecdd958..9aa46e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,11 +70,22 @@ pub type CAudioCb = Option; pub type CBoundaryCb2 = Option; +/// Mark/bookmark callback: cb(name, char_offset, start_s, end_s, userdata). +/// char_offset is -1 when unknown; start/end are the measured (or +/// estimated) audio position the mark fires at. +pub type CMarkCb = Option; +/// Boundary callback with the estimated flag: cb(word, char_offset, +/// char_len, start_s, end_s, estimated, userdata). `estimated` is 1 when +/// the timings are proportional estimates (unpatched voice), 0 when +/// measured from the model's duration tensor. +pub type CBoundaryCb3 = + Option; pub type CVisemeCb = Option; pub type CVoidCb = Option; pub type CErrorCb = Option; type BoxedAudioCb = Box; type BoxedBoundaryCb = Box; +type BoxedMarkCb = Box; pub struct tts_ctx { // The TtsEngine trait already requires Send + Sync, and every engine @@ -105,6 +116,8 @@ pub struct tts_ctx { on_end: Mutex, on_error: Mutex, on_boundary2: Mutex, + on_boundary3: Mutex, + on_mark: Mutex, on_viseme: Mutex, } @@ -143,6 +156,18 @@ struct BoundaryCallback2 { userdata: *mut std::ffi::c_void, } +#[derive(Clone, Copy)] +struct MarkCallback { + cb: CMarkCb, + userdata: *mut std::ffi::c_void, +} + +#[derive(Clone, Copy)] +struct BoundaryCallback3 { + cb: CBoundaryCb3, + userdata: *mut std::ffi::c_void, +} + /// Bundled viseme callback. #[derive(Clone, Copy)] struct VisemeCallback { @@ -266,6 +291,14 @@ fn tts_create_inner(engine_id: *const c_char, credentials_json: *const c_char) - cb: None, userdata: ptr::null_mut(), }), + on_boundary3: Mutex::new(BoundaryCallback3 { + cb: None, + userdata: std::ptr::null_mut(), + }), + on_mark: Mutex::new(MarkCallback { + cb: None, + userdata: std::ptr::null_mut(), + }), on_boundary2: Mutex::new(BoundaryCallback2 { cb: None, userdata: ptr::null_mut(), @@ -374,6 +407,8 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) let audio = { *ctx_ref.on_audio.lock().unwrap() }; let boundary = { *ctx_ref.on_boundary.lock().unwrap() }; let boundary2 = { *ctx_ref.on_boundary2.lock().unwrap() }; + let boundary3 = { *ctx_ref.on_boundary3.lock().unwrap() }; + let mark = { *ctx_ref.on_mark.lock().unwrap() }; let mut on_audio_closure: Option = match audio.cb { Some(cb) => Some(Box::new(move |bytes: &[u8]| { @@ -403,10 +438,34 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) ); } } + if let Some(cb) = boundary3.cb { + if let Ok(c_word) = CString::new(word) { + cb( + c_word.as_ptr(), + char_offset, + char_len, + start, + end, + 0, + boundary3.userdata, + ); + } + } }, )), }; + let mut on_mark_closure: Option = mark.cb.map(|cb| { + let ud = mark.userdata; + let b: BoxedMarkCb = + Box::new(move |name: &str, start: f32, end: f32, char_offset: i32| { + if let Ok(c_name) = CString::new(name) { + cb(c_name.as_ptr(), char_offset, start, end, ud); + } + }); + b + }); + let start_cb = { *ctx_ref.on_start.lock().unwrap() }; let end_cb = { *ctx_ref.on_end.lock().unwrap() }; let error_cb = { *ctx_ref.on_error.lock().unwrap() }; @@ -440,6 +499,9 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) on_boundary_closure .as_mut() .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32, i32)), + on_mark_closure + .as_mut() + .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32)), ); match result { @@ -473,6 +535,7 @@ fn tts_speak_impl_inner(ctx: *mut tts_ctx, text: *const c_char, raw_ssml: bool) /// `ctx` must be a valid pointer from [`tts_create`]. /// `text` must be a valid null-terminated C string. #[no_mangle] +#[allow(clippy::too_many_lines)] pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 { ffi_catch!({ if ctx.is_null() || text.is_null() { @@ -490,6 +553,18 @@ pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 let audio = { *ctx_ref.on_audio.lock().unwrap() }; let boundary = { *ctx_ref.on_boundary.lock().unwrap() }; let boundary2 = { *ctx_ref.on_boundary2.lock().unwrap() }; + let mark = { *ctx_ref.on_mark.lock().unwrap() }; + + let mut on_mark_closure: Option = mark.cb.map(|cb| { + let ud = mark.userdata; + let b: BoxedMarkCb = + Box::new(move |name: &str, start: f32, end: f32, char_offset: i32| { + if let Ok(c_name) = CString::new(name) { + cb(c_name.as_ptr(), char_offset, start, end, ud); + } + }); + b + }); let mut on_audio_closure: Option = match audio.cb { Some(cb) => Some(Box::new(move |bytes: &[u8]| { @@ -558,6 +633,9 @@ pub extern "C" fn tts_speak_sync(ctx: *mut tts_ctx, text: *const c_char) -> i32 on_boundary_closure .as_mut() .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32, i32)), + on_mark_closure + .as_mut() + .map(|f| &mut **f as &mut dyn FnMut(&str, f32, f32, i32)), ); match result { @@ -849,6 +927,44 @@ pub extern "C" fn tts_set_on_boundary2( })); } +/// Set the mark/bookmark callback: cb(name, char_offset, start_s, end_s, userdata). +/// Fires for ``/`` SSML tags at their measured audio +/// position on engines that report them (floravox). +/// +/// # Safety +/// `ctx` must be valid. +#[no_mangle] +pub extern "C" fn tts_set_on_mark(ctx: *mut tts_ctx, cb: CMarkCb, userdata: *mut std::ffi::c_void) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if ctx.is_null() { + return; + } + let ctx_ref = unsafe { &*ctx }; + *ctx_ref.on_mark.lock().unwrap() = MarkCallback { cb, userdata }; + })); +} + +/// Boundary callback with the estimated flag: +/// cb(word, char_offset, char_len, start_s, end_s, estimated, userdata). +/// `estimated` != 0 means proportional estimates, not measured timings. +/// +/// # Safety +/// `ctx` must be valid. +#[no_mangle] +pub extern "C" fn tts_set_on_boundary3( + ctx: *mut tts_ctx, + cb: CBoundaryCb3, + userdata: *mut std::ffi::c_void, +) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if ctx.is_null() { + return; + } + let ctx_ref = unsafe { &*ctx }; + *ctx_ref.on_boundary3.lock().unwrap() = BoundaryCallback3 { cb, userdata }; + })); +} + /// Viseme callback for lip-sync / facial animation. /// cb(viseme_id, audio_offset_sec, userdata) /// diff --git a/src/sherpaonnx_engine.rs b/src/sherpaonnx_engine.rs index a721ace..1775fa3 100644 --- a/src/sherpaonnx_engine.rs +++ b/src/sherpaonnx_engine.rs @@ -338,6 +338,7 @@ impl TtsEngine for SherpaOnnxEngine { volume: f32, mut on_audio: Option, mut on_boundary: Option, + _on_mark: Option, ) -> TtsResult<()> { if self.loaded_model_id.is_empty() { return Err(TtsError( @@ -712,8 +713,18 @@ impl TtsEngine for SherpaOnnxEngine { volume: f32, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()> { - self.speak(text, voice, rate, pitch, volume, on_audio, on_boundary) + self.speak( + text, + voice, + rate, + pitch, + volume, + on_audio, + on_boundary, + on_mark, + ) } fn stop(&self) -> TtsResult<()> { @@ -2363,7 +2374,7 @@ mod tests { fn test_engine_speak_without_model_id_errors_clearly() { let engine = SherpaOnnxEngine::new(""); let err = engine - .speak("hi", None, 1.0, 1.0, 1.0, None, None) + .speak("hi", None, 1.0, 1.0, 1.0, None, None, None) .unwrap_err(); assert!( err.to_string().contains("modelId"), @@ -2401,7 +2412,7 @@ mod tests { fn test_engine_speak_with_unknown_model_id_errors_with_count() { let engine = SherpaOnnxEngine::new(r#"{"modelId":"not-a-real-model"}"#); let err = engine - .speak("hi", None, 1.0, 1.0, 1.0, None, None) + .speak("hi", None, 1.0, 1.0, 1.0, None, None, None) .unwrap_err(); // Error message should hint at how many models ARE available so the // caller can pick a valid one. diff --git a/src/system_engine.rs b/src/system_engine.rs index 718db9d..fdeb216 100644 --- a/src/system_engine.rs +++ b/src/system_engine.rs @@ -44,6 +44,7 @@ impl TtsEngine for SystemEngine { volume: f32, _on_audio: Option, mut on_boundary: Option, + _on_mark: Option, ) -> TtsResult<()> { let guard = self.conn.lock().unwrap(); let conn = guard @@ -93,8 +94,18 @@ impl TtsEngine for SystemEngine { volume: f32, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()> { - self.speak(text, voice, rate, pitch, volume, on_audio, on_boundary) + self.speak( + text, + voice, + rate, + pitch, + volume, + on_audio, + on_boundary, + on_mark, + ) } fn stop(&self) -> TtsResult<()> { diff --git a/tests/sherpaonnx_live.rs b/tests/sherpaonnx_live.rs index 4bc5077..c336f64 100644 --- a/tests/sherpaonnx_live.rs +++ b/tests/sherpaonnx_live.rs @@ -143,6 +143,7 @@ fn vits_piper_synthesises_nonempty_audio() { 1.0, Some(&mut cb), None, + None, ) .expect("speak"); @@ -167,7 +168,7 @@ fn vits_piper_rate_changes_audio_size() { *fb.lock().unwrap() += c.len(); }; engine - .speak(text, None, 2.0, 1.0, 1.0, Some(&mut cb_fast), None) + .speak(text, None, 2.0, 1.0, 1.0, Some(&mut cb_fast), None, None) .unwrap(); let slow_bytes = Arc::new(Mutex::new(0usize)); @@ -176,7 +177,7 @@ fn vits_piper_rate_changes_audio_size() { *sb.lock().unwrap() += c.len(); }; engine - .speak(text, None, 0.5, 1.0, 1.0, Some(&mut cb_slow), None) + .speak(text, None, 0.5, 1.0, 1.0, Some(&mut cb_slow), None, None) .unwrap(); let fast = *fast_bytes.lock().unwrap(); @@ -223,6 +224,7 @@ fn vits_piper_volume_changes_amplitude() { volume, Some(&mut cb), None, + None, ) .unwrap(); let peak_value = *peak.lock().unwrap(); @@ -282,6 +284,7 @@ fn sherpa_streams_audio_per_sentence_batch() { seq_a.lock().unwrap().push(A); }), Some(&mut move |_w, _s, _e, _o, _l| seq_b.lock().unwrap().push(B)), + None, ) .expect("speak"); @@ -322,7 +325,7 @@ fn vits_piper_word_boundaries_fire_per_word() { }; let text = "one two three four five"; engine - .speak(text, None, 1.0, 1.0, 1.0, None, Some(&mut bound_cb)) + .speak(text, None, 1.0, 1.0, 1.0, None, Some(&mut bound_cb), None) .expect("speak with boundaries"); let words = sink.lock().unwrap().words.clone(); @@ -364,7 +367,7 @@ fn vits_piper_streaming_vs_buffered_match() { *s.lock().unwrap() += c.len(); }; engine - .speak(text, None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak(text, None, 1.0, 1.0, 1.0, Some(&mut cb), None, None) .expect("speak"); let streamed = *streamed.lock().unwrap(); @@ -407,6 +410,7 @@ fn vits_piper_multi_speaker_voice_id_selectable() { 1.0, Some(&mut cb), None, + None, ) .expect("speak with speaker id"); assert!( @@ -437,6 +441,7 @@ fn matcha_synthesises_nonempty_audio() { 1.0, Some(&mut cb), None, + None, ) .expect("matcha speak"); assert!(*total.lock().unwrap() > 0, "matcha produced no audio"); @@ -453,7 +458,16 @@ fn matcha_word_boundaries_fire() { s.lock().unwrap().words.push((w.into(), st, e)); }; engine - .speak("one two three", None, 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak( + "one two three", + None, + 1.0, + 1.0, + 1.0, + None, + Some(&mut cb), + None, + ) .expect("matcha speak"); assert_eq!(sink.lock().unwrap().words.len(), 3); } @@ -480,6 +494,7 @@ fn kokoro_synthesises_nonempty_audio() { 1.0, Some(&mut cb), None, + None, ) .expect("kokoro speak"); assert!(*total.lock().unwrap() > 0, "kokoro produced no audio"); @@ -526,6 +541,7 @@ fn supertonic_synthesises_nonempty_audio() { 1.0, Some(&mut cb), None, + None, ) .expect("supertonic speak"); assert!(*total.lock().unwrap() > 0, "supertonic produced no audio"); @@ -562,7 +578,7 @@ fn supertonic_switches_language_via_voice_id() { *t.lock().unwrap() += c.len(); }; engine - .speak(text, Some(voice), 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak(text, Some(voice), 1.0, 1.0, 1.0, Some(&mut cb), None, None) .unwrap_or_else(|e| panic!("speak {voice} failed: {e}")); assert!( *total.lock().unwrap() > 0, @@ -595,6 +611,7 @@ fn speechmarkdown_input_does_not_break_synthesis() { 1.0, Some(&mut cb), None, + None, ) .expect("speak with speechmarkdown input"); assert!(*total.lock().unwrap() > 0); @@ -628,7 +645,7 @@ fn pitch_shift_changes_sample_count() { *nc.lock().unwrap() += c.len(); }; engine - .speak(text, None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak(text, None, 1.0, 1.0, 1.0, Some(&mut cb), None, None) .unwrap(); let v = *n.lock().unwrap(); v @@ -641,7 +658,7 @@ fn pitch_shift_changes_sample_count() { *sc.lock().unwrap() += c.len(); }; engine - .speak(text, None, 1.0, 2.0, 1.0, Some(&mut cb), None) + .speak(text, None, 1.0, 2.0, 1.0, Some(&mut cb), None, None) .unwrap(); let v = *s.lock().unwrap(); v @@ -682,6 +699,7 @@ fn zipvoice_clones_bundled_reference_voice() { 1.0, Some(&mut cb), None, + None, ) .expect("zipvoice speak"); assert!(*total.lock().unwrap() > 0, "zipvoice produced no audio"); @@ -716,7 +734,16 @@ fn zipvoice_reference_transcript_override() { *t.lock().unwrap() += c.len(); }; engine - .speak("Override check.", None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak( + "Override check.", + None, + 1.0, + 1.0, + 1.0, + Some(&mut cb), + None, + None, + ) .expect("zipvoice speak with overridden reference"); assert!(*total.lock().unwrap() > 0); } @@ -741,6 +768,7 @@ fn pocket_synthesises_with_bundled_reference() { 1.0, Some(&mut cb), None, + None, ) .expect("pocket speak"); assert!(*total.lock().unwrap() > 0, "pocket produced no audio"); diff --git a/tests/trait_methods.rs b/tests/trait_methods.rs index 1c4f6e7..34fee48 100644 --- a/tests/trait_methods.rs +++ b/tests/trait_methods.rs @@ -290,7 +290,7 @@ fn speak_with_options_threads_options_to_engine() { // We don't assert on the Result — either it succeeds (network) or it // fails (no network / 401). The contract is that the option plumbing // doesn't panic. - let _ = e.speak_with_options("hi", Some(&opts), None, None); + let _ = e.speak_with_options("hi", Some(&opts), None, None, None); } // ===== synth_with_boundaries returns (audio, boundaries) of the right shape ===== From ee0651b180015c3652d392917bbb5aa20552fe94 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 20:49:45 +0000 Subject: [PATCH 2/5] fix: avsynth/sapi engines + newer-toolchain clippy lints --- src/avsynth_engine.rs | 12 +++++++++++- src/sapi_engine.rs | 12 +++++++++++- src/sherpaonnx_engine.rs | 8 +++++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/avsynth_engine.rs b/src/avsynth_engine.rs index d076dad..d8b4bb6 100644 --- a/src/avsynth_engine.rs +++ b/src/avsynth_engine.rs @@ -133,8 +133,18 @@ impl TtsEngine for AvSynthEngine { volume: f32, on_audio: Option, on_boundary: Option, + _on_mark: Option, ) -> TtsResult<()> { - self.speak(text, voice, rate, pitch, volume, on_audio, on_boundary) + self.speak( + text, + voice, + rate, + pitch, + volume, + on_audio, + on_boundary, + _on_mark, + ) } fn stop(&self) -> TtsResult<()> { diff --git a/src/sapi_engine.rs b/src/sapi_engine.rs index a11f986..42c93e5 100644 --- a/src/sapi_engine.rs +++ b/src/sapi_engine.rs @@ -404,6 +404,7 @@ impl TtsEngine for SapiEngine { volume: f32, _on_audio: Option, on_boundary: Option, + _on_mark: Option, ) -> TtsResult<()> { let mut guard = self.voice.lock().unwrap(); let sp_voice = guard @@ -466,7 +467,16 @@ impl TtsEngine for SapiEngine { on_audio: Option, on_boundary: Option, ) -> TtsResult<()> { - self.speak(text, voice, rate, pitch, volume, on_audio, on_boundary) + self.speak( + text, + voice, + rate, + pitch, + volume, + on_audio, + on_boundary, + _on_mark, + ) } /// Render speech to a WAV byte buffer instead of playing it. diff --git a/src/sherpaonnx_engine.rs b/src/sherpaonnx_engine.rs index 1775fa3..a947548 100644 --- a/src/sherpaonnx_engine.rs +++ b/src/sherpaonnx_engine.rs @@ -921,7 +921,7 @@ fn resolve_model_scan_dir(dir: &std::path::Path) -> std::path::PathBuf { || dir.join("model.onnx").exists() || dir.join("voices.bin").exists() || dir.join("espeak-ng-data").exists() - || std::fs::read_dir(dir).ok().is_some_and(|entries| { + || std::fs::read_dir(dir).is_ok_and(|entries| { entries .filter_map(Result::ok) .any(|e| e.path().extension().is_some_and(|ext| ext == "onnx")) @@ -1269,8 +1269,10 @@ fn read_wav_mono_16bit(path: &std::path::Path) -> TtsResult<(Vec, i32)> { b"data" => { let end = (body + len).min(bytes.len()); samples = bytes[body..end] - .chunks_exact(2) - .map(|c| f32::from(i16::from_le_bytes([c[0], c[1]])) / 32768.0) + .as_chunks::<2>() + .0 + .iter() + .map(|c| f32::from(i16::from_le_bytes(*c)) / 32768.0) .collect(); } _ => {} From 5c6b35bde9f8a5f8f07af8e4b0204f553dd79d12 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 20:55:27 +0000 Subject: [PATCH 3/5] fix: avsynth speak signature (missed first impl) --- src/avsynth_engine.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/avsynth_engine.rs b/src/avsynth_engine.rs index d8b4bb6..e14824f 100644 --- a/src/avsynth_engine.rs +++ b/src/avsynth_engine.rs @@ -62,6 +62,7 @@ impl TtsEngine for AvSynthEngine { volume: f32, _on_audio: Option, mut on_boundary: Option, + _on_mark: Option, ) -> TtsResult<()> { let guard = self.handle.lock().unwrap(); if guard.is_null() { From 989eaaf9fa73a85988e43e63115f946456306950 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 21:00:42 +0000 Subject: [PATCH 4/5] fix: avsynth sync passes on_mark through (underscore binding) --- src/avsynth_engine.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/avsynth_engine.rs b/src/avsynth_engine.rs index e14824f..c68f8ed 100644 --- a/src/avsynth_engine.rs +++ b/src/avsynth_engine.rs @@ -134,7 +134,7 @@ impl TtsEngine for AvSynthEngine { volume: f32, on_audio: Option, on_boundary: Option, - _on_mark: Option, + on_mark: Option, ) -> TtsResult<()> { self.speak( text, @@ -144,7 +144,7 @@ impl TtsEngine for AvSynthEngine { volume, on_audio, on_boundary, - _on_mark, + on_mark, ) } From 9a13020431815d52942a48a188aa219060fb93b3 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 20 Aug 2026 21:06:37 +0000 Subject: [PATCH 5/5] fix: sapi speak_sync signature --- src/sapi_engine.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sapi_engine.rs b/src/sapi_engine.rs index 42c93e5..ef069d2 100644 --- a/src/sapi_engine.rs +++ b/src/sapi_engine.rs @@ -466,6 +466,7 @@ impl TtsEngine for SapiEngine { volume: f32, on_audio: Option, on_boundary: Option, + on_mark: Option, ) -> TtsResult<()> { self.speak( text, @@ -475,7 +476,7 @@ impl TtsEngine for SapiEngine { volume, on_audio, on_boundary, - _on_mark, + on_mark, ) }