From f19196ba781f0151564ad753ad838af1ff047226 Mon Sep 17 00:00:00 2001 From: ark Date: Fri, 30 Oct 2026 17:42:54 +0100 Subject: [PATCH] feat: type words while speaking, correct them at the pause --- src-tauri/examples/dictation_eval.rs | 44 ++++- src-tauri/src/commands.rs | 34 +++- src-tauri/src/paste/macos.rs | 4 + src-tauri/src/paste/mod.rs | 13 +- src-tauri/src/paste/wayland.rs | 6 +- src-tauri/src/paste/win.rs | 9 + src-tauri/src/pipeline.rs | 140 +++++++++++--- src-tauri/src/portal.rs | 10 + src-tauri/src/text.rs | 279 ++++++++++++++++++++++++--- src-tauri/src/x11.rs | 4 + 10 files changed, 473 insertions(+), 70 deletions(-) diff --git a/src-tauri/examples/dictation_eval.rs b/src-tauri/examples/dictation_eval.rs index 1e510b1..c292444 100644 --- a/src-tauri/examples/dictation_eval.rs +++ b/src-tauri/examples/dictation_eval.rs @@ -4,6 +4,7 @@ //! LANGUAGE is da or en; PUNCT_DIR none keeps whisper's own marks. //! A case is `reference \t audio`: `|` separates phrases, `+` joins clips into one phrase //! with a pause between them. Paths are relative to CASES.tsv, `\n` in a reference is a line break. +//! STREAM=ms also previews each phrase every ms of its audio, typing what previews agree on. #[path = "common/wav.rs"] mod wav; @@ -136,6 +137,12 @@ fn main() -> Result<(), String> { let (mut cases, mut exact, mut lost) = (0, 0, 0); let (mut word_errors, mut ref_words, mut mark_errors, mut ref_marks) = (0, 0, 0, 0); let (mut phrases, mut seconds) = (0, 0.0); + let stream = match std::env::var("STREAM") { + Ok(ms) => Some(ms.parse::().map_err(|e| format!("STREAM: {e}"))? * 16), + Err(_) => None, + }; + let (mut previews, mut preview_seconds, mut early, mut typed_chars) = (0, 0.0, 0, 0); + let mut corrected = 0; // true positives, false positives, misses: commas, sentence ends, capitals let mut punct = [[0usize; 3]; 3]; for row in rows.lines().filter(|l| !l.is_empty()).take(n) { @@ -153,11 +160,7 @@ fn main() -> Result<(), String> { } pcm.extend(wav::read_16k_mono(&dir.join(clip).to_string_lossy())?); } - let t = Instant::now(); - let text = stt.transcribe(&pcm, joiner.prompt())?; - seconds += t.elapsed().as_secs_f32(); - phrases += 1; - let label = |words: &[String]| { + let mut label = |words: &[String]| { let Some(punctuator) = &punctuator else { return Vec::new(); }; @@ -169,11 +172,28 @@ fn main() -> Result<(), String> { punct_runs += 1; labels }; - joiner - .next(&text, i + 1 == parts.len(), true, label) - .apply(&mut doc); + for end in stream + .into_iter() + .flat_map(|step| (step..pcm.len()).step_by(step)) + { + let t = Instant::now(); + let text = stt.transcribe(&pcm[..end], joiner.prompt())?; + preview_seconds += t.elapsed().as_secs_f32(); + previews += 1; + let (edit, _) = joiner.heard(&text, &mut label); + early += edit.text.chars().count(); + edit.apply(&mut doc); + } + let t = Instant::now(); + let text = stt.transcribe(&pcm, joiner.prompt())?; + seconds += t.elapsed().as_secs_f32(); + phrases += 1; + let edit = joiner.next(&text, i + 1 == parts.len(), true, &mut label); + corrected += edit.delete; + edit.apply(&mut doc); heard.push(text); } + typed_chars += doc.chars().count(); let typed = doc.trim_end(); let (want, got) = (words(&reference), words(typed)); let word_err = distance(&want, &got); @@ -231,6 +251,14 @@ fn main() -> Result<(), String> { pct(tp, miss) ); } + if stream.is_some() { + println!( + "previews: {:.2}s each, {:.0}% of the text typed before its pause, {:.0}% of that deleted again", + preview_seconds / previews.max(1) as f32, + 100.0 * early as f32 / typed_chars.max(1) as f32, + 100.0 * corrected as f32 / early.max(1) as f32 + ); + } println!( "punctuation: {:.0} ms per run", 1000.0 * punct_seconds / punct_runs.max(1) as f32 diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index e989bb9..728b847 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -34,12 +34,15 @@ pub enum Action { /// A command's phrasings, in lowercase, and what it does. pub type Command = (&'static [&'static str], Action); -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Item { Word(String), Do(Action), } +// a phrase this short, said alone, may be a misheard command +const MISHEARD_WORDS: usize = 3; + /// Splits a transcribed phrase into words and commands. Editing commands count only /// in a phrase of commands alone, so "hun kom slet ikke" stays text. pub fn parse(phrase: &str, profile: &Profile) -> Vec { @@ -57,6 +60,24 @@ pub fn parse(phrase: &str, profile: &Profile) -> Vec { scan(&words, false, commands) } +/// Whether a phrase still being spoken is plainly text, so its words can be typed before +/// it ends: too long to be a misheard command, and not commands alone. +pub fn is_text(phrase: &str, profile: &Profile) -> bool { + let text = plain(phrase, profile); + let words: Vec<&str> = text.split_whitespace().collect(); + words.len() > MISHEARD_WORDS + && scan(&words, true, profile.commands) + .iter() + .any(|item| matches!(item, Item::Word(_))) +} + +/// Splits a phrase that is text, where an editing command is a word. +pub fn parse_text(phrase: &str, profile: &Profile) -> Vec { + let text = plain(phrase, profile); + let words: Vec<&str> = text.split_whitespace().collect(); + scan(&words, false, profile.commands) +} + fn scan(words: &[&str], editing: bool, commands: &'static [Command]) -> Vec { let mut items = Vec::new(); let mut rest = words; @@ -114,7 +135,7 @@ fn command_at(words: &[&str], commands: &'static [Command]) -> Option<(usize, Ac /// A short phrase said alone may be a misheard command, such as "nye linjer". fn misheard(words: &[&str], commands: &'static [Command]) -> Option { - if words.is_empty() || words.len() > 3 { + if words.is_empty() || words.len() > MISHEARD_WORDS { return None; } let heard: Vec = words @@ -136,13 +157,18 @@ fn misheard(words: &[&str], commands: &'static [Command]) -> Option { .map(|(_, action)| action) } -fn core(word: &str) -> String { +pub fn core(word: &str) -> String { word.trim_matches(|c: char| !c.is_alphanumeric()) .to_lowercase() } /// Levenshtein distance between two sequences. pub fn distance(a: &[T], b: &[T]) -> usize { + distances(a, b)[b.len()] +} + +/// Levenshtein distance between `a` and each start of `b`, by its length. +pub fn distances(a: &[T], b: &[T]) -> Vec { let mut row: Vec = (0..=b.len()).collect(); for (i, x) in a.iter().enumerate() { let mut prev = row[0]; @@ -153,7 +179,7 @@ pub fn distance(a: &[T], b: &[T]) -> usize { prev = cur; } } - row[b.len()] + row } /// The phrase's words, edda's tags repaired first, with each split compound joined again. diff --git a/src-tauri/src/paste/macos.rs b/src-tauri/src/paste/macos.rs index 3081e14..7bd045a 100644 --- a/src-tauri/src/paste/macos.rs +++ b/src-tauri/src/paste/macos.rs @@ -30,6 +30,10 @@ impl Keys { pub fn backspace(&mut self) -> Result<(), String> { tap(BACKSPACE, CGEventFlags::empty()) } + + pub fn delete_word(&mut self) -> Result<(), String> { + tap(BACKSPACE, CGEventFlags::MaskAlternate) + } } fn tap(key: CGKeyCode, flags: CGEventFlags) -> Result<(), String> { diff --git a/src-tauri/src/paste/mod.rs b/src-tauri/src/paste/mod.rs index 27964b1..e3deb05 100644 --- a/src-tauri/src/paste/mod.rs +++ b/src-tauri/src/paste/mod.rs @@ -58,6 +58,14 @@ impl Keys { Self::Portal(keys) => keys.backspace(), } } + + fn delete_word(&mut self) -> Result<(), String> { + match self { + Self::Wayland(keys) => keys.delete_word(), + Self::X11(keys) => keys.delete_word(), + Self::Portal(keys) => keys.delete_word(), + } + } } use crate::text::Edit; @@ -95,13 +103,16 @@ impl Paster { /// Returns false when the text is only on the clipboard and the user must paste it. pub fn apply(&mut self, edit: &Edit) -> Result { self.find_keys(); - if edit.delete > 0 { + if edit.delete > 0 || edit.delete_words > 0 { let Some(keys) = &mut self.keys else { return Ok(false); }; for _ in 0..edit.delete { keys.backspace()?; } + for _ in 0..edit.delete_words { + keys.delete_word()?; + } } match edit.text.as_str() { "" => Ok(true), diff --git a/src-tauri/src/paste/wayland.rs b/src-tauri/src/paste/wayland.rs index f957c42..66801c0 100644 --- a/src-tauri/src/paste/wayland.rs +++ b/src-tauri/src/paste/wayland.rs @@ -1,4 +1,4 @@ -//! Ctrl+V, Enter and Backspace through the Wayland virtual keyboard protocol (wlroots compositors, niri, Hyprland). +//! Ctrl+V, Enter, Backspace and Ctrl+Backspace through the Wayland virtual keyboard protocol (wlroots compositors, niri, Hyprland). use std::io::Write; use std::os::fd::AsFd; @@ -74,6 +74,10 @@ impl Keys { self.tap(KEY_BACKSPACE, 0) } + pub fn delete_word(&mut self) -> Result<(), String> { + self.tap(KEY_BACKSPACE, CONTROL_MASK) + } + fn tap(&mut self, key: u32, modifiers: u32) -> Result<(), String> { let t = self.epoch.elapsed().as_millis() as u32; // same sequence as wtype: modifier state set directly, no Ctrl key events diff --git a/src-tauri/src/paste/win.rs b/src-tauri/src/paste/win.rs index a894dc4..fecbfb4 100644 --- a/src-tauri/src/paste/win.rs +++ b/src-tauri/src/paste/win.rs @@ -28,6 +28,15 @@ impl Keys { pub fn backspace(&mut self) -> Result<(), String> { send(&[(VK_BACK, true), (VK_BACK, false)]) } + + pub fn delete_word(&mut self) -> Result<(), String> { + send(&[ + (VK_CONTROL, true), + (VK_BACK, true), + (VK_BACK, false), + (VK_CONTROL, false), + ]) + } } fn send(keys: &[(VIRTUAL_KEY, bool)]) -> Result<(), String> { diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index d92612c..0b9c056 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -1,6 +1,6 @@ -//! Dictation: record until toggled off, cutting at pauses so each phrase is transcribed -//! and pasted while the user keeps speaking. One worker transcribes the phrases in order, -//! and one typist pastes them in that order. +//! Dictation: record until toggled off, typing each phrase's words as soon as two previews +//! agree on them, and the rest once a pause ends the phrase. One worker transcribes in +//! order, and one typist pastes in that order. use crate::audio::{RATE, Recorder}; use crate::models; @@ -25,6 +25,10 @@ const MAX_SILENCE: usize = 10 * RATE as usize; const SETTLE: usize = RATE as usize / 2; // whisper's window; a preview of a longer phrase shows only its end const PREVIEW_MAX: usize = 28 * RATE as usize; +// a phrase this long is cut where its last speech began, so previews keep seeing it whole +const LONG_PHRASE: usize = 20 * RATE as usize; +// silence after which a phrase's held mark is typed, as the user may edit next +const MARK_AFTER: Duration = Duration::from_secs(2); /// What the overlay shows. #[derive(Clone, serde::Serialize)] @@ -33,7 +37,7 @@ pub enum Event { Recording(bool), /// Input RMS, about 30 times a second while recording. Level(f32), - /// Text so far of the phrase being spoken; empty once it is written. + /// Text of the phrase being spoken that is not typed yet. Preview(String), /// Phrases waiting to be written. Pending(usize), @@ -42,12 +46,17 @@ pub enum Event { enum Job { Warm, - Preview(Vec), + /// `early`: words may be typed before the pause, as the audio starts where the phrase does. + Preview { + pcm: Vec, + early: bool, + }, /// `last` ends the dictation. Phrase { pcm: Vec, last: bool, }, + Settle, } struct Session { @@ -55,6 +64,7 @@ struct Session { pcm: Vec, phrases: usize, heard: Heard, + mark_due: Option, } /// Speech in a session's pcm, found by scanning only the audio that may still change. @@ -153,11 +163,11 @@ impl Pipeline { let vad = models.join(models::VAD.path); let segmenter_live = live.clone(); std::thread::spawn(move || segmenter(&segmenter_live, wake_rx, &vad)); - let (edits, edit_rx) = mpsc::channel(); + let (writes, write_rx) = mpsc::channel(); let typist_live = live.clone(); - std::thread::spawn(move || typist(&typist_live, edit_rx)); + std::thread::spawn(move || typist(&typist_live, write_rx)); let worker_live = live.clone(); - std::thread::spawn(move || worker(&worker_live, job_rx, edits, &models)); + std::thread::spawn(move || worker(&worker_live, job_rx, writes, &models)); Self { live, wake } } @@ -174,6 +184,7 @@ impl Pipeline { pcm: Vec::new(), phrases: 0, heard: Heard::default(), + mark_due: None, }); (self.live.on_event)(Event::Recording(true)); // load hides behind the speech @@ -241,10 +252,19 @@ fn segmenter(live: &Live, wake: mpsc::Receiver<()>, vad_path: &Path) { .then(|| phrase_end(session.heard.last.as_ref(), len, pause)) .flatten(); let speech_end = session.heard.last.as_ref().map(|last| last.end); + if speech_end.is_some() { + session.mark_due = None; + } else if session.mark_due.is_some_and(|due| Instant::now() >= due) { + session.mark_due = None; + let _ = live.jobs.send(Job::Settle); + } if let Some(at) = cut { live.queue_phrase(session.pcm.drain(..at).collect(), false); session.heard.drain(at); session.phrases += 1; + // the cut comes a pause after the speech + let pause = Duration::from_secs_f32(pause as f32 / RATE as f32); + session.mark_due = Some(Instant::now() + MARK_AFTER.saturating_sub(pause)); } else if speech_end.is_none() && len > MAX_SILENCE { session.pcm.drain(..len - RATE as usize); session.heard.drain(len - RATE as usize); @@ -257,20 +277,28 @@ fn segmenter(live: &Live, wake: mpsc::Receiver<()>, vad_path: &Path) { live.previewing.store(true, SeqCst); live.stale_preview.store(false, SeqCst); session.heard.previewed = speech_end; - let pcm = session.pcm[len.saturating_sub(PREVIEW_MAX)..].to_vec(); - let _ = live.jobs.send(Job::Preview(pcm)); + let from = len.saturating_sub(PREVIEW_MAX); + let _ = live.jobs.send(Job::Preview { + pcm: session.pcm[from..].to_vec(), + // pause 0 promises nothing typed before dictation stops + early: from == 0 && pause > 0, + }); } } } } -/// Cut point inside the pause after the last speech, once that pause is `pause` samples long. +/// Cut point inside the pause after the last speech, once that pause is `pause` samples long; +/// in a long phrase, where its last speech began after a shorter one. fn phrase_end(last: Option<&Range>, len: usize, pause: usize) -> Option { let last = last?; - (len - last.end >= pause).then_some(last.end + pause / 2) + if len - last.end >= pause { + return Some(last.end + pause / 2); + } + (len >= LONG_PHRASE && last.start >= LONG_PHRASE / 2).then_some(last.start) } -fn worker(live: &Live, rx: mpsc::Receiver, edits: mpsc::Sender, models: &Path) { +fn worker(live: &Live, rx: mpsc::Receiver, writes: mpsc::Sender, models: &Path) { let mut stt: Option = None; let mut punct: Option = None; let mut language = live.settings.lock().unwrap().language; @@ -303,9 +331,12 @@ fn worker(live: &Live, rx: mpsc::Receiver, edits: mpsc::Sender, model }; // a new language takes effect when a dictation starts, never inside one let wanted = live.settings.lock().unwrap().language; - if matches!(job, Job::Warm) && wanted != language { - log::info!("dictating in {wanted:?} from now on"); - (stt, punct, language) = (None, None, wanted); + if matches!(job, Job::Warm) { + if wanted != language { + log::info!("dictating in {wanted:?} from now on"); + (stt, punct, language) = (None, None, wanted); + } + // the page may have changed since the last dictation, so nothing typed then is trusted joiner = Joiner::new(language.profile()); } if stt.is_none() { @@ -328,19 +359,43 @@ fn worker(live: &Live, rx: mpsc::Receiver, edits: mpsc::Sender, model let Some(model) = stt.as_mut() else { match job { Job::Warm => live.notice(text(Text::ModelFailed)), - Job::Preview(_) => live.previewing.store(false, SeqCst), + Job::Preview { .. } => live.previewing.store(false, SeqCst), Job::Phrase { .. } => live.phrase_done(), + Job::Settle => {} } continue; }; match job { Job::Warm => {} - Job::Preview(pcm) => { - match model.transcribe_unless(&pcm, joiner.prompt(), &live.stale_preview) { - // a preview finishing after dictation stopped is stale - Ok(text) if live.session.lock().unwrap().is_some() => { - let preview = joiner.preview(&text, |w| labels(&punct, w)); - (live.on_event)(Event::Preview(preview)); + Job::Settle => { + let _ = writes.send(Write { + edit: joiner.settle(), + preview: Some(String::new()), + }); + } + Job::Preview { pcm, early } => { + let t = Instant::now(); + let result = model.transcribe_unless(&pcm, joiner.prompt(), &live.stale_preview); + log::info!( + "previewed {:.1}s in {:.2}s", + pcm.len() as f32 / RATE as f32, + t.elapsed().as_secs_f32() + ); + let current = + live.session.lock().unwrap().is_some() && !live.stale_preview.load(SeqCst); + match result { + // a preview finishing after its phrase ended or dictation stopped is stale + Ok(text) if current => { + let label = |w: &[String]| labels(&punct, w); + let (edit, preview) = match early { + true => joiner.heard(&text, label), + // ponytail: speech with no gap past whisper's window types only at the pause + false => (Edit::default(), joiner.preview(&text, label)), + }; + let _ = writes.send(Write { + edit, + preview: Some(preview), + }); } Ok(_) => {} Err(_) if live.stale_preview.load(SeqCst) => {} @@ -367,18 +422,27 @@ fn worker(live: &Live, rx: mpsc::Receiver, edits: mpsc::Sender, model kept = None; } } - let _ = edits.send(edit); + let _ = writes.send(Write { + edit, + preview: None, + }); } } } } -/// Pastes each phrase's edit, apart from the worker: a paste waits for the app to fetch the clipboard. -fn typist(live: &Live, edits: mpsc::Receiver) { +/// An edit for the typist, then the preview to show; none ends a phrase. +struct Write { + edit: Edit, + preview: Option, +} + +/// Pastes each edit, apart from the worker: a paste waits for the app to fetch the clipboard. +fn typist(live: &Live, writes: mpsc::Receiver) { let mut paster = Paster::new() .inspect_err(|e| log::error!("clipboard: {e}")) .ok(); - for edit in edits { + for Write { edit, preview } in writes { if !edit.is_empty() { match paster.as_mut().map(|p| p.apply(&edit)) { Some(Ok(true)) => {} @@ -393,8 +457,11 @@ fn typist(live: &Live, edits: mpsc::Receiver) { } } } - (live.on_event)(Event::Preview(String::new())); - live.phrase_done(); + let done = preview.is_none(); + (live.on_event)(Event::Preview(preview.unwrap_or_default())); + if done { + live.phrase_done(); + } } } @@ -457,4 +524,19 @@ mod tests { assert_eq!(phrase_end(Some(&last), 5800, 800), Some(5400)); assert_eq!(phrase_end(None, 99_999, 800), None); } + + #[test] + fn cuts_a_long_phrase_where_its_last_speech_began() { + let last = LONG_PHRASE / 2..LONG_PHRASE - 100; + assert_eq!(phrase_end(Some(&last), LONG_PHRASE - 1, 800), None); + assert_eq!( + phrase_end(Some(&last), LONG_PHRASE, 800), + Some(LONG_PHRASE / 2) + ); + // one long stretch of speech has nowhere to cut + assert_eq!( + phrase_end(Some(&(100..LONG_PHRASE)), LONG_PHRASE, 800), + None + ); + } } diff --git a/src-tauri/src/portal.rs b/src-tauri/src/portal.rs index 411e341..5a55ee3 100644 --- a/src-tauri/src/portal.rs +++ b/src-tauri/src/portal.rs @@ -232,6 +232,16 @@ impl Keys { pub fn backspace(&mut self) -> Result<(), String> { self.tap(crate::x11::BACKSPACE) } + + pub fn delete_word(&mut self) -> Result<(), String> { + use crate::x11::{BACKSPACE, CONTROL_L}; + self.keys(&[ + (CONTROL_L, KeyState::Pressed), + (BACKSPACE, KeyState::Pressed), + (BACKSPACE, KeyState::Released), + (CONTROL_L, KeyState::Released), + ]) + } } #[cfg(test)] diff --git a/src-tauri/src/text.rs b/src-tauri/src/text.rs index 80bfb54..396ff80 100644 --- a/src-tauri/src/text.rs +++ b/src-tauri/src/text.rs @@ -12,16 +12,19 @@ const KEEP_TYPED: usize = 4096; // words before a phrase that the punctuation model sees const CONTEXT_WORDS: usize = 48; -/// Delete `delete` characters before the cursor, then type `text`. -#[derive(Debug, PartialEq)] +/// Delete `delete` characters before the cursor, then `delete_words` words with the app's +/// own word-delete key, then type `text`. +#[derive(Debug, Default, PartialEq)] pub struct Edit { pub delete: usize, + /// words from before this dictation, which only the app knows + pub delete_words: usize, pub text: String, } impl Edit { pub fn is_empty(&self) -> bool { - self.delete == 0 && self.text.is_empty() + self.delete == 0 && self.delete_words == 0 && self.text.is_empty() } /// What the edit does to `doc`, the text before the cursor. @@ -29,6 +32,13 @@ impl Edit { for _ in 0..self.delete { doc.pop(); } + for _ in 0..self.delete_words { + // ponytail: apps differ on the marks and spaces around a word; this takes them all + let rest = doc + .trim_end() + .trim_end_matches(|c: char| !c.is_whitespace()); + doc.truncate(rest.len()); + } doc.push_str(&self.text); } } @@ -42,6 +52,8 @@ enum Tail { /// a word whose natural mark is typed only once the next word comes: a spoken mark /// replaces it, and at a phrase's end the next phrase gives the model its right context Held(Option), + /// a word whose held mark a long pause typed; the page may have changed after it + Settled, Spoken(Fit), } @@ -56,6 +68,12 @@ pub struct Joiner { // during a phrase: typed is unchanged before low, and this many characters went from there low: usize, deleted: usize, + deleted_words: usize, + // the phrase being spoken: its items typed already, and its last preview + spoken: Vec, + previewed: Vec, + // before the phrase's first early word; its pause types the whole phrase from here again + before_phrase: Option>, } impl Joiner { @@ -67,6 +85,10 @@ impl Joiner { typed: String::new(), low: 0, deleted: 0, + deleted_words: 0, + spoken: Vec::new(), + previewed: Vec::new(), + before_phrase: None, } } @@ -74,7 +96,8 @@ impl Joiner { &self.context } - /// The edit for the next `phrase` of the dictation; `last` ends the dictation. + /// The edit for the next `phrase` of the dictation; `last` ends the dictation. Words + /// `heard` typed early are corrected where the whole phrase heard them differently. /// `punct` labels words, the ones typed before first; no labels keeps whisper's marks. pub fn next( &mut self, @@ -82,17 +105,112 @@ impl Joiner { last: bool, trailing_space: bool, punct: impl FnOnce(&[String]) -> Vec