Type while speaking

src-tauri/examples/dictation_eval.rs

@@ -4,6 +4,7 @@
 //! LANGUAGE is da or en; PUNCT_DIR none keeps whisper's own marks.
 //! A case is `reference \t audio`: `|` separates phrases, `+` joins clips into one phrase
 //! with a pause between them. Paths are relative to CASES.tsv, `\n` in a reference is a line break.
+//! STREAM=ms also previews each phrase every ms of its audio, typing what previews agree on.
 
 #[path = "common/wav.rs"]
 mod wav;
@@ -136,6 +137,12 @@ fn main() -> Result<(), String> {
     let (mut cases, mut exact, mut lost) = (0, 0, 0);
     let (mut word_errors, mut ref_words, mut mark_errors, mut ref_marks) = (0, 0, 0, 0);
     let (mut phrases, mut seconds) = (0, 0.0);
+    let stream = match std::env::var("STREAM") {
+        Ok(ms) => Some(ms.parse::<usize>().map_err(|e| format!("STREAM: {e}"))? * 16),
+        Err(_) => None,
+    };
+    let (mut previews, mut preview_seconds, mut early, mut typed_chars) = (0, 0.0, 0, 0);
+    let mut corrected = 0;
     // true positives, false positives, misses: commas, sentence ends, capitals
     let mut punct = [[0usize; 3]; 3];
     for row in rows.lines().filter(|l| !l.is_empty()).take(n) {
@@ -153,11 +160,7 @@ fn main() -> Result<(), String> {
                 }
                 pcm.extend(wav::read_16k_mono(&dir.join(clip).to_string_lossy())?);
             }
-            let t = Instant::now();
-            let text = stt.transcribe(&pcm, joiner.prompt())?;
-            seconds += t.elapsed().as_secs_f32();
-            phrases += 1;
-            let label = |words: &[String]| {
+            let mut label = |words: &[String]| {
                 let Some(punctuator) = &punctuator else {
                     return Vec::new();
                 };
@@ -169,11 +172,28 @@ fn main() -> Result<(), String> {
                 punct_runs += 1;
                 labels
             };
-            joiner
-                .next(&text, i + 1 == parts.len(), true, label)
-                .apply(&mut doc);
+            for end in stream
+                .into_iter()
+                .flat_map(|step| (step..pcm.len()).step_by(step))
+            {
+                let t = Instant::now();
+                let text = stt.transcribe(&pcm[..end], joiner.prompt())?;
+                preview_seconds += t.elapsed().as_secs_f32();
+                previews += 1;
+                let (edit, _) = joiner.heard(&text, &mut label);
+                early += edit.text.chars().count();
+                edit.apply(&mut doc);
+            }
+            let t = Instant::now();
+            let text = stt.transcribe(&pcm, joiner.prompt())?;
+            seconds += t.elapsed().as_secs_f32();
+            phrases += 1;
+            let edit = joiner.next(&text, i + 1 == parts.len(), true, &mut label);
+            corrected += edit.delete;
+            edit.apply(&mut doc);
             heard.push(text);
         }
+        typed_chars += doc.chars().count();
         let typed = doc.trim_end();
         let (want, got) = (words(&reference), words(typed));
         let word_err = distance(&want, &got);
@@ -231,6 +251,14 @@ fn main() -> Result<(), String> {
             pct(tp, miss)
         );
     }
+    if stream.is_some() {
+        println!(
+            "previews: {:.2}s each, {:.0}% of the text typed before its pause, {:.0}% of that deleted again",
+            preview_seconds / previews.max(1) as f32,
+            100.0 * early as f32 / typed_chars.max(1) as f32,
+            100.0 * corrected as f32 / early.max(1) as f32
+        );
+    }
     println!(
         "punctuation: {:.0} ms per run",
         1000.0 * punct_seconds / punct_runs.max(1) as f32

src-tauri/src/commands.rs

@@ -34,12 +34,15 @@ pub enum Action {
 /// A command's phrasings, in lowercase, and what it does.
 pub type Command = (&'static [&'static str], Action);
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Item {
     Word(String),
     Do(Action),
 }
 
+// a phrase this short, said alone, may be a misheard command
+const MISHEARD_WORDS: usize = 3;
+
 /// Splits a transcribed phrase into words and commands. Editing commands count only
 /// in a phrase of commands alone, so "hun kom slet ikke" stays text.
 pub fn parse(phrase: &str, profile: &Profile) -> Vec<Item> {
@@ -57,6 +60,24 @@ pub fn parse(phrase: &str, profile: &Profile) -> Vec<Item> {
     scan(&words, false, commands)
 }
 
+/// Whether a phrase still being spoken is plainly text, so its words can be typed before
+/// it ends: too long to be a misheard command, and not commands alone.
+pub fn is_text(phrase: &str, profile: &Profile) -> bool {
+    let text = plain(phrase, profile);
+    let words: Vec<&str> = text.split_whitespace().collect();
+    words.len() > MISHEARD_WORDS
+        && scan(&words, true, profile.commands)
+            .iter()
+            .any(|item| matches!(item, Item::Word(_)))
+}
+
+/// Splits a phrase that is text, where an editing command is a word.
+pub fn parse_text(phrase: &str, profile: &Profile) -> Vec<Item> {
+    let text = plain(phrase, profile);
+    let words: Vec<&str> = text.split_whitespace().collect();
+    scan(&words, false, profile.commands)
+}
+
 fn scan(words: &[&str], editing: bool, commands: &'static [Command]) -> Vec<Item> {
     let mut items = Vec::new();
     let mut rest = words;
@@ -114,7 +135,7 @@ fn command_at(words: &[&str], commands: &'static [Command]) -> Option<(usize, Ac
 
 /// A short phrase said alone may be a misheard command, such as "nye linjer".
 fn misheard(words: &[&str], commands: &'static [Command]) -> Option<Action> {
-    if words.is_empty() || words.len() > 3 {
+    if words.is_empty() || words.len() > MISHEARD_WORDS {
         return None;
     }
     let heard: Vec<char> = words
@@ -136,13 +157,18 @@ fn misheard(words: &[&str], commands: &'static [Command]) -> Option<Action> {
         .map(|(_, action)| action)
 }
 
-fn core(word: &str) -> String {
+pub fn core(word: &str) -> String {
     word.trim_matches(|c: char| !c.is_alphanumeric())
         .to_lowercase()
 }
 
 /// Levenshtein distance between two sequences.
 pub fn distance<T: PartialEq>(a: &[T], b: &[T]) -> usize {
+    distances(a, b)[b.len()]
+}
+
+/// Levenshtein distance between `a` and each start of `b`, by its length.
+pub fn distances<T: PartialEq>(a: &[T], b: &[T]) -> Vec<usize> {
     let mut row: Vec<usize> = (0..=b.len()).collect();
     for (i, x) in a.iter().enumerate() {
         let mut prev = row[0];
@@ -153,7 +179,7 @@ pub fn distance<T: PartialEq>(a: &[T], b: &[T]) -> usize {
             prev = cur;
         }
     }
-    row[b.len()]
+    row
 }
 
 /// The phrase's words, edda's tags repaired first, with each split compound joined again.

src-tauri/src/paste/macos.rs

@@ -30,6 +30,10 @@ impl Keys {
     pub fn backspace(&mut self) -> Result<(), String> {
         tap(BACKSPACE, CGEventFlags::empty())
     }
+
+    pub fn delete_word(&mut self) -> Result<(), String> {
+        tap(BACKSPACE, CGEventFlags::MaskAlternate)
+    }
 }
 
 fn tap(key: CGKeyCode, flags: CGEventFlags) -> Result<(), String> {

src-tauri/src/paste/mod.rs

@@ -58,6 +58,14 @@ impl Keys {
             Self::Portal(keys) => keys.backspace(),
         }
     }
+
+    fn delete_word(&mut self) -> Result<(), String> {
+        match self {
+            Self::Wayland(keys) => keys.delete_word(),
+            Self::X11(keys) => keys.delete_word(),
+            Self::Portal(keys) => keys.delete_word(),
+        }
+    }
 }
 
 use crate::text::Edit;
@@ -95,13 +103,16 @@ impl Paster {
     /// Returns false when the text is only on the clipboard and the user must paste it.
     pub fn apply(&mut self, edit: &Edit) -> Result<bool, String> {
         self.find_keys();
-        if edit.delete > 0 {
+        if edit.delete > 0 || edit.delete_words > 0 {
             let Some(keys) = &mut self.keys else {
                 return Ok(false);
             };
             for _ in 0..edit.delete {
                 keys.backspace()?;
             }
+            for _ in 0..edit.delete_words {
+                keys.delete_word()?;
+            }
         }
         match edit.text.as_str() {
             "" => Ok(true),

src-tauri/src/paste/wayland.rs

@@ -1,4 +1,4 @@
-//! Ctrl+V, Enter and Backspace through the Wayland virtual keyboard protocol (wlroots compositors, niri, Hyprland).
+//! Ctrl+V, Enter, Backspace and Ctrl+Backspace through the Wayland virtual keyboard protocol (wlroots compositors, niri, Hyprland).
 
 use std::io::Write;
 use std::os::fd::AsFd;
@@ -74,6 +74,10 @@ impl Keys {
         self.tap(KEY_BACKSPACE, 0)
     }
 
+    pub fn delete_word(&mut self) -> Result<(), String> {
+        self.tap(KEY_BACKSPACE, CONTROL_MASK)
+    }
+
     fn tap(&mut self, key: u32, modifiers: u32) -> Result<(), String> {
         let t = self.epoch.elapsed().as_millis() as u32;
         // same sequence as wtype: modifier state set directly, no Ctrl key events

src-tauri/src/paste/win.rs

@@ -28,6 +28,15 @@ impl Keys {
     pub fn backspace(&mut self) -> Result<(), String> {
         send(&[(VK_BACK, true), (VK_BACK, false)])
     }
+
+    pub fn delete_word(&mut self) -> Result<(), String> {
+        send(&[
+            (VK_CONTROL, true),
+            (VK_BACK, true),
+            (VK_BACK, false),
+            (VK_CONTROL, false),
+        ])
+    }
 }
 
 fn send(keys: &[(VIRTUAL_KEY, bool)]) -> Result<(), String> {