From b718ad2a2b5d70894316601a43342e43b370614a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:58:57 -0500 Subject: [PATCH 1/4] feat(desktop-gpui): download captions as SRT or WebVTT --- apps/desktop-gpui/locales/de.toml | 5 + apps/desktop-gpui/locales/en.toml | 5 + apps/desktop-gpui/locales/es.toml | 5 + apps/desktop-gpui/locales/fr.toml | 5 + apps/desktop-gpui/locales/hi.toml | 5 + apps/desktop-gpui/locales/ja.toml | 5 + apps/desktop-gpui/locales/zh-CN.toml | 5 + apps/desktop-gpui/src/caption_export.rs | 311 +++++++++++++++++++++++ apps/desktop-gpui/src/editor_export.rs | 2 +- apps/desktop-gpui/src/editor_window.rs | 77 ++++++ apps/desktop-gpui/src/main.rs | 1 + apps/desktop-gpui/src/transcript/view.rs | 17 ++ 12 files changed, 442 insertions(+), 1 deletion(-) create mode 100644 apps/desktop-gpui/src/caption_export.rs diff --git a/apps/desktop-gpui/locales/de.toml b/apps/desktop-gpui/locales/de.toml index c96d36f22a2..6501bcf4a04 100644 --- a/apps/desktop-gpui/locales/de.toml +++ b/apps/desktop-gpui/locales/de.toml @@ -2908,10 +2908,15 @@ video = "Video" zoom = "Zoom" [editor.transcript] +captions_save_failed = "Die Untertitel konnten nicht gespeichert werden: {error}" +captions_saved = "Untertitel gespeichert" cleanup = "Aufräumen" cleanup_tooltip = "Füllwörter entfernen und Pausen kürzen" delete_matches = "Alle löschen" delete_matches_tooltip = "Jeden Treffer aus dem Video schneiden" +download_captions = "Untertitel herunterladen" +download_srt = "SubRip (.srt)" +download_vtt = "WebVTT (.vtt)" empty_description = "Transkribieren Sie diese Aufnahme auf diesem Computer und bearbeiten Sie dann das Video über seinen Text." empty_title = "Noch kein Transkript" hint = "Klicken Sie auf ein Wort, um dorthin zu springen. Wählen Sie Wörter aus und drücken Sie die Löschtaste, um sie aus dem Video zu schneiden." diff --git a/apps/desktop-gpui/locales/en.toml b/apps/desktop-gpui/locales/en.toml index 29f268aa5c6..6b3ef67b964 100644 --- a/apps/desktop-gpui/locales/en.toml +++ b/apps/desktop-gpui/locales/en.toml @@ -2910,10 +2910,15 @@ video = { text = "Video", context = "Timeline gutter label of the recording's vi zoom = "Zoom" [editor.transcript] +captions_save_failed = { text = "Could not save the captions: {error}", context = "{error} is a technical message that stays in English." } +captions_saved = "Captions saved" cleanup = "Clean up" cleanup_tooltip = "Remove filler words and shorten pauses" delete_matches = "Delete all" delete_matches_tooltip = "Cut every match from the video" +download_captions = "Download captions" +download_srt = { text = "SubRip (.srt)", context = "Menu item that saves captions as an SRT subtitle file. SubRip is a format name; keep it." } +download_vtt = { text = "WebVTT (.vtt)", context = "Menu item that saves captions as a WebVTT subtitle file. WebVTT is a format name; keep it." } empty_description = "Transcribe this recording on this computer, then edit the video by editing its text." empty_title = "No transcript yet" hint = "Click a word to jump to it. Select words and press Delete to cut them from the video." diff --git a/apps/desktop-gpui/locales/es.toml b/apps/desktop-gpui/locales/es.toml index 27fbf306d17..a6143cbd614 100644 --- a/apps/desktop-gpui/locales/es.toml +++ b/apps/desktop-gpui/locales/es.toml @@ -2908,10 +2908,15 @@ video = "Vídeo" zoom = "Zoom" [editor.transcript] +captions_save_failed = "No se pudieron guardar los subtítulos: {error}" +captions_saved = "Subtítulos guardados" cleanup = "Limpiar" cleanup_tooltip = "Quitar muletillas y acortar pausas" delete_matches = "Eliminar todo" delete_matches_tooltip = "Cortar del vídeo cada coincidencia" +download_captions = "Descargar subtítulos" +download_srt = "SubRip (.srt)" +download_vtt = "WebVTT (.vtt)" empty_description = "Transcribe esta grabación en este ordenador y luego edita el vídeo editando su texto." empty_title = "Aún no hay transcripción" hint = "Haz clic en una palabra para ir a ella. Selecciona palabras y pulsa la tecla de borrar para cortarlas del vídeo." diff --git a/apps/desktop-gpui/locales/fr.toml b/apps/desktop-gpui/locales/fr.toml index 87b46c59460..86b0f3de3cd 100644 --- a/apps/desktop-gpui/locales/fr.toml +++ b/apps/desktop-gpui/locales/fr.toml @@ -2908,10 +2908,15 @@ video = "Vidéo" zoom = "Zoom" [editor.transcript] +captions_save_failed = "Impossible d'enregistrer les sous-titres : {error}" +captions_saved = "Sous-titres enregistrés" cleanup = "Nettoyer" cleanup_tooltip = "Supprimer les mots de remplissage et raccourcir les pauses" delete_matches = "Tout supprimer" delete_matches_tooltip = "Couper chaque résultat de la vidéo" +download_captions = "Télécharger les sous-titres" +download_srt = "SubRip (.srt)" +download_vtt = "WebVTT (.vtt)" empty_description = "Transcrivez cet enregistrement sur cet ordinateur, puis modifiez la vidéo en modifiant son texte." empty_title = "Pas encore de transcription" hint = "Cliquez sur un mot pour y aller. Sélectionnez des mots et appuyez sur la touche Supprimer pour les couper de la vidéo." diff --git a/apps/desktop-gpui/locales/hi.toml b/apps/desktop-gpui/locales/hi.toml index 319ed6675bb..48144e58a7c 100644 --- a/apps/desktop-gpui/locales/hi.toml +++ b/apps/desktop-gpui/locales/hi.toml @@ -2908,10 +2908,15 @@ video = "वीडियो" zoom = "ज़ूम" [editor.transcript] +captions_save_failed = "कैप्शन सेव नहीं हो सके: {error}" +captions_saved = "कैप्शन सेव हो गए" cleanup = "साफ़ करें" cleanup_tooltip = "फ़िलर शब्द हटाएँ और ठहराव छोटे करें" delete_matches = "सभी हटाएँ" delete_matches_tooltip = "हर मिलान को वीडियो से काटें" +download_captions = "कैप्शन डाउनलोड करें" +download_srt = "SubRip (.srt)" +download_vtt = "WebVTT (.vtt)" empty_description = "इस कंप्यूटर पर इस रिकॉर्डिंग को ट्रांसक्राइब करें, फिर उसका टेक्स्ट बदलकर वीडियो एडिट करें।" empty_title = "अभी कोई ट्रांसक्रिप्ट नहीं" hint = "किसी शब्द पर जाने के लिए उस पर क्लिक करें। शब्द चुनें और डिलीट कुंजी दबाएँ, वे वीडियो से कट जाएँगे।" diff --git a/apps/desktop-gpui/locales/ja.toml b/apps/desktop-gpui/locales/ja.toml index d007debfd05..a8260b46812 100644 --- a/apps/desktop-gpui/locales/ja.toml +++ b/apps/desktop-gpui/locales/ja.toml @@ -2908,10 +2908,15 @@ video = "動画" zoom = "ズーム" [editor.transcript] +captions_save_failed = "字幕を保存できませんでした:{error}" +captions_saved = "字幕を保存しました" cleanup = "クリーンアップ" cleanup_tooltip = "フィラーを削除し、間を短くします" delete_matches = "すべて削除" delete_matches_tooltip = "一致した箇所をすべて動画からカットします" +download_captions = "字幕をダウンロード" +download_srt = "SubRip(.srt)" +download_vtt = "WebVTT(.vtt)" empty_description = "このコンピュータで録画を文字起こしすると、テキストを編集して動画を編集できます。" empty_title = "文字起こしはまだありません" hint = "単語をクリックするとその位置に移動します。単語を選択して削除キーを押すと、動画からカットされます。" diff --git a/apps/desktop-gpui/locales/zh-CN.toml b/apps/desktop-gpui/locales/zh-CN.toml index 922d4ee4a5d..82caca831f1 100644 --- a/apps/desktop-gpui/locales/zh-CN.toml +++ b/apps/desktop-gpui/locales/zh-CN.toml @@ -2908,10 +2908,15 @@ video = "视频" zoom = "缩放" [editor.transcript] +captions_save_failed = "无法保存字幕:{error}" +captions_saved = "字幕已保存" cleanup = "清理" cleanup_tooltip = "移除填充词并缩短停顿" delete_matches = "全部删除" delete_matches_tooltip = "从视频中剪掉所有匹配项" +download_captions = "下载字幕" +download_srt = "SubRip(.srt)" +download_vtt = "WebVTT(.vtt)" empty_description = "在这台电脑上转写此录制内容,然后通过编辑文字来编辑视频。" empty_title = "还没有转写内容" hint = "点击单词即可跳转。选中单词后按删除键,即可从视频中剪掉。" diff --git a/apps/desktop-gpui/src/caption_export.rs b/apps/desktop-gpui/src/caption_export.rs new file mode 100644 index 00000000000..ad37dddb588 --- /dev/null +++ b/apps/desktop-gpui/src/caption_export.rs @@ -0,0 +1,311 @@ +//! Captions as subtitle files: SubRip (`.srt`) and WebVTT (`.vtt`), timed to +//! the edited video. +//! +//! Cues come from the caption track the renderer draws, which is already +//! projected through cuts, clip speeds, transitions and fullscreen holds, and +//! are shaped the way the renderer shows them: fillers it hides are left out, +//! a caption ends where it stops being drawn, and where two captions overlap +//! only the one on screen is kept. + +use cap_project::caption_words::caption_display_segment; +use cap_project::{CaptionTrackSegment, MAX_CAPTION_WORD_DURATION, ProjectConfiguration}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CaptionFormat { + Srt, + Vtt, +} + +impl CaptionFormat { + pub const ALL: [Self; 2] = [Self::Srt, Self::Vtt]; + + pub fn label(self) -> &'static str { + match self { + Self::Srt => crate::i18n::tr::editor::transcript::download_srt(), + Self::Vtt => crate::i18n::tr::editor::transcript::download_vtt(), + } + } + + pub fn extension(self) -> &'static str { + match self { + Self::Srt => "srt", + Self::Vtt => "vtt", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Cue { + pub start_ms: u64, + pub end_ms: u64, + pub text: String, +} + +fn drawn_until(segment: &CaptionTrackSegment) -> f64 { + match segment.words.last() { + Some(last) => segment + .end + .min(f64::from(last.start) + f64::from(MAX_CAPTION_WORD_DURATION)), + None => segment.end, + } +} + +fn milliseconds(seconds: f64) -> u64 { + (seconds * 1000.0).round().max(0.0) as u64 +} + +/// One line per line of caption text, with control characters dropped and +/// runs of whitespace collapsed, so a cue can never end early on a blank line. +fn cue_text(text: &str) -> String { + text.replace("\r\n", "\n") + .replace('\r', "\n") + .split('\n') + .map(|line| { + line.chars() + .map(|char| if char.is_whitespace() { ' ' } else { char }) + .filter(|char| !char.is_control()) + .collect::() + .split_whitespace() + .collect::>() + .join(" ") + }) + .filter(|line| !line.is_empty()) + .collect::>() + .join("\n") +} + +pub(crate) fn cues(project: &ProjectConfiguration) -> Vec { + let (Some(captions), Some(timeline)) = (&project.captions, &project.timeline) else { + return Vec::new(); + }; + let hide_fillers = captions.settings.hides_fillers(); + let mut shown: Vec<(f64, f64, String)> = + timeline + .caption_segments + .iter() + .filter_map(|segment| { + let display = caption_display_segment(segment, hide_fillers)?; + let (start, end) = (display.start, drawn_until(&display)); + let text = cue_text(&display.text); + (start.is_finite() && end.is_finite() && end > start && !text.is_empty()) + .then_some((start.max(0.0), end, text)) + }) + .collect(); + shown.sort_by(|a, b| a.0.total_cmp(&b.0)); + + let mut cues = Vec::with_capacity(shown.len()); + let mut covered = f64::NEG_INFINITY; + for (start, end, text) in shown { + let visible = start.max(covered); + covered = covered.max(end); + let start_ms = milliseconds(visible); + let end_ms = milliseconds(end); + if end_ms > start_ms { + cues.push(Cue { + start_ms, + end_ms, + text, + }); + } + } + cues +} + +fn timestamp(ms: u64, separator: char) -> String { + format!( + "{:02}:{:02}:{:02}{separator}{:03}", + ms / 3_600_000, + ms / 60_000 % 60, + ms / 1000 % 60, + ms % 1000 + ) +} + +fn escape_vtt(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +pub(crate) fn format(cues: &[Cue], format: CaptionFormat) -> String { + let (separator, mut out) = match format { + CaptionFormat::Srt => (',', String::new()), + CaptionFormat::Vtt => ('.', String::from("WEBVTT\n")), + }; + for (index, cue) in cues.iter().enumerate() { + if index > 0 || format == CaptionFormat::Vtt { + out.push('\n'); + } + let text = match format { + CaptionFormat::Srt => cue.text.clone(), + CaptionFormat::Vtt => escape_vtt(&cue.text), + }; + out.push_str(&format!( + "{}\n{} --> {}\n{text}\n", + index + 1, + timestamp(cue.start_ms, separator), + timestamp(cue.end_ms, separator) + )); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn project(timeline: serde_json::Value, settings: serde_json::Value) -> ProjectConfiguration { + serde_json::from_value(serde_json::json!({ + "timeline": timeline, + "captions": { "segments": [], "settings": settings, "sourceTimed": true } + })) + .unwrap() + } + + fn track(segments: serde_json::Value) -> ProjectConfiguration { + project( + serde_json::json!({ + "segments": [{ "start": 0.0, "end": 100.0, "timescale": 1.0 }], + "zoomSegments": [], + "captionSegments": segments + }), + serde_json::json!({}), + ) + } + + fn cue(start_ms: u64, end_ms: u64, text: &str) -> Cue { + Cue { + start_ms, + end_ms, + text: text.to_string(), + } + } + + #[test] + fn srt_and_vtt_are_written_to_spec() { + let cues = [ + cue(111, 445, "hello"), + cue(3_723_004, 3_725_000, "Cap & fast --> captions"), + ]; + assert_eq!( + format(&cues, CaptionFormat::Srt), + "1\n00:00:00,111 --> 00:00:00,445\nhello\n\n2\n01:02:03,004 --> 01:02:05,000\nCap & fast --> captions\n" + ); + assert_eq!( + format(&cues, CaptionFormat::Vtt), + "WEBVTT\n\n1\n00:00:00.111 --> 00:00:00.445\nhello\n\n2\n01:02:03.004 --> 01:02:05.000\nCap <b> & fast --> captions\n" + ); + assert_eq!(format(&[], CaptionFormat::Srt), ""); + assert_eq!(format(&[], CaptionFormat::Vtt), "WEBVTT\n"); + } + + #[test] + fn cue_text_never_breaks_a_cue_and_keeps_unicode() { + let project = track(serde_json::json!([ + { "id": "a", "start": 1.0, "end": 2.0, "text": " line\tone \r\n\r\n\u{0007}line two\u{000B} " }, + { "id": "b", "start": 3.0, "end": 4.0, "text": "שלום עולם 👋🏽 你好、世界 e\u{0301}" }, + { "id": "c", "start": 5.0, "end": 6.0, "text": " \n " } + ])); + assert_eq!( + cues(&project), + [ + cue(1000, 2000, "line one\nline two"), + cue(3000, 4000, "שלום עולם 👋🏽 你好、世界 e\u{0301}") + ] + ); + } + + #[test] + fn overlapping_captions_keep_the_one_on_screen() { + let project = track(serde_json::json!([ + { "id": "a", "start": 1.0, "end": 5.0, "text": "first" }, + { "id": "b", "start": 3.0, "end": 4.0, "text": "hidden" }, + { "id": "c", "start": 4.5, "end": 7.0, "text": "second" }, + { "id": "d", "start": 7.0, "end": 8.0, "text": "third" } + ])); + assert_eq!( + cues(&project), + [ + cue(1000, 5000, "first"), + cue(5000, 7000, "second"), + cue(7000, 8000, "third") + ] + ); + } + + #[test] + fn hidden_fillers_and_word_lingers_match_the_renderer() { + let mut project = track(serde_json::json!([ + { "id": "a", "start": 1.0, "end": 9.0, "text": "um so this", + "words": [ + { "text": "um", "start": 1.0, "end": 1.4 }, + { "text": "so", "start": 1.5, "end": 1.8 }, + { "text": "this", "start": 2.0, "end": 9.0 } + ] }, + { "id": "b", "start": 10.0, "end": 10.5, "text": "uh", + "words": [{ "text": "uh", "start": 10.0, "end": 10.5 }] } + ])); + let settings = &mut project.captions.as_mut().unwrap().settings; + settings.filler_language = Some("en".into()); + let linger = (2.0 + f64::from(MAX_CAPTION_WORD_DURATION)) * 1000.0; + assert_eq!( + cues(&project), + [cue(1500, linger.round() as u64, "so this")] + ); + + project + .captions + .as_mut() + .unwrap() + .settings + .show_filler_words = true; + assert_eq!(cues(&project).len(), 2); + assert_eq!(cues(&project)[0].text, "um so this"); + } + + #[test] + fn timings_follow_cuts_speed_and_holds_of_the_edited_video() { + let mut project = project( + serde_json::json!({ + "segments": [ + { "start": 0.0, "end": 4.0, "timescale": 2.0 }, + { "start": 6.0, "end": 10.0, "timescale": 1.0 } + ], + "zoomSegments": [], + "textSegments": [{ "start": 2.0, "end": 3.0, "layout": "fullscreen" }] + }), + serde_json::json!({ "enabled": true }), + ); + project.captions.as_mut().unwrap().segments = serde_json::from_value(serde_json::json!([ + { "id": "fast", "start": 1.0, "end": 3.0, "text": "fast" }, + { "id": "gone", "start": 4.5, "end": 5.5, "text": "cut" }, + { "id": "late", "start": 7.0, "end": 8.0, "text": "after the card" } + ])) + .unwrap(); + cap_project::synchronize_captions(&mut project, &[10.0]); + assert_eq!( + cues(&project), + [cue(500, 1500, "fast"), cue(4000, 5000, "after the card")] + ); + assert!(cues(&ProjectConfiguration::default()).is_empty()); + } + + #[test] + fn two_hours_of_captions_export_quickly_with_long_timestamps() { + let segments: Vec = (0..4_000) + .map(|index| { + let start = index as f64 * 1.9; + serde_json::json!({ "id": format!("s{index}"), "start": start, "end": start + 1.5, "text": format!("caption {index}") }) + }) + .collect(); + let mut project = track(serde_json::Value::Array(segments)); + project.timeline.as_mut().unwrap().segments[0].end = 7_700.0; + let started = std::time::Instant::now(); + let cues = cues(&project); + let text = format(&cues, CaptionFormat::Vtt); + let elapsed = started.elapsed(); + assert_eq!(cues.len(), 4_000); + assert!(text.contains("4000\n02:06:38.100 --> 02:06:39.600\ncaption 3999\n")); + assert!(elapsed < std::time::Duration::from_secs(1), "{elapsed:?}"); + } +} diff --git a/apps/desktop-gpui/src/editor_export.rs b/apps/desktop-gpui/src/editor_export.rs index 7d00053e0a1..de5bbfc220e 100644 --- a/apps/desktop-gpui/src/editor_export.rs +++ b/apps/desktop-gpui/src/editor_export.rs @@ -513,7 +513,7 @@ fn format_duration(seconds: f64) -> String { } } -fn export_file_name(pretty_name: &str, extension: &str) -> String { +pub(crate) fn export_file_name(pretty_name: &str, extension: &str) -> String { let mut name = sanitize_filename::sanitize_with_options( pretty_name, sanitize_filename::Options { diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index d19371448c6..79c3517b2d9 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -2002,6 +2002,7 @@ fn clip_menu_row( enum ToolbarMenu { AspectRatio, PreviewSpeed, + CaptionDownload, } struct OpenToolbarMenu { @@ -7630,7 +7631,77 @@ impl EditorWindow { .iter() .map(|speed| ui::MenuItem::new(speed.label(), *speed == self.preview_speed)) .collect(), + ToolbarMenu::CaptionDownload => crate::caption_export::CaptionFormat::ALL + .iter() + .map(|format| ui::MenuItem::new(format.label(), false)) + .collect(), + } + } + + pub(crate) fn caption_download_menu_open(&self) -> bool { + self.toolbar_menu + .as_ref() + .is_some_and(|menu| menu.kind == ToolbarMenu::CaptionDownload) + } + + pub(crate) fn open_caption_download_menu( + &mut self, + trigger_bounds: Bounds, + window: &mut Window, + cx: &mut Context, + ) { + self.open_toolbar_menu(ToolbarMenu::CaptionDownload, trigger_bounds, window, cx); + } + + /// Saves the captions as a subtitle file timed to the edited video. + pub(crate) fn download_captions( + &mut self, + format: crate::caption_export::CaptionFormat, + window: &mut Window, + cx: &mut Context, + ) { + let cues = crate::caption_export::cues(&self.project); + if cues.is_empty() { + return; } + let contents = crate::caption_export::format(&cues, format); + let extension = format.extension(); + let pretty_name = self + .summary() + .map(|summary| summary.pretty_name.clone()) + .filter(|name| !name.trim().is_empty()) + .unwrap_or_else(|| "Cap Recording".into()); + let name = crate::editor_export::export_file_name(&pretty_name, extension); + tracing::info!(format = extension, cues = cues.len(), "caption download"); + cx.spawn_in(window, async move |this, cx| { + let Some(path) = crate::platform::save_file_panel_async(&name, &[extension], cx).await + else { + return; + }; + let written = cx + .background_executor() + .spawn({ + let path = path.clone(); + async move { std::fs::write(&path, contents) } + }) + .await; + this.update_in(cx, |this, window, cx| match written { + Ok(()) => { + tracing::info!(path = %path.display(), "captions saved"); + this.show_notice(tr::editor::transcript::captions_saved(), window, cx); + } + Err(error) => { + tracing::error!(%error, "saving captions failed"); + this.show_notice( + tr::editor::transcript::captions_save_failed(error.to_string()), + window, + cx, + ); + } + }) + .ok(); + }) + .detach(); } fn open_toolbar_menu( @@ -7708,6 +7779,11 @@ impl EditorWindow { self.set_preview_speed(*speed, window, cx); } } + ToolbarMenu::CaptionDownload => { + if let Some(format) = crate::caption_export::CaptionFormat::ALL.get(index) { + self.download_captions(*format, window, cx); + } + } } cx.notify(); window.refresh(); @@ -7722,6 +7798,7 @@ impl EditorWindow { .min_width(match kind { ToolbarMenu::AspectRatio => px(200.), ToolbarMenu::PreviewSpeed => px(96.), + ToolbarMenu::CaptionDownload => px(160.), }) .on_select(cx.listener(move |this, index: &usize, window, cx| { this.choose_toolbar_menu(kind, *index, window, cx); diff --git a/apps/desktop-gpui/src/main.rs b/apps/desktop-gpui/src/main.rs index acdd53da302..45ccef4a0e2 100644 --- a/apps/desktop-gpui/src/main.rs +++ b/apps/desktop-gpui/src/main.rs @@ -15,6 +15,7 @@ mod camera_blur; #[cfg(any(not(target_os = "macos"), test))] mod camera_blur_portable; mod camera_loading; +mod caption_export; mod camera_window; mod controls_window; mod crash_capture; diff --git a/apps/desktop-gpui/src/transcript/view.rs b/apps/desktop-gpui/src/transcript/view.rs index 0d2009a2317..00bf8b72891 100644 --- a/apps/desktop-gpui/src/transcript/view.rs +++ b/apps/desktop-gpui/src/transcript/view.rs @@ -1567,6 +1567,11 @@ impl EditorWindow { .doc .as_ref() .is_some_and(|doc| !doc.is_empty()); + let has_cues = self + .project + .timeline + .as_ref() + .is_some_and(|timeline| !timeline.caption_segments.is_empty()); crate::editor_side_panel::panel_header( &theme, tr::editor::transcript::title(), @@ -1578,6 +1583,18 @@ impl EditorWindow { .flex_row() .items_center() .gap(px(2.)) + .child( + ui::EditorButton::plain(&theme, "transcript-download") + .left_icon("icons/download.svg") + .tooltip(&theme, tr::editor::transcript::download_captions()) + .disabled(!has_cues) + .pressed(self.caption_download_menu_open()) + .on_open(cx.listener( + |this, bounds: &gpui::Bounds, window, cx| { + this.open_caption_download_menu(*bounds, window, cx); + }, + )), + ) .child( ui::EditorButton::plain(&theme, "transcript-search") .left_icon("icons/search.svg") From 2121d73ad790867848f1fda31086779b9712a9d7 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:48:17 -0500 Subject: [PATCH 2/4] test(desktop-gpui): cover the caption download strings --- .../src/i18n/editor_shell_surface_tests.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/desktop-gpui/src/i18n/editor_shell_surface_tests.rs b/apps/desktop-gpui/src/i18n/editor_shell_surface_tests.rs index 05e50b7b5a8..fa4b66adf3d 100644 --- a/apps/desktop-gpui/src/i18n/editor_shell_surface_tests.rs +++ b/apps/desktop-gpui/src/i18n/editor_shell_surface_tests.rs @@ -20,6 +20,7 @@ pub(super) fn is_editor_shell_surface_key(key: &str) -> bool { } const FORMATTED: &[&str] = &[ + "editor.transcript.captions_save_failed", "editor.transcript.fillers.remove_all", "editor.transcript.pauses.summary", "editor.transcript.restore_hint", @@ -1068,12 +1069,19 @@ fn editor_shell_surfaces_match_the_literals_they_replaced() { "Open recording bundle", ), (tr::editor::window::header::presets(), "Presets"), + (tr::editor::transcript::captions_saved(), "Captions saved"), (tr::editor::transcript::cleanup(), "Clean up"), ( tr::editor::transcript::cleanup_tooltip(), "Remove filler words and shorten pauses", ), (tr::editor::transcript::delete_matches(), "Delete all"), + ( + tr::editor::transcript::download_captions(), + "Download captions", + ), + (tr::editor::transcript::download_srt(), "SubRip (.srt)"), + (tr::editor::transcript::download_vtt(), "WebVTT (.vtt)"), ( tr::editor::transcript::delete_matches_tooltip(), "Cut every match from the video", @@ -1702,6 +1710,10 @@ fn editor_shell_messages_format_like_the_code_they_replaced() { tr::editor::window::presets::delete_confirm(&name), format!("Are you sure you want to delete \"{}\"?", name) ); + assert_eq!( + tr::editor::transcript::captions_save_failed(&error), + format!("Could not save the captions: {error}") + ); assert_eq!( tr::editor::window::save_failed(path.display(), &error), format!( From a87b07af166e7d591e719ac3003e1380a2b02b58 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:42:47 -0500 Subject: [PATCH 3/4] fix(desktop-gpui): time subtitle cues by the caption on screen, fade included --- apps/desktop-gpui/src/caption_export.rs | 203 ++++++++++++++++++++---- apps/desktop-gpui/src/editor_window.rs | 1 - 2 files changed, 169 insertions(+), 35 deletions(-) diff --git a/apps/desktop-gpui/src/caption_export.rs b/apps/desktop-gpui/src/caption_export.rs index ad37dddb588..553896f5a43 100644 --- a/apps/desktop-gpui/src/caption_export.rs +++ b/apps/desktop-gpui/src/caption_export.rs @@ -4,8 +4,8 @@ //! Cues come from the caption track the renderer draws, which is already //! projected through cuts, clip speeds, transitions and fullscreen holds, and //! are shaped the way the renderer shows them: fillers it hides are left out, -//! a caption ends where it stops being drawn, and where two captions overlap -//! only the one on screen is kept. +//! a caption lasts while it is drawn or fading out, and where two captions +//! overlap only the one on screen is kept. use cap_project::caption_words::caption_display_segment; use cap_project::{CaptionTrackSegment, MAX_CAPTION_WORD_DURATION, ProjectConfiguration}; @@ -74,39 +74,134 @@ fn cue_text(text: &str) -> String { .join("\n") } +/// Disjoint, sorted time ranges. +#[derive(Default)] +struct Covered(Vec<(f64, f64)>); + +impl Covered { + fn uncovered(&self, (start, end): (f64, f64)) -> Vec<(f64, f64)> { + let mut pieces = Vec::new(); + let mut cursor = start; + let first = self.0.partition_point(|range| range.1 <= start); + for &(from, until) in &self.0[first..] { + if from >= end || cursor >= end { + break; + } + if from > cursor { + pieces.push((cursor, from)); + } + cursor = cursor.max(until); + } + if cursor < end { + pieces.push((cursor, end)); + } + pieces + } + + fn add(&mut self, (start, end): (f64, f64)) { + let first = self.0.partition_point(|range| range.1 < start); + let mut last = first; + let (mut from, mut until) = (start, end); + while let Some(&(other_from, other_until)) = self.0.get(last) { + if other_from > until { + break; + } + from = from.min(other_from); + until = until.max(other_until); + last += 1; + } + self.0.splice(first..last, [(from, until)]); + } +} + +struct Shown { + drawn: (f64, f64), + fading: (f64, f64), + text: String, +} + +/// Mirrors `find_active_caption_segment`: at any moment the first caption in +/// track order that is drawn there wins, and a fade-out shows only where no +/// caption is drawn, again first in track order. pub(crate) fn cues(project: &ProjectConfiguration) -> Vec { let (Some(captions), Some(timeline)) = (&project.captions, &project.timeline) else { return Vec::new(); }; - let hide_fillers = captions.settings.hides_fillers(); - let mut shown: Vec<(f64, f64, String)> = - timeline - .caption_segments - .iter() - .filter_map(|segment| { - let display = caption_display_segment(segment, hide_fillers)?; - let (start, end) = (display.start, drawn_until(&display)); - let text = cue_text(&display.text); - (start.is_finite() && end.is_finite() && end > start && !text.is_empty()) - .then_some((start.max(0.0), end, text)) - }) - .collect(); - shown.sort_by(|a, b| a.0.total_cmp(&b.0)); - - let mut cues = Vec::with_capacity(shown.len()); - let mut covered = f64::NEG_INFINITY; - for (start, end, text) in shown { - let visible = start.max(covered); - covered = covered.max(end); - let start_ms = milliseconds(visible); - let end_ms = milliseconds(end); - if end_ms > start_ms { - cues.push(Cue { + let settings = &captions.settings; + let hide_fillers = settings.hides_fillers(); + let shown: Vec = timeline + .caption_segments + .iter() + .filter_map(|segment| { + let display = caption_display_segment(segment, hide_fillers)?; + let start = display.start.max(segment.start).max(0.0); + let end = drawn_until(&display).min(drawn_until(segment)); + let fade = f64::from( + display + .fade_duration_override + .unwrap_or(settings.fade_duration), + ) + .max(0.0); + let text = cue_text(&display.text); + (start.is_finite() && end.is_finite() && end > start && !text.is_empty()).then_some( + Shown { + drawn: (start, end), + fading: (end, if fade.is_finite() { end + fade } else { end }), + text, + }, + ) + }) + .collect(); + + let mut pieces: Vec<(f64, f64, usize)> = Vec::new(); + let mut drawn = Covered::default(); + for (index, caption) in shown.iter().enumerate() { + pieces.extend( + drawn + .uncovered(caption.drawn) + .into_iter() + .map(|(from, until)| (from, until, index)), + ); + drawn.add(caption.drawn); + } + let mut fading = Covered::default(); + for (index, caption) in shown.iter().enumerate() { + if caption.fading.1 <= caption.fading.0 { + continue; + } + for piece in drawn.uncovered(caption.fading) { + pieces.extend( + fading + .uncovered(piece) + .into_iter() + .map(|(from, until)| (from, until, index)), + ); + } + fading.add(caption.fading); + } + pieces.sort_by(|a, b| a.0.total_cmp(&b.0)); + + let mut cues: Vec = Vec::with_capacity(shown.len()); + let mut last: Option<(usize, f64)> = None; + for (from, until, index) in pieces { + let start_ms = milliseconds(from); + let end_ms = milliseconds(until); + if end_ms <= start_ms { + continue; + } + match (last, cues.last_mut()) { + (Some((previous, previous_end)), Some(cue)) + if previous == index && (from - previous_end).abs() < 1e-9 => + { + cue.end_ms = end_ms; + } + _ => cues.push(Cue { start_ms, end_ms, - text, - }); + text: shown[index].text.clone(), + }), } + last = Some((index, until)); } cues } @@ -209,8 +304,8 @@ mod tests { assert_eq!( cues(&project), [ - cue(1000, 2000, "line one\nline two"), - cue(3000, 4000, "שלום עולם 👋🏽 你好、世界 e\u{0301}") + cue(1000, 2150, "line one\nline two"), + cue(3000, 4150, "שלום עולם 👋🏽 你好、世界 e\u{0301}") ] ); } @@ -228,9 +323,49 @@ mod tests { [ cue(1000, 5000, "first"), cue(5000, 7000, "second"), - cue(7000, 8000, "third") + cue(7000, 8150, "third") + ] + ); + } + + #[test] + fn the_earlier_caption_in_the_track_wins_even_when_it_shows_later() { + let mut project = track(serde_json::json!([ + { "id": "a", "start": 8.0, "end": 10.0, "text": "um first", + "words": [ + { "text": "um", "start": 8.0, "end": 8.4 }, + { "text": "first", "start": 9.2, "end": 10.0 } + ] }, + { "id": "b", "start": 8.5, "end": 10.5, "text": "second" } + ])); + project.captions.as_mut().unwrap().settings.filler_language = Some("en".into()); + assert_eq!( + cues(&project), + [ + cue(8500, 9200, "second"), + cue(9200, 10000, "first"), + cue(10000, 10650, "second") + ] + ); + } + + #[test] + fn a_fade_out_lasts_until_the_next_caption_or_its_own_length() { + let mut project = track(serde_json::json!([ + { "id": "a", "start": 1.0, "end": 2.0, "text": "slow", "fadeDurationOverride": 0.5 }, + { "id": "b", "start": 2.2, "end": 3.0, "text": "none", "fadeDurationOverride": 0.0 }, + { "id": "c", "start": 4.0, "end": 5.0, "text": "default" } + ])); + assert_eq!( + cues(&project), + [ + cue(1000, 2200, "slow"), + cue(2200, 3000, "none"), + cue(4000, 5150, "default") ] ); + project.captions.as_mut().unwrap().settings.fade_duration = 0.0; + assert_eq!(cues(&project)[2], cue(4000, 5000, "default")); } #[test] @@ -247,7 +382,7 @@ mod tests { ])); let settings = &mut project.captions.as_mut().unwrap().settings; settings.filler_language = Some("en".into()); - let linger = (2.0 + f64::from(MAX_CAPTION_WORD_DURATION)) * 1000.0; + let linger = (2.0 + f64::from(MAX_CAPTION_WORD_DURATION) + 0.15) * 1000.0; assert_eq!( cues(&project), [cue(1500, linger.round() as u64, "so this")] @@ -285,7 +420,7 @@ mod tests { cap_project::synchronize_captions(&mut project, &[10.0]); assert_eq!( cues(&project), - [cue(500, 1500, "fast"), cue(4000, 5000, "after the card")] + [cue(500, 1650, "fast"), cue(4000, 5150, "after the card")] ); assert!(cues(&ProjectConfiguration::default()).is_empty()); } @@ -305,7 +440,7 @@ mod tests { let text = format(&cues, CaptionFormat::Vtt); let elapsed = started.elapsed(); assert_eq!(cues.len(), 4_000); - assert!(text.contains("4000\n02:06:38.100 --> 02:06:39.600\ncaption 3999\n")); + assert!(text.contains("4000\n02:06:38.100 --> 02:06:39.750\ncaption 3999\n")); assert!(elapsed < std::time::Duration::from_secs(1), "{elapsed:?}"); } } diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 79c3517b2d9..e859c258b56 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -7653,7 +7653,6 @@ impl EditorWindow { self.open_toolbar_menu(ToolbarMenu::CaptionDownload, trigger_bounds, window, cx); } - /// Saves the captions as a subtitle file timed to the edited video. pub(crate) fn download_captions( &mut self, format: crate::caption_export::CaptionFormat, From 19db9accadd952a510ec14b80c730d97ae2814ff Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:11:20 +0000 Subject: [PATCH 4/4] docs(desktop-gpui): drop the caption export intro comment --- apps/desktop-gpui/src/caption_export.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/desktop-gpui/src/caption_export.rs b/apps/desktop-gpui/src/caption_export.rs index 553896f5a43..c128ddf1b0d 100644 --- a/apps/desktop-gpui/src/caption_export.rs +++ b/apps/desktop-gpui/src/caption_export.rs @@ -1,6 +1,3 @@ -//! Captions as subtitle files: SubRip (`.srt`) and WebVTT (`.vtt`), timed to -//! the edited video. -//! //! Cues come from the caption track the renderer draws, which is already //! projected through cuts, clip speeds, transitions and fullscreen holds, and //! are shaped the way the renderer shows them: fillers it hides are left out,