Skip to content

Commit 4fb2fa7

Browse files
Merge pull request #4246 from CapSoftware/feat/gpui-add-caption-at-playhead
feat(desktop-gpui): add captions at the playhead and on the empty caption lane
2 parents 2edd7ff + c6ea2f9 commit 4fb2fa7

12 files changed

Lines changed: 639 additions & 9 deletions

File tree

‎apps/desktop-gpui/locales/de.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,7 @@ model_downloaded = "Modell {model} heruntergeladen"
26332633
model_label = "Modell"
26342634
model_updated = "Empfohlenes Modell aktualisiert"
26352635
model_updated_description = "Wir empfehlen die neue Version. Sie ist genauer und erkennt auch kurze Aufnahmen, die der alten entgangen sind."
2636+
new_caption = "Neuer Untertitel"
26362637
position = "Position"
26372638
refining_word_timings = "Wort-Timings werden verfeinert…"
26382639
regenerate = "Untertitel neu erstellen"
@@ -2908,6 +2909,7 @@ video = "Video"
29082909
zoom = "Zoom"
29092910

29102911
[editor.transcript]
2912+
add_caption = "Untertitel am Abspielkopf hinzufügen"
29112913
cleanup = "Aufräumen"
29122914
cleanup_tooltip = "Füllwörter entfernen und Pausen kürzen"
29132915
delete_matches = "Alle löschen"
@@ -2967,6 +2969,7 @@ audio_segment_changed = "Dieses Audiosegment hat sich geändert. Wählen Sie es
29672969
crop = "Zuschneiden"
29682970
crop_tooltip = "Video zuschneiden"
29692971
no_room_for_3d_shot = "Kein Platz für eine weitere 3D-Einstellung"
2972+
no_room_for_caption = "Hier ist kein Platz für einen weiteren Untertitel"
29702973
redo = "Wiederholen"
29712974
save_failed = "{path} konnte nicht gespeichert werden. Lassen Sie den Editor geöffnet und versuchen Sie es erneut: {error}"
29722975
undo = "Rückgängig"

‎apps/desktop-gpui/locales/en.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2635,6 +2635,7 @@ model_downloaded = "{model} model downloaded"
26352635
model_label = "Model"
26362636
model_updated = "Recommended model updated"
26372637
model_updated_description = "We recommend the new version. It is more accurate and catches short recordings the old one missed."
2638+
new_caption = "New caption"
26382639
position = "Position"
26392640
refining_word_timings = "Refining word timings…"
26402641
regenerate = "Regenerate Captions"
@@ -2910,6 +2911,7 @@ video = { text = "Video", context = "Timeline gutter label of the recording's vi
29102911
zoom = "Zoom"
29112912

29122913
[editor.transcript]
2914+
add_caption = "Add caption at playhead"
29132915
cleanup = "Clean up"
29142916
cleanup_tooltip = "Remove filler words and shorten pauses"
29152917
delete_matches = "Delete all"
@@ -2969,6 +2971,7 @@ audio_segment_changed = "This audio segment changed. Select it and choose a repl
29692971
crop = "Crop"
29702972
crop_tooltip = "Crop Video"
29712973
no_room_for_3d_shot = "No room for another 3D shot"
2974+
no_room_for_caption = "No room for another caption here"
29722975
redo = "Redo"
29732976
save_failed = { text = "Could not save {path}. Keep the editor open and try again: {error}", context = "{path} is the recording's folder; {error} is a technical message that stays in English." }
29742977
undo = "Undo"

‎apps/desktop-gpui/locales/es.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,7 @@ model_downloaded = "Modelo {model} descargado"
26332633
model_label = "Modelo"
26342634
model_updated = "Modelo recomendado actualizado"
26352635
model_updated_description = "Te recomendamos la nueva versión. Es más precisa y transcribe grabaciones cortas que la anterior pasaba por alto."
2636+
new_caption = "Subtítulo nuevo"
26362637
position = "Posición"
26372638
refining_word_timings = "Ajustando los tiempos de las palabras…"
26382639
regenerate = "Regenerar subtítulos"
@@ -2908,6 +2909,7 @@ video = "Vídeo"
29082909
zoom = "Zoom"
29092910

29102911
[editor.transcript]
2912+
add_caption = "Añadir subtítulo en el cabezal"
29112913
cleanup = "Limpiar"
29122914
cleanup_tooltip = "Quitar muletillas y acortar pausas"
29132915
delete_matches = "Eliminar todo"
@@ -2967,6 +2969,7 @@ audio_segment_changed = "Este segmento de audio cambió. Selecciónalo y vuelve
29672969
crop = "Recortar"
29682970
crop_tooltip = "Recortar vídeo"
29692971
no_room_for_3d_shot = "No hay espacio para otra toma 3D"
2972+
no_room_for_caption = "No hay espacio para otro subtítulo aquí"
29702973
redo = "Rehacer"
29712974
save_failed = "No se pudo guardar {path}. Mantén el editor abierto e inténtalo de nuevo: {error}"
29722975
undo = "Deshacer"

‎apps/desktop-gpui/locales/fr.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,7 @@ model_downloaded = "Modèle {model} téléchargé"
26332633
model_label = "Modèle"
26342634
model_updated = "Modèle recommandé mis à jour"
26352635
model_updated_description = "Nous recommandons la nouvelle version. Elle est plus précise et transcrit les enregistrements courts que l'ancienne ne captait pas."
2636+
new_caption = "Nouveau sous-titre"
26362637
position = "Position"
26372638
refining_word_timings = "Affinage du minutage des mots…"
26382639
regenerate = "Régénérer les sous-titres"
@@ -2908,6 +2909,7 @@ video = "Vidéo"
29082909
zoom = "Zoom"
29092910

29102911
[editor.transcript]
2912+
add_caption = "Ajouter un sous-titre à la tête de lecture"
29112913
cleanup = "Nettoyer"
29122914
cleanup_tooltip = "Supprimer les mots de remplissage et raccourcir les pauses"
29132915
delete_matches = "Tout supprimer"
@@ -2967,6 +2969,7 @@ audio_segment_changed = "Ce segment audio a changé. Sélectionnez-le et choisis
29672969
crop = "Recadrer"
29682970
crop_tooltip = "Recadrer la vidéo"
29692971
no_room_for_3d_shot = "Plus de place pour un autre plan 3D"
2972+
no_room_for_caption = "Pas de place pour un autre sous-titre ici"
29702973
redo = "Rétablir"
29712974
save_failed = "Impossible d'enregistrer {path}. Gardez l'éditeur ouvert et réessayez : {error}"
29722975
undo = "Annuler"

‎apps/desktop-gpui/locales/hi.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,7 @@ model_downloaded = "{model} मॉडल डाउनलोड हो गया"
26332633
model_label = "मॉडल"
26342634
model_updated = "सुझाया गया मॉडल अपडेट हुआ"
26352635
model_updated_description = "हम नया वर्ज़न इस्तेमाल करने की सलाह देते हैं। यह ज़्यादा सटीक है और उन छोटी रिकॉर्डिंग को भी पकड़ता है जो पुराने वर्ज़न से छूट जाती थीं।"
2636+
new_caption = "नया कैप्शन"
26362637
position = "पोज़िशन"
26372638
refining_word_timings = "शब्दों का समय सुधारा जा रहा है…"
26382639
regenerate = "कैप्शन दोबारा बनाएँ"
@@ -2908,6 +2909,7 @@ video = "वीडियो"
29082909
zoom = "ज़ूम"
29092910

29102911
[editor.transcript]
2912+
add_caption = "प्लेहेड पर कैप्शन जोड़ें"
29112913
cleanup = "साफ़ करें"
29122914
cleanup_tooltip = "फ़िलर शब्द हटाएँ और ठहराव छोटे करें"
29132915
delete_matches = "सभी हटाएँ"
@@ -2967,6 +2969,7 @@ audio_segment_changed = "यह ऑडियो सेगमेंट बदल
29672969
crop = "क्रॉप"
29682970
crop_tooltip = "वीडियो क्रॉप करें"
29692971
no_room_for_3d_shot = "एक और 3D शॉट के लिए जगह नहीं है"
2972+
no_room_for_caption = "यहाँ एक और कैप्शन के लिए जगह नहीं है"
29702973
redo = "फिर करें"
29712974
save_failed = "{path} सेव नहीं हो सका। एडिटर खुला रखें और फिर से कोशिश करें: {error}"
29722975
undo = "पहले जैसा करें"

‎apps/desktop-gpui/locales/ja.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,7 @@ model_downloaded = "{model}モデルをダウンロード済み"
26332633
model_label = "モデル"
26342634
model_updated = "推奨モデルを更新しました"
26352635
model_updated_description = "新しいバージョンをおすすめします。より正確になり、以前のバージョンでは聞き取れなかった短い録画も文字起こしできます。"
2636+
new_caption = "新しい字幕"
26362637
position = "位置"
26372638
refining_word_timings = "単語のタイミングを調整中…"
26382639
regenerate = "字幕を再生成"
@@ -2908,6 +2909,7 @@ video = "動画"
29082909
zoom = "ズーム"
29092910

29102911
[editor.transcript]
2912+
add_caption = "再生ヘッドに字幕を追加"
29112913
cleanup = "クリーンアップ"
29122914
cleanup_tooltip = "フィラーを削除し、間を短くします"
29132915
delete_matches = "すべて削除"
@@ -2967,6 +2969,7 @@ audio_segment_changed = "この音声セグメントは変更されました。
29672969
crop = "切り抜き"
29682970
crop_tooltip = "動画を切り抜く"
29692971
no_room_for_3d_shot = "別の3Dショットを入れる余地がありません"
2972+
no_room_for_caption = "ここには字幕を追加する余地がありません"
29702973
redo = "やり直す"
29712974
save_failed = "{path}を保存できませんでした。エディタを開いたまま、もう一度お試しください:{error}"
29722975
undo = "取り消す"

‎apps/desktop-gpui/locales/zh-CN.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,7 @@ model_downloaded = "已下载“{model}”模型"
26332633
model_label = "模型"
26342634
model_updated = "推荐模型已更新"
26352635
model_updated_description = "推荐使用新版本。它更准确,还能转写旧版本漏掉的短录制内容。"
2636+
new_caption = "新字幕"
26362637
position = "位置"
26372638
refining_word_timings = "正在优化单词时间…"
26382639
regenerate = "重新生成字幕"
@@ -2908,6 +2909,7 @@ video = "视频"
29082909
zoom = "缩放"
29092910

29102911
[editor.transcript]
2912+
add_caption = "在播放头处添加字幕"
29112913
cleanup = "清理"
29122914
cleanup_tooltip = "移除填充词并缩短停顿"
29132915
delete_matches = "全部删除"
@@ -2967,6 +2969,7 @@ audio_segment_changed = "此音频分段已更改。请选中它并重新选择
29672969
crop = "裁剪"
29682970
crop_tooltip = "裁剪视频"
29692971
no_room_for_3d_shot = "没有空间再添加 3D 镜头"
2972+
no_room_for_caption = "这里没有空间再添加字幕"
29702973
redo = "重做"
29712974
save_failed = "无法保存 {path}。请保持编辑器打开并重试:{error}"
29722975
undo = "撤销"

‎apps/desktop-gpui/src/editor_edits.rs‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2109,6 +2109,83 @@ pub fn insert_scene_segment(timeline: &mut TimelineConfiguration, segment: Scene
21092109
index
21102110
}
21112111

2112+
pub const NEW_CAPTION_LENGTH: f64 = 2.0;
2113+
2114+
pub const MIN_ADDED_CAPTION: f64 = 0.25;
2115+
2116+
/// Adds a caption with no word timings over `output`, in edited-timeline
2117+
/// seconds, to the source-time captions, and switches captions on. Returns
2118+
/// its index in `captions.segments`.
2119+
///
2120+
/// It is written in recording time like every other caption, so it stays on
2121+
/// its moment as clips are trimmed, cut or moved. It ends where its clip does:
2122+
/// past that point the recording jumps, and the caption would land on footage
2123+
/// it was never placed over. `None` when the clip leaves less than
2124+
/// [`MIN_ADDED_CAPTION`] of room, as inside a fullscreen text hold, or when
2125+
/// legacy captions cannot be moved to recording time yet.
2126+
pub fn add_caption_segment(
2127+
project: &mut ProjectConfiguration,
2128+
output: std::ops::Range<f64>,
2129+
id: String,
2130+
text: &str,
2131+
recording_durations: &[f64],
2132+
) -> Option<usize> {
2133+
let timeline = project.timeline.as_ref()?;
2134+
let holds = timeline.hold_windows();
2135+
if holds
2136+
.iter()
2137+
.any(|(start, end)| output.start >= *start && output.start < *end)
2138+
{
2139+
return None;
2140+
}
2141+
let effective = |time: f64| time - held_time_before(&holds, time);
2142+
let (start, end) = (effective(output.start), effective(output.end));
2143+
let mappings = crate::transcript::model::build_mappings(timeline, recording_durations);
2144+
let mapping = mappings
2145+
[..mappings.partition_point(|mapping| mapping.effective_start <= start + 1e-9)]
2146+
.last()?;
2147+
let start = start.max(mapping.effective_start);
2148+
let end = end.min(mapping.effective_end());
2149+
if !start.is_finite() || !end.is_finite() || end - start < MIN_ADDED_CAPTION {
2150+
return None;
2151+
}
2152+
let source = |time: f64| {
2153+
(mapping.source_start + (time - mapping.effective_start) * mapping.timescale) as f32
2154+
};
2155+
let (source_start, source_end) = (source(start), source(end));
2156+
if project
2157+
.captions
2158+
.as_ref()
2159+
.is_some_and(|captions| !captions.source_timed)
2160+
{
2161+
cap_project::synchronize_captions(project, recording_durations);
2162+
}
2163+
let captions = project
2164+
.captions
2165+
.get_or_insert_with(|| cap_project::CaptionsData {
2166+
source_timed: true,
2167+
..Default::default()
2168+
});
2169+
if !captions.source_timed {
2170+
return None;
2171+
}
2172+
captions.settings.enabled = true;
2173+
let index = captions
2174+
.segments
2175+
.partition_point(|segment| segment.start <= source_start);
2176+
captions.segments.insert(
2177+
index,
2178+
cap_project::CaptionSegment {
2179+
id,
2180+
start: source_start,
2181+
end: source_end,
2182+
text: text.to_string(),
2183+
words: Vec::new(),
2184+
},
2185+
);
2186+
Some(index)
2187+
}
2188+
21122189
pub fn find_placement<T: TrackSegmentOps>(
21132190
segments: &[T],
21142191
time: f64,
@@ -5788,3 +5865,142 @@ mod style_image_replacement_tests {
57885865
);
57895866
}
57905867
}
5868+
5869+
#[cfg(test)]
5870+
mod add_caption_tests {
5871+
use super::*;
5872+
5873+
fn project(timeline: serde_json::Value) -> ProjectConfiguration {
5874+
serde_json::from_value(serde_json::json!({ "timeline": timeline })).unwrap()
5875+
}
5876+
5877+
fn add(project: &mut ProjectConfiguration, output: std::ops::Range<f64>) -> Option<usize> {
5878+
add_caption_segment(project, output, "caption-test".into(), "Hello", &[10.0])
5879+
}
5880+
5881+
fn source(project: &ProjectConfiguration) -> Vec<(String, f32, f32)> {
5882+
project
5883+
.captions
5884+
.as_ref()
5885+
.unwrap()
5886+
.segments
5887+
.iter()
5888+
.map(|segment| (segment.text.clone(), segment.start, segment.end))
5889+
.collect()
5890+
}
5891+
5892+
fn track(project: &mut ProjectConfiguration) -> Vec<(String, f64, f64)> {
5893+
cap_project::synchronize_captions(project, &[10.0]);
5894+
project
5895+
.timeline
5896+
.as_ref()
5897+
.unwrap()
5898+
.caption_segments
5899+
.iter()
5900+
.map(|segment| (segment.text.clone(), segment.start, segment.end))
5901+
.collect()
5902+
}
5903+
5904+
#[test]
5905+
fn a_caption_with_no_transcript_turns_captions_on_in_recording_time() {
5906+
let mut project = project(serde_json::json!({
5907+
"segments": [{ "start": 0.0, "end": 10.0, "timescale": 1.0 }],
5908+
"zoomSegments": []
5909+
}));
5910+
assert_eq!(add(&mut project, 1.0..3.0), Some(0));
5911+
let captions = project.captions.as_ref().unwrap();
5912+
assert!(captions.settings.enabled && captions.source_timed);
5913+
assert!(captions.segments[0].words.is_empty());
5914+
assert_eq!(captions.segments[0].id, "caption-test");
5915+
assert_eq!(track(&mut project), [("Hello".to_string(), 1.0, 3.0)]);
5916+
}
5917+
5918+
#[test]
5919+
fn a_caption_follows_clip_speed_and_stops_at_its_clip_edge() {
5920+
let mut project = project(serde_json::json!({
5921+
"segments": [
5922+
{ "start": 0.0, "end": 4.0, "timescale": 2.0 },
5923+
{ "start": 6.0, "end": 10.0, "timescale": 1.0 }
5924+
],
5925+
"zoomSegments": []
5926+
}));
5927+
assert_eq!(add(&mut project, 0.5..1.5), Some(0));
5928+
assert_eq!(add(&mut project, 1.5..3.5), Some(1));
5929+
assert_eq!(add(&mut project, 4.0..6.0), Some(2));
5930+
assert_eq!(
5931+
source(&project),
5932+
[
5933+
("Hello".to_string(), 1.0, 3.0),
5934+
("Hello".to_string(), 3.0, 4.0),
5935+
("Hello".to_string(), 8.0, 10.0),
5936+
]
5937+
);
5938+
assert_eq!(
5939+
track(&mut project),
5940+
[
5941+
("Hello".to_string(), 0.5, 1.5),
5942+
("Hello".to_string(), 1.5, 2.0),
5943+
("Hello".to_string(), 4.0, 6.0),
5944+
]
5945+
);
5946+
assert_eq!(add(&mut project, 1.9..3.0), None);
5947+
}
5948+
5949+
#[test]
5950+
fn a_caption_under_a_fullscreen_hold_is_refused() {
5951+
let mut project = project(serde_json::json!({
5952+
"segments": [{ "start": 0.0, "end": 10.0, "timescale": 1.0 }],
5953+
"zoomSegments": [],
5954+
"textSegments": [{ "start": 2.0, "end": 4.0, "layout": "fullscreen" }]
5955+
}));
5956+
assert_eq!(add(&mut project, 2.5..3.5), None);
5957+
assert_eq!(add(&mut project, 3.0..5.0), None);
5958+
assert_eq!(add(&mut project, 2.0..4.5), None);
5959+
assert!(project.captions.is_none());
5960+
assert_eq!(add(&mut project, 4.0..6.0), Some(0));
5961+
assert_eq!(source(&project), [("Hello".to_string(), 2.0, 4.0)]);
5962+
assert_eq!(track(&mut project), [("Hello".to_string(), 4.0, 6.0)]);
5963+
}
5964+
5965+
#[test]
5966+
fn a_caption_inside_a_later_cut_leaves_with_it_and_comes_back_on_restore() {
5967+
let mut project = project(serde_json::json!({
5968+
"segments": [{ "start": 0.0, "end": 10.0, "timescale": 1.0 }],
5969+
"zoomSegments": []
5970+
}));
5971+
assert_eq!(add(&mut project, 4.0..5.0), Some(0));
5972+
let uncut = project.clone();
5973+
let timeline = project.timeline.as_mut().unwrap();
5974+
crate::transcript::cuts::apply_source_cuts(timeline, &[10.0], &[3.5..5.5]);
5975+
assert!(track(&mut project).is_empty());
5976+
assert_eq!(source(&project), [("Hello".to_string(), 4.0, 5.0)]);
5977+
let mut restored = uncut;
5978+
assert_eq!(track(&mut restored), [("Hello".to_string(), 4.0, 5.0)]);
5979+
}
5980+
5981+
#[test]
5982+
fn legacy_output_time_captions_move_to_recording_time_first() {
5983+
let mut project: ProjectConfiguration = serde_json::from_value(serde_json::json!({
5984+
"timeline": {
5985+
"segments": [
5986+
{ "start": 0.0, "end": 4.0, "timescale": 1.0 },
5987+
{ "start": 6.0, "end": 10.0, "timescale": 1.0 }
5988+
],
5989+
"zoomSegments": []
5990+
},
5991+
"captions": {
5992+
"segments": [{ "id": "segment-0", "start": 5.0, "end": 6.0, "text": "old" }],
5993+
"settings": {}
5994+
}
5995+
}))
5996+
.unwrap();
5997+
assert_eq!(add(&mut project, 1.0..2.0), Some(0));
5998+
assert_eq!(
5999+
source(&project),
6000+
[
6001+
("Hello".to_string(), 1.0, 2.0),
6002+
("old".to_string(), 7.0, 8.0),
6003+
]
6004+
);
6005+
}
6006+
}

0 commit comments

Comments
 (0)