diff --git a/apps/desktop-gpui/src/editor_clips.rs b/apps/desktop-gpui/src/editor_clips.rs index d8cc2448c6..1f16e2946e 100644 --- a/apps/desktop-gpui/src/editor_clips.rs +++ b/apps/desktop-gpui/src/editor_clips.rs @@ -187,6 +187,19 @@ fn ripple_track(track: &mut [T], boundary: f64, shift: f64) } } +fn ripple_keyboard_track( + track: &mut [cap_project::KeyboardTrackSegment], + boundary: f64, + shift: f64, +) { + for segment in track { + if segment.end <= boundary { + continue; + } + segment.remap_times(|time| if time >= boundary { time + shift } else { time }); + } +} + /// `moveClip` (`ClipsSidebar.tsx:639-690`): reorder `timeline.segments`, /// remapping the transitions that survive and -- for each one that does not -- /// rippling every other track across the boundary the removed overlap used to @@ -225,16 +238,16 @@ pub(crate) fn move_clip( .copied() .unwrap_or(0.) + effective.duration; + let boundary = edits::effective_to_output(&timeline.hold_windows(), boundary); timeline .transitions .retain(|candidate| candidate.segment_index != transition.segment_index); - // The source ripples these seven tracks and no others (`:672-682`). ripple_track(&mut timeline.zoom_segments, boundary, effective.duration); ripple_track(&mut timeline.scene_segments, boundary, effective.duration); ripple_track(&mut timeline.mask_segments, boundary, effective.duration); ripple_track(&mut timeline.text_segments, boundary, effective.duration); ripple_track(&mut timeline.caption_segments, boundary, effective.duration); - ripple_track( + ripple_keyboard_track( &mut timeline.keyboard_segments, boundary, effective.duration, @@ -3791,12 +3804,57 @@ mod tests { edge_snap_ratio: 0.25, }]; + config.text_segments = serde_json::from_value(serde_json::json!([{ + "start": 10.0, + "end": 12.0, + "track": 0, + "content": "Hold", + "layout": "fullscreen" + }])) + .unwrap(); + config.keyboard_segments = serde_json::from_value(serde_json::json!([ + { + "id": "before-boundary", + "start": 11.0, + "end": 12.0, + "displayText": "a", + "keys": [{ "key": "a", "timeOffset": 500.0 }] + }, + { + "id": "spanning-boundary", + "start": 11.0, + "end": 13.0, + "displayText": "bc", + "keys": [ + { "key": "b", "timeOffset": 500.0 }, + { "key": "c", "timeOffset": 1500.0 } + ] + } + ])) + .unwrap(); // Moving clip 0 to the end separates the 0|1 pair, dropping the 1s // transition whose boundary sat at offset(1) + 1.0 = 10.0. assert!(move_clip(&mut config, 0, 3)); assert!(config.transitions.is_empty()); assert_eq!(config.zoom_segments[0].start, 16.0); assert_eq!(config.zoom_segments[0].end, 19.0); + assert_eq!( + ( + config.keyboard_segments[0].start, + config.keyboard_segments[0].end, + config.keyboard_segments[0].keys[0].time_offset, + ), + (11.0, 12.0, 500.0) + ); + assert_eq!( + ( + config.keyboard_segments[1].start, + config.keyboard_segments[1].end, + ), + (11.0, 14.0) + ); + assert_eq!(config.keyboard_segments[1].keys[0].time_offset, 500.0); + assert_eq!(config.keyboard_segments[1].keys[1].time_offset, 2500.0); } /// `computeDropIndex` (`:692-703`): the insertion point is after every diff --git a/apps/desktop-gpui/src/editor_edits.rs b/apps/desktop-gpui/src/editor_edits.rs index b465d9af0f..5cf07c9337 100644 --- a/apps/desktop-gpui/src/editor_edits.rs +++ b/apps/desktop-gpui/src/editor_edits.rs @@ -548,11 +548,10 @@ impl TrackSegmentOps for KeyboardTrackSegment { fn set_end(&mut self, value: f64) { self.end = value; } - /// `id: \`kb-split-${Date.now()}-${random}\`` (`ED/context.ts:983`). fn split_tail(&self, at: f64) -> Self { let mut tail = self.clone(); tail.start = self.start + at; - tail.id = split_id("kb"); + tail.id = split_id("kb-edit"); tail } } @@ -907,6 +906,24 @@ pub fn delete_clip_segments(timeline: &mut TimelineConfiguration, indices: &[usi if timeline.segments.len() < 2 { break; } + let offsets = editor_timeline::clip_timeline_offsets(timeline); + let Some(segment) = timeline.segments.get(index) else { + continue; + }; + let clip_start = offsets.get(index).copied().unwrap_or(0.0); + let incoming_transition = timeline + .effective_transition(index) + .map_or(0.0, |transition| transition.duration); + let outgoing_transition = timeline + .effective_transition(index + 1) + .map_or(0.0, |transition| transition.duration); + let cut_start = clip_start + incoming_transition; + let cut_end = clip_start + segment.duration() - outgoing_transition; + let holds = timeline.hold_windows(); + let output_cut_start = effective_to_output_end(&holds, cut_start); + let output_cut_end = effective_to_output_end(&holds, cut_end); + let duration_before = clip_timeline_duration(timeline); + timeline.segments.remove(index); // `transitionsAfterClipDelete` (`ED/clip-transitions.ts:259-275`): the // transitions on both sides of the deleted clip go, and everything @@ -920,11 +937,342 @@ pub fn delete_clip_segments(timeline: &mut TimelineConfiguration, indices: &[usi transition.segment_index -= 1; } } + let gapless_shift = (duration_before - clip_timeline_duration(timeline)).max(0.0); + let held_shift = (output_cut_end - output_cut_start - (cut_end - cut_start)).max(0.0); + ripple_delete_output_tracks( + timeline, + output_cut_start, + output_cut_end, + gapless_shift + held_shift, + ); deleted = true; } deleted } +pub(crate) fn effective_to_output_end(holds: &[(f64, f64)], effective: f64) -> f64 { + let mut output = effective; + for (start, finish) in holds { + if output > *start { + output += finish - start; + } else { + break; + } + } + output +} + +pub(crate) fn effective_to_output(holds: &[(f64, f64)], effective: f64) -> f64 { + let mut output = effective; + for (start, finish) in holds { + if output >= *start { + output += finish - start; + } else { + break; + } + } + output +} + +fn ripple_delete_track( + segments: &mut Vec, + cut_start: f64, + cut_end: f64, + shift: f64, +) { + segments.retain_mut(|segment| { + let Some((start, end)) = + ripple_deleted_bounds(segment.start(), segment.end(), cut_start, cut_end, shift) + else { + return false; + }; + segment.set_start(start); + segment.set_end(end); + true + }); +} + +fn ripple_deleted_bounds( + start: f64, + end: f64, + cut_start: f64, + cut_end: f64, + shift: f64, +) -> Option<(f64, f64)> { + let (start, end) = if end <= cut_start { + (start, end) + } else if start >= cut_start && end <= cut_end { + return None; + } else if start >= cut_end { + (start - shift, end - shift) + } else if start < cut_start && end > cut_end { + (start, end - shift) + } else if start < cut_start { + (start, cut_start) + } else { + let start = cut_end - shift; + (start, (end - shift).max(start)) + }; + (end > start).then_some((start, end)) +} + +fn ripple_relative_keyframes( + keyframes: &mut Vec, + old_start: f64, + new_start: f64, + new_end: f64, + cut_start: f64, + cut_end: f64, + shift: f64, + time: impl for<'a> Fn(&'a mut T) -> &'a mut f64, +) { + keyframes.retain_mut(|keyframe| { + let value = time(keyframe); + let absolute = old_start + *value; + if absolute >= cut_start && absolute < cut_end { + return false; + } + let mapped = if absolute >= cut_end { + absolute - shift + } else { + absolute + }; + *value = (mapped - new_start).clamp(0.0, new_end - new_start); + true + }); +} + +fn ripple_delete_mask_track( + segments: &mut Vec, + cut_start: f64, + cut_end: f64, + shift: f64, +) { + segments.retain_mut(|segment| { + let old_start = segment.start; + let Some((new_start, new_end)) = + ripple_deleted_bounds(old_start, segment.end, cut_start, cut_end, shift) + else { + return false; + }; + ripple_relative_keyframes( + &mut segment.keyframes.position, + old_start, + new_start, + new_end, + cut_start, + cut_end, + shift, + |keyframe| &mut keyframe.time, + ); + ripple_relative_keyframes( + &mut segment.keyframes.size, + old_start, + new_start, + new_end, + cut_start, + cut_end, + shift, + |keyframe| &mut keyframe.time, + ); + ripple_relative_keyframes( + &mut segment.keyframes.intensity, + old_start, + new_start, + new_end, + cut_start, + cut_end, + shift, + |keyframe| &mut keyframe.time, + ); + segment.start = new_start; + segment.end = new_end; + true + }); +} + +fn ripple_delete_camera3d_track( + segments: &mut Vec, + cut_start: f64, + cut_end: f64, + shift: f64, +) { + segments.retain_mut(|segment| { + let old_start = segment.start; + let old_end = segment.end; + if old_end <= cut_start { + return true; + } + if old_start >= cut_end { + segment.start -= shift; + segment.end -= shift; + return true; + } + if old_start >= cut_start && old_end <= cut_end { + return false; + } + let Some((new_start, new_end)) = + ripple_deleted_bounds(old_start, old_end, cut_start, cut_end, shift) + else { + return false; + }; + let retains_left = old_start < cut_start; + let retains_right = old_end > cut_end; + let ripple = Camera3DRipple { + old_start, + new_start, + cut_start, + cut_end, + shift, + retains_left, + retains_right, + }; + for track in segment.tracks.all_tracks_mut() { + ripple_camera3d_keyframes(track, ripple); + } + if retains_left && !retains_right { + segment.transition_out = 0.0; + } else if !retains_left && retains_right { + segment.transition_in = 0.0; + } + segment.start = new_start; + segment.end = new_end; + true + }); +} + +#[derive(Clone, Copy)] +struct Camera3DRipple { + old_start: f64, + new_start: f64, + cut_start: f64, + cut_end: f64, + shift: f64, + retains_left: bool, + retains_right: bool, +} + +fn ripple_camera3d_keyframes( + keyframes: &mut Vec, + ripple: Camera3DRipple, +) { + if keyframes.is_empty() { + return; + } + let mut old = std::mem::take(keyframes); + old.sort_by(|left, right| left.time.total_cmp(&right.time)); + let cut_start_local = ripple.cut_start - ripple.old_start; + let cut_end_local = ripple.cut_end - ripple.old_start; + let boundary = |time: f64, left: bool| { + let previous = old.iter().rev().find(|keyframe| keyframe.time <= time); + let next = old.iter().find(|keyframe| keyframe.time >= time); + cap_project::Camera3DKeyframe { + time, + value: cap_rendering::camera3d::sample_track(0.0, &old, time), + out_easing: if left { + None + } else { + previous.and_then(|keyframe| keyframe.out_easing) + }, + in_easing: if left { + next.and_then(|keyframe| keyframe.in_easing) + } else { + None + }, + } + }; + + keyframes.extend( + old.iter() + .filter(|keyframe| ripple.old_start + keyframe.time < ripple.cut_start) + .cloned(), + ); + if ripple.retains_left { + let mut keyframe = boundary(cut_start_local, true); + keyframe.time = ripple.cut_start - ripple.new_start; + keyframes.push(keyframe); + } + if ripple.retains_right { + let mut keyframe = boundary(cut_end_local, false); + keyframe.time = ripple.cut_end - ripple.shift - ripple.new_start; + keyframes.push(keyframe); + } + keyframes.extend( + old.into_iter() + .filter(|keyframe| ripple.old_start + keyframe.time > ripple.cut_end) + .map(|mut keyframe| { + keyframe.time = ripple.old_start + keyframe.time - ripple.shift - ripple.new_start; + keyframe + }), + ); + keyframes.sort_by(|left, right| left.time.total_cmp(&right.time)); +} + +fn ripple_delete_audio_track( + segments: &mut Vec, + cut_start: f64, + cut_end: f64, + shift: f64, +) { + let mut next = Vec::with_capacity(segments.len()); + for mut segment in std::mem::take(segments) { + if segment.end <= cut_start { + next.push(segment); + } else if segment.start >= cut_start && segment.end <= cut_end { + continue; + } else if segment.start >= cut_end { + segment.start -= shift; + segment.end -= shift; + next.push(segment); + } else if segment.start < cut_start && segment.end > cut_end { + let mut tail = segment.clone(); + let old_start = tail.start; + segment.end = cut_start; + segment.fade_out = 0.0; + tail.start = cut_end - shift; + tail.end -= shift; + tail.trim_start += cut_end - old_start; + tail.fade_in = 0.0; + next.push(segment); + if tail.end > tail.start { + next.push(tail); + } + } else if segment.start < cut_start { + segment.end = cut_start; + segment.fade_out = 0.0; + next.push(segment); + } else { + let old_start = segment.start; + segment.start = cut_end - shift; + segment.end = (segment.end - shift).max(segment.start); + segment.trim_start += cut_end - old_start; + segment.fade_in = 0.0; + if segment.end > segment.start { + next.push(segment); + } + } + } + *segments = next; +} + +fn ripple_delete_output_tracks( + timeline: &mut TimelineConfiguration, + cut_start: f64, + cut_end: f64, + shift: f64, +) { + ripple_delete_track(&mut timeline.zoom_segments, cut_start, cut_end, shift); + ripple_delete_track(&mut timeline.scene_segments, cut_start, cut_end, shift); + ripple_delete_camera3d_track(&mut timeline.camera3d_segments, cut_start, cut_end, shift); + ripple_delete_mask_track(&mut timeline.mask_segments, cut_start, cut_end, shift); + ripple_delete_track(&mut timeline.text_segments, cut_start, cut_end, shift); + ripple_delete_track(&mut timeline.caption_segments, cut_start, cut_end, shift); + timeline + .keyboard_segments + .retain_mut(|segment| segment.ripple_delete(cut_start, cut_end, shift)); + ripple_delete_audio_track(&mut timeline.audio_segments, cut_start, cut_end, shift); +} + /// The seven plain splits. 3D is **not** among them: `splitCamera3DSegment` /// rebuilds both halves' pose tracks around the pose the segment held at the /// cut (`ED/context.ts:640-676`), which needs the keyframe evaluator; see the @@ -939,6 +1287,22 @@ pub fn split_segment( return false; } let min = min_split_duration(kind); + if kind == TrackKind::Keyboard { + let Some(segment) = timeline.keyboard_segments.get(index) else { + return false; + }; + let duration = segment.end - segment.start; + if at < min || duration - at < min { + return false; + } + let Some((head, mut tail)) = segment.split_at(segment.start + at) else { + return false; + }; + tail.id = split_id("kb-edit"); + timeline.keyboard_segments[index] = head; + timeline.keyboard_segments.insert(index + 1, tail); + return true; + } with_track!(timeline, kind, |segments| split_at( segments, index, at, min )) @@ -1832,8 +2196,7 @@ pub fn set_clip_segment_timescale( segment.end += shift(segment.end); } for segment in &mut timeline.keyboard_segments { - segment.start += shift(segment.start); - segment.end += shift(segment.end); + segment.remap_times(|time| time + shift(time)); } for segment in &mut timeline.camera3d_segments { let previous_duration = segment.end - segment.start; @@ -2492,6 +2855,44 @@ mod tests { ); } + #[test] + fn splitting_a_keyboard_segment_partitions_and_rebases_keys() { + let mut project = config(serde_json::json!({ + "timeline": { + "segments": [{ "recordingSegment": 0, "timescale": 1.0, "start": 0.0, "end": 10.0 }], + "zoomSegments": [], + "keyboardSegments": [{ + "id": "typed", + "start": 0.0, + "end": 4.0, + "displayText": "ab", + "keys": [ + { "key": "a", "timeOffset": 500.0 }, + { "key": "b", "timeOffset": 2500.0 } + ] + }] + } + })); + let timeline = project.timeline.as_mut().unwrap(); + + assert!(split_segment(timeline, TrackKind::Keyboard, 0, 2.0)); + assert_eq!(timeline.keyboard_segments.len(), 2); + let left = &timeline.keyboard_segments[0]; + let right = &timeline.keyboard_segments[1]; + assert_eq!( + (left.start, left.end, left.display_text.as_str()), + (0.0, 2.0, "a") + ); + assert_eq!( + (right.start, right.end, right.display_text.as_str()), + (2.0, 4.0, "b") + ); + assert_eq!(left.keys[0].time_offset, 500.0); + assert_eq!(right.keys[0].time_offset, 500.0); + assert_ne!(left.id, right.id); + assert!(right.id.starts_with("kb-edit-split-")); + } + #[test] fn a_split_on_a_boundary_is_refused() { let mut config = two_clip_config(); @@ -2674,6 +3075,254 @@ mod tests { assert_eq!(timeline.segments.len(), 1); } + #[test] + fn deleting_a_clip_removes_its_hold_and_ripples_keyboard_keys() { + let mut project = config(serde_json::json!({ + "timeline": { + "segments": [ + { "recordingSegment": 0, "timescale": 1.0, "start": 0.0, "end": 5.0 }, + { "recordingSegment": 1, "timescale": 1.0, "start": 0.0, "end": 1.0 }, + { "recordingSegment": 2, "timescale": 1.0, "start": 0.0, "end": 4.0 } + ], + "zoomSegments": [{ "start": 8.0, "end": 9.0, "amount": 1.5, "mode": "auto" }], + "textSegments": [{ + "start": 5.25, + "end": 6.25, + "track": 0, + "enabled": true, + "layout": "fullscreen", + "content": "Hold" + }], + "keyboardSegments": [{ + "id": "typed", + "start": 4.5, + "end": 8.5, + "displayText": "abc", + "keys": [ + { "key": "a", "timeOffset": 0.0 }, + { "key": "b", "timeOffset": 1000.0 }, + { "key": "c", "timeOffset": 3500.0 } + ] + }] + } + })); + let timeline = project.timeline.as_mut().unwrap(); + + assert!(delete_clip_segments(timeline, &[1])); + assert_eq!(timeline.segments.len(), 2); + assert!(timeline.text_segments.is_empty()); + assert_eq!( + ( + timeline.zoom_segments[0].start, + timeline.zoom_segments[0].end + ), + (6.0, 7.0) + ); + let keyboard = &timeline.keyboard_segments[0]; + assert_eq!((keyboard.start, keyboard.end), (4.5, 6.5)); + assert_eq!(keyboard.display_text, "ac"); + assert_eq!(keyboard.keys.len(), 2); + assert_eq!(keyboard.keys[0].time_offset, 0.0); + assert_eq!(keyboard.keys[1].time_offset, 1500.0); + } + + #[test] + fn deleting_a_clip_preserves_transition_overlaps_and_boundary_holds() { + let mut project = config(serde_json::json!({ + "timeline": { + "segments": [ + { "recordingSegment": 0, "timescale": 1.0, "start": 0.0, "end": 10.0 }, + { "recordingSegment": 1, "timescale": 1.0, "start": 0.0, "end": 10.0 }, + { "recordingSegment": 2, "timescale": 1.0, "start": 0.0, "end": 10.0 } + ], + "transitions": [ + { "segmentIndex": 1, "type": "cross-fade", "duration": 1.0 }, + { "segmentIndex": 2, "type": "cross-fade", "duration": 1.0 } + ], + "zoomSegments": [], + "textSegments": [ + { "start": 10.0, "end": 12.0, "track": 0, "content": "removed", "layout": "fullscreen" }, + { "start": 20.0, "end": 22.0, "track": 0, "content": "retained", "layout": "fullscreen" } + ], + "keyboardSegments": [ + { + "id": "left-overlap", "start": 9.0, "end": 10.0, + "displayText": "a", "keys": [{ "key": "a", "timeOffset": 500.0 }] + }, + { + "id": "right-overlap", "start": 20.0, "end": 21.0, + "displayText": "b", "keys": [{ "key": "b", "timeOffset": 500.0 }] + } + ], + "audioSegments": [{ + "start": 15.0, "end": 23.0, "track": 0, "path": "/tmp/a.mp3", + "trimStart": 4.0 + }] + } + })); + let timeline = project.timeline.as_mut().unwrap(); + + assert!(delete_clip_segments(timeline, &[1])); + + assert!(timeline.transitions.is_empty()); + assert_eq!(timeline.text_segments.len(), 1); + assert_eq!( + ( + timeline.text_segments[0].start, + timeline.text_segments[0].end + ), + (10.0, 12.0) + ); + assert_eq!( + ( + timeline.keyboard_segments[0].start, + timeline.keyboard_segments[0].end, + timeline.keyboard_segments[0].keys[0].time_offset, + ), + (9.0, 10.0, 500.0) + ); + assert_eq!( + ( + timeline.keyboard_segments[1].start, + timeline.keyboard_segments[1].end, + timeline.keyboard_segments[1].keys[0].time_offset, + ), + (10.0, 11.0, 500.0) + ); + let audio = &timeline.audio_segments[0]; + assert_eq!( + (audio.start, audio.end, audio.trim_start), + (10.0, 13.0, 9.0) + ); + } + + #[test] + fn output_ripple_preserves_audio_and_keyframe_payload_alignment() { + let mut project = two_clip_config(); + let timeline = project.timeline.as_mut().unwrap(); + let mut mask = default_mask_segment(5.5, 9.0, 0); + mask.keyframes.position = vec![ + cap_project::MaskVectorKeyframe { + time: 0.25, + x: 0.2, + y: 0.3, + }, + cap_project::MaskVectorKeyframe { + time: 2.5, + x: 0.7, + y: 0.8, + }, + ]; + timeline.mask_segments.push(mask); + let mut removed_camera = default_camera3d_segment(5.2, 6.2); + removed_camera + .tracks + .tilt_x + .push(cap_project::Camera3DKeyframe { + time: 0.5, + value: 1.0, + out_easing: None, + in_easing: None, + }); + let mut retained_camera = default_camera3d_segment(8.0, 12.0); + retained_camera + .tracks + .tilt_x + .push(cap_project::Camera3DKeyframe { + time: 2.0, + value: 2.0, + out_easing: None, + in_easing: None, + }); + timeline.camera3d_segments = vec![removed_camera, retained_camera]; + let mut audio = default_audio_segment( + 5.5, + 9.0, + 0, + "content/audio/test.wav".into(), + "Test".into(), + Some(12.0), + ); + audio.trim_start = 10.0; + timeline.audio_segments.push(audio); + + ripple_delete_output_tracks(timeline, 5.0, 7.0, 2.0); + + let mask = &timeline.mask_segments[0]; + assert_eq!((mask.start, mask.end), (5.0, 7.0)); + assert_eq!(mask.keyframes.position.len(), 1); + assert_eq!(mask.keyframes.position[0].time, 1.0); + assert_eq!(timeline.camera3d_segments.len(), 1); + let camera = &timeline.camera3d_segments[0]; + assert_eq!((camera.start, camera.end), (6.0, 10.0)); + assert_eq!(camera.tracks.tilt_x[0].time, 2.0); + let audio = &timeline.audio_segments[0]; + assert_eq!((audio.start, audio.end), (5.0, 7.0)); + assert_eq!(audio.trim_start, 11.5); + } + + #[test] + fn camera3d_ripple_drops_cut_keys_and_preserves_both_boundary_poses() { + let mut camera = default_camera3d_segment(0.0, 10.0); + camera.tracks.tilt_x = [2.0, 5.0, 8.0] + .into_iter() + .map(|time| cap_project::Camera3DKeyframe { + time, + value: time, + out_easing: Some([0.0, 0.0]), + in_easing: Some([1.0, 1.0]), + }) + .collect(); + let old_track = camera.tracks.tilt_x.clone(); + let mut segments = vec![camera]; + + ripple_delete_camera3d_track(&mut segments, 3.0, 6.0, 3.0); + + let camera = &segments[0]; + assert_eq!((camera.start, camera.end), (0.0, 7.0)); + let track = &camera.tracks.tilt_x; + assert_eq!( + track + .iter() + .map(|keyframe| (keyframe.time, keyframe.value)) + .collect::>(), + vec![(2.0, 2.0), (3.0, 3.0), (3.0, 6.0), (5.0, 8.0)] + ); + assert_eq!(track[1].in_easing, Some([1.0, 1.0])); + assert_eq!(track[1].out_easing, None); + assert_eq!(track[2].in_easing, None); + assert_eq!(track[2].out_easing, Some([0.0, 0.0])); + assert_eq!( + cap_rendering::camera3d::sample_track(0.0, track, 2.5), + cap_rendering::camera3d::sample_track(0.0, &old_track, 2.5) + ); + let after_join = cap_rendering::camera3d::sample_track(0.0, track, 3.001); + let before_cut = cap_rendering::camera3d::sample_track(0.0, &old_track, 6.001); + assert!((after_join - before_cut).abs() < 1e-9); + } + + #[test] + fn ripple_right_tails_start_at_the_mapped_cut_end() { + assert_eq!( + ripple_deleted_bounds(4.0, 8.0, 3.0, 6.0, 2.0), + Some((4.0, 6.0)) + ); + let mut audio = vec![default_audio_segment( + 4.0, + 8.0, + 0, + "content/audio/test.wav".into(), + "Test".into(), + Some(10.0), + )]; + audio[0].trim_start = 1.0; + + ripple_delete_audio_track(&mut audio, 3.0, 6.0, 2.0); + + assert_eq!((audio[0].start, audio[0].end), (4.0, 6.0)); + assert_eq!(audio[0].trim_start, 3.0); + } + #[test] fn deleting_a_masks_only_lane_renumbers_the_lanes_above_it() { let mut config = config(serde_json::json!({ @@ -2767,11 +3416,26 @@ mod tests { fn setting_clip_timescale_ripples_later_tracks() { let mut project = zoom_fixture(); let timeline = project.timeline.as_mut().unwrap(); + timeline.keyboard_segments = serde_json::from_value(serde_json::json!([{ + "id": "typed", + "start": 2.0, + "end": 8.0, + "displayText": "ab", + "keys": [ + { "key": "a", "timeOffset": 1000.0 }, + { "key": "b", "timeOffset": 5000.0 } + ] + }])) + .unwrap(); assert!(set_clip_segment_timescale(timeline, 0, 2.0)); assert_eq!(timeline.segments[0].timescale, 2.0); assert!((timeline.zoom_segments[0].start - 1.0).abs() < 1e-9); assert!((timeline.zoom_segments[0].end - 2.5).abs() < 1e-9); assert!((timeline.zoom_segments[1].start - 10.0).abs() < 1e-9); + let keyboard = &timeline.keyboard_segments[0]; + assert_eq!((keyboard.start, keyboard.end), (1.0, 4.0)); + assert_eq!(keyboard.keys[0].time_offset, 500.0); + assert_eq!(keyboard.keys[1].time_offset, 2500.0); assert!(!set_clip_segment_timescale(timeline, 0, 2.0)); } diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index a61daa1c11..cd65197d8f 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -24,7 +24,8 @@ use cap_project::CaptionsData; use cap_project::{ BackgroundBlurConfig, BackgroundBlurMode, CameraShape, CameraXPosition, CameraYPosition, CaptionSegment, CaptionSettings, CornerStyle, CursorAnimationStyle, CursorRippleConfig, - KeyboardData, KeyboardSettings, ProjectConfiguration, ShadowConfiguration, StereoMode, + KeyboardData, KeyboardSettings, ProjectConfiguration, RecordingMeta, ShadowConfiguration, + StereoMode, }; use gpui::{ AnyElement, Bounds, Context, EntityId, FontWeight, Hsla, InteractiveElement, IntoElement, @@ -528,6 +529,15 @@ pub fn keyboard_settings(project: &ProjectConfiguration) -> KeyboardSettings { .unwrap_or_default() } +fn keyboard_generation_settings_fingerprint(settings: &KeyboardSettings) -> (u64, u32, bool, bool) { + ( + settings.grouping_threshold_ms.to_bits(), + settings.linger_duration.to_bits(), + settings.show_modifiers, + settings.show_special_keys, + ) +} + /// `updateCaptionSetting` (`CaptionsTab.tsx:321-338`): a settings write is a /// no-op when the project has no captions block at all, which is what the /// source's `if (!project?.captions) return` says. @@ -1825,6 +1835,116 @@ impl EditorWindow { // -- Keyboard ------------------------------------------------------------ + fn generate_keyboard_segments_clicked(&mut self, window: &mut Window, cx: &mut Context) { + if self.generating_keyboard { + return; + } + let Some(timeline) = self.project.timeline.clone() else { + self.keyboard_generation_message = Some("The project timeline is unavailable.".into()); + cx.notify(); + return; + }; + let Ok(timeline_fingerprint) = serde_json::to_vec(&timeline) else { + self.keyboard_generation_message = + Some("The project timeline could not be read.".into()); + cx.notify(); + return; + }; + let settings = keyboard_settings(&self.project); + let settings_fingerprint = keyboard_generation_settings_fingerprint(&settings); + let path = self.project_path.clone(); + self.generating_keyboard = true; + self.keyboard_generation_message = None; + cx.notify(); + window.refresh(); + + cx.spawn_in(window, async move |this, cx| { + let result = cx + .background_executor() + .spawn(async move { + let meta = RecordingMeta::load_for_project(&path) + .map_err(|error| format!("Failed to load recording data: {error}"))?; + cap_project::generate_project_keyboard_segments(&meta, &timeline, &settings) + }) + .await; + + this.update_in(cx, |this, window, cx| { + this.generating_keyboard = false; + let segments = match result { + Ok(segments) => segments, + Err(error) => { + tracing::error!("keyboard generation failed: {error}"); + this.keyboard_generation_message = Some(error); + cx.notify(); + window.refresh(); + return; + } + }; + let timeline_unchanged = this + .project + .timeline + .as_ref() + .and_then(|timeline| serde_json::to_vec(timeline).ok()) + .is_some_and(|fingerprint| fingerprint == timeline_fingerprint); + let settings_unchanged = + keyboard_generation_settings_fingerprint(&keyboard_settings(&this.project)) + == settings_fingerprint; + if !timeline_unchanged || !settings_unchanged { + this.keyboard_generation_message = Some( + "The timeline or keyboard settings changed during generation. Try again." + .into(), + ); + cx.notify(); + window.refresh(); + return; + } + let has_segments = !segments.is_empty(); + let already_empty = this + .project + .timeline + .as_ref() + .is_none_or(|timeline| timeline.keyboard_segments.is_empty()); + if !has_segments && already_empty { + this.keyboard_generation_message = + Some("No recorded keyboard presses were found.".into()); + cx.notify(); + window.refresh(); + return; + } + + if has_segments { + this.tracks.keyboard = true; + } + this.edit_project("keyboard-generate", window, cx, move |project| { + if has_segments { + let keyboard = project.keyboard.get_or_insert_with(KeyboardData::default); + keyboard.settings.enabled = true; + } + let Some(timeline) = project.timeline.as_mut() else { + return false; + }; + timeline.keyboard_segments = segments; + true + }); + if has_segments { + this.keyboard_generation_message = None; + } else { + if this.selection.as_ref().is_some_and(|selection| { + selection.track == crate::editor_timeline::TrackKind::Keyboard + }) { + this.set_selection(None, cx); + } + this.keyboard_generation_message = + Some("No recorded keyboard presses were found.".into()); + cx.notify(); + window.refresh(); + } + }) + .ok(); + }) + .detach(); + } + /// `KeyboardTab` (`KeyboardTab.tsx:128-553`). pub(crate) fn render_keyboard_tab(&self, cx: &mut Context) -> AnyElement { let theme = self.theme; @@ -2034,9 +2154,6 @@ impl EditorWindow { ), ), ) - // `Generate Keyboard Segments` -- `commands.generateKeyboardSegments` - // reads the recording's own key log through a Tauri command this - // app does not have, so the button renders and says so. .child( div().pt(px(8.)).child( ui::Button::plain( @@ -2045,15 +2162,27 @@ impl EditorWindow { ui::ButtonVariant::Primary, ui::ButtonSize::Md, ) - .label(if has_segments { + .label(if self.generating_keyboard { + "Generating Keyboard Segments..." + } else if has_segments { "Regenerate Keyboard Segments" } else { "Generate Keyboard Segments" }) .full_width() - .disabled(true), + .disabled(self.generating_keyboard || self.project.timeline.is_none()) + .on_click(cx.listener(|this, _, window, cx| { + this.generate_keyboard_segments_clicked(window, cx); + })), ), ) + .children(self.keyboard_generation_message.as_ref().map(|message| { + div() + .text_size(px(12.)) + .text_color(Hsla::from(theme.gray_10)) + .child(message.clone()) + .into_any_element() + })) .children((!has_segments).then(|| { div() .py(px(16.)) diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 465f718f77..44e39b6ac9 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -1349,6 +1349,8 @@ pub struct EditorWindow { auto_zoom_message: Option<&'static str>, zoom_prompt_dismissed: bool, hovering_generate_zoom: bool, + pub(crate) generating_keyboard: bool, + pub(crate) keyboard_generation_message: Option, clip_speed: Option, timeline_scroll: gpui::ScrollHandle, minimap_drag: Option, @@ -1462,11 +1464,7 @@ pub struct EditorWindow { /// exactly as the Solid dialogs replace the dropdown. presets_menu: Option, preset_dialog: Option, - /// The last caption-projection signature -- clip list, transitions, text - /// holds and the caption source master -- so `project_changed` only - /// re-derives `timeline.captionSegments` when one of those moved, the - /// same inputs the Solid effect keys on (`ED/context.ts:1630-1661`). - caption_track_sig: Option, + caption_sync_signature: Option, pub(crate) export: Option, /// The Clips layout mode (`ClipsSidebar.tsx`): while open, the config /// sidebar's column draws the clips sidebar instead. @@ -1710,6 +1708,8 @@ impl EditorWindow { auto_zoom_message: None, zoom_prompt_dismissed: false, hovering_generate_zoom: false, + generating_keyboard: false, + keyboard_generation_message: None, clip_speed: None, timeline_scroll: gpui::ScrollHandle::new(), minimap_drag: None, @@ -1756,7 +1756,7 @@ impl EditorWindow { timeline_resize: None, presets_menu: None, preset_dialog: None, - caption_track_sig: None, + caption_sync_signature: None, poster: None, export: None, clips: crate::editor_clips::ClipsState::default(), @@ -1851,6 +1851,7 @@ impl EditorWindow { cx: &mut Context, ) { self.project = config; + self.synchronize_caption_track(true); self.history = ProjectHistory::new(self.project.clone()); self.tracks = TrackLanes::from_project(&self.project, self.has_camera); self.rebuild_timeline(); @@ -1919,6 +1920,19 @@ impl EditorWindow { /// * **the disk** -- `scheduleProjectConfigSave`'s 250ms debounce /// (`ED/context.ts:1235-1244`), so a drag writes once rather than sixty /// times. + fn edit_caption_project( + &mut self, + change: impl FnOnce(&mut ProjectConfiguration) -> bool, + window: &mut Window, + cx: &mut Context, + ) -> bool { + if !change(&mut self.project) { + return false; + } + self.project_changed(window, cx); + true + } + fn edit( &mut self, change: impl FnOnce(&mut TimelineConfiguration) -> bool, @@ -1980,9 +1994,7 @@ impl EditorWindow { if !self.project_ready() { return; } - // Before `history.record`, so the re-projected caption track is part - // of the same undo entry as the edit that moved it. - self.rederive_caption_track(); + self.synchronize_caption_track(false); self.history.record(&self.project); self.rebuild_timeline(); self.publish_project(); @@ -1995,52 +2007,27 @@ impl EditorWindow { if !self.project_ready() { return; } + self.synchronize_caption_track(false); self.publish_project(); cx.notify(); } - /// The Solid effect at `ED/context.ts:1630-1704`: whenever the clip list, - /// transitions, text holds or the caption source master move, re-project - /// `timeline.captionSegments` through the edit list so captions follow - /// trims, deletes, reorders and inserts with no re-transcription. - fn rederive_caption_track(&mut self) { - let Some(sig) = self.caption_projection_signature() else { - self.caption_track_sig = None; - return; - }; - if self.caption_track_sig == Some(sig) { + fn synchronize_caption_track(&mut self, force: bool) { + let signature = self.caption_projection_signature(); + if !force && self.caption_sync_signature == signature { return; } - self.caption_track_sig = Some(sig); - let Some(summary) = self.summary() else { - return; - }; - let durations = summary.clip_display_durations.clone(); - let Some(captions) = self.project.captions.as_ref() else { - return; - }; - let segments = captions.segments.clone(); - if let Some(timeline) = self.project.timeline.as_mut() { - timeline.caption_segments = crate::transcription::derive_caption_track_segments( - &segments, timeline, &durations, - ); - } + cap_project::synchronize_captions(&mut self.project, &self.clip_display_durations); + self.caption_sync_signature = self.caption_projection_signature(); } - /// The effect's dependency signature (`ED/context.ts:1632-1661`): caption - /// sources, clip segments, transitions and hold windows. `None` when - /// there is nothing to project -- no captions, legacy non-source-timed - /// data, or no timeline. fn caption_projection_signature(&self) -> Option { use std::hash::{Hash, Hasher}; let captions = self.project.captions.as_ref()?; - if !captions.source_timed || captions.segments.is_empty() { - return None; - } let timeline = self.project.timeline.as_ref()?; - let mut hasher = std::hash::DefaultHasher::new(); + captions.source_timed.hash(&mut hasher); for segment in &captions.segments { segment.id.hash(&mut hasher); segment.start.to_bits().hash(&mut hasher); @@ -2145,6 +2132,7 @@ impl EditorWindow { || crate::editor_sidebar::is_none_background(&self.project) != crate::editor_sidebar::is_none_background(&config); self.project = config; + self.synchronize_caption_track(false); self.rebuild_timeline(); if self.animated_gradient_config().is_some() { self.sidebar.source_tab = crate::editor_sidebar::initial_source_tab(&self.project); @@ -3854,6 +3842,175 @@ pub(crate) struct Camera3DSetup { type GhostClipLayout = (Vec<(f64, f64)>, Option<(f64, f64)>); +fn caption_text_from_words(words: &[cap_project::CaptionWord]) -> String { + let mut text = String::new(); + for word in words { + let value = word.text.trim(); + if value.is_empty() { + continue; + } + let attaches = value.chars().next().is_some_and(|value| { + matches!( + value, + ',' | '.' + | '!' + | '?' + | ';' + | ':' + | '%' + | ')' + | ']' + | '}' + | '\'' + | '’' + | '、' + | '。' + | '！' + | '？' + | '；' + | '：' + | '，' + ) + }); + if !text.is_empty() && !attaches { + text.push(' '); + } + text.push_str(value); + } + text +} + +fn fresh_caption_source_id(project: &ProjectConfiguration) -> String { + use std::hash::{BuildHasher, Hasher}; + + loop { + let millis = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_millis()) + .unwrap_or_default(); + let random = std::collections::hash_map::RandomState::new() + .build_hasher() + .finish(); + let id = format!("cap-split-{millis}-{random:x}"); + if project + .captions + .as_ref() + .is_none_or(|captions| captions.segments.iter().all(|segment| segment.id != id)) + { + return id; + } + } +} + +fn split_caption_segment( + project: &mut ProjectConfiguration, + index: usize, + at: f64, + recording_durations: &[f64], +) -> bool { + let Some(timeline) = project.timeline.as_ref() else { + return false; + }; + let Some(track) = timeline.caption_segments.get(index) else { + return false; + }; + let source_id = cap_project::source_caption_id(&track.id); + let Some((source_index, source)) = project + .captions + .as_ref() + .and_then(|captions| { + captions + .segments + .iter() + .enumerate() + .find(|(_, segment)| segment.id == source_id) + }) + .map(|(index, segment)| (index, segment.clone())) + else { + return false; + }; + let split_output = track.start + at; + let source_range = Some((f64::from(source.start), f64::from(source.end))); + let Some(split_source) = crate::transcription::map_edited_time_to_source( + split_output, + timeline, + recording_durations, + source_range, + ) else { + return false; + }; + if split_source <= f64::from(source.start) || split_source >= f64::from(source.end) { + return false; + } + let tail_id = fresh_caption_source_id(project); + let Some(timeline) = project.timeline.as_mut() else { + return false; + }; + if !edits::split_segment(timeline, TrackKind::Caption, index, at) { + return false; + } + timeline.caption_segments[index + 1].id.clone_from(&tail_id); + + let split_source = split_source as f32; + let mut head = source.clone(); + let mut tail = source; + head.end = split_source; + tail.id = tail_id; + tail.start = split_source; + if !head.words.is_empty() { + let words = std::mem::take(&mut head.words); + tail.words.clear(); + for word in words { + if word.start < split_source { + let mut value = word.clone(); + value.end = value.end.min(split_source); + if value.end > value.start { + head.words.push(value); + } + } + if word.end > split_source { + let mut value = word; + value.start = value.start.max(split_source); + if value.end > value.start { + tail.words.push(value); + } + } + } + head.text = caption_text_from_words(&head.words); + tail.text = caption_text_from_words(&tail.words); + } + let Some(captions) = project.captions.as_mut() else { + return false; + }; + captions + .segments + .splice(source_index..=source_index, [head, tail]); + true +} + +fn delete_caption_segments(project: &mut ProjectConfiguration, indices: &[usize]) -> bool { + let Some(timeline) = project.timeline.as_ref() else { + return false; + }; + let source_ids = indices + .iter() + .filter_map(|index| timeline.caption_segments.get(*index)) + .map(|segment| cap_project::source_caption_id(&segment.id).to_string()) + .collect::>(); + let Some(timeline) = project.timeline.as_mut() else { + return false; + }; + if !edits::delete_segments(timeline, TrackKind::Caption, indices) { + return false; + } + if let Some(captions) = project.captions.as_mut() { + captions + .segments + .retain(|segment| !source_ids.contains(&segment.id)); + } + true +} + impl EditorWindow { fn clamp_timeline_height(&self, value: f32, viewport_height: f32) -> f32 { let available = (viewport_height - HEADER_HEIGHT - 8.).max(MIN_TIMELINE_HEIGHT); @@ -4577,6 +4734,17 @@ impl EditorWindow { self.seek_to_time(drag.press_time, cx); } + if drag.moved && drag.track == TrackKind::Caption { + crate::transcription::write_caption_edit_to_source( + &mut self.project, + drag.index, + &self.clip_display_durations, + ); + self.synchronize_caption_track(false); + self.rebuild_timeline(); + self.publish_project(); + } + if drag.paused { let config = self.project.clone(); self.history.resume(&config); @@ -4758,11 +4926,21 @@ impl EditorWindow { return; } let local = ((x - left) / width) * (segment.end - segment.start); - if self.edit( - |timeline| edits::split_segment(timeline, kind, index, local), - window, - cx, - ) { + let split = if kind == TrackKind::Caption { + let recording_durations = self.clip_display_durations.clone(); + self.edit_caption_project( + |project| split_caption_segment(project, index, local, &recording_durations), + window, + cx, + ) + } else { + self.edit( + |timeline| edits::split_segment(timeline, kind, index, local), + window, + cx, + ) + }; + if split { self.note_edit("split", Some(kind)); } } @@ -4778,11 +4956,19 @@ impl EditorWindow { let Some(selection) = self.selection.clone() else { return; }; - let deleted = self.edit( - |timeline| edits::delete_segments(timeline, selection.track, &selection.indices), - window, - cx, - ); + let deleted = if selection.track == TrackKind::Caption { + self.edit_caption_project( + |project| delete_caption_segments(project, &selection.indices), + window, + cx, + ) + } else { + self.edit( + |timeline| edits::delete_segments(timeline, selection.track, &selection.indices), + window, + cx, + ) + }; if deleted { self.set_selection(None, cx); self.note_edit("delete", Some(selection.track)); @@ -9248,6 +9434,178 @@ fn hex_to_color(rgba: [u8; 4]) -> cap_project::Color { mod tests { use super::*; + #[test] + fn caption_split_and_delete_update_the_source_master() { + let words = vec![ + cap_project::CaptionWord { + text: "hello".into(), + start: 1.0, + end: 2.0, + }, + cap_project::CaptionWord { + text: "world".into(), + start: 3.0, + end: 4.0, + }, + ]; + let track = cap_project::CaptionTrackSegment { + id: "spoken".into(), + start: 1.0, + end: 4.0, + text: "hello world".into(), + words: words.clone(), + fade_duration_override: None, + linger_duration_override: None, + position_override: None, + color_override: None, + background_color_override: None, + font_size_override: None, + }; + let mut project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + segments: vec![cap_project::TimelineSegment { + recording_clip: 0, + start: 0.0, + end: 10.0, + timescale: 1.0, + name: None, + speed_audio_mode: None, + }], + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: vec![track], + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: Vec::new(), + }), + captions: Some(cap_project::CaptionsData { + segments: vec![cap_project::CaptionSegment { + id: "spoken".into(), + start: 1.0, + end: 4.0, + text: "hello world".into(), + words, + }], + source_timed: true, + ..Default::default() + }), + ..Default::default() + }; + + assert!(split_caption_segment(&mut project, 0, 1.5, &[10.0])); + cap_project::synchronize_captions(&mut project, &[10.0]); + let captions = project.captions.as_ref().unwrap(); + assert_eq!(captions.segments.len(), 2); + assert_eq!(captions.segments[0].text, "hello"); + assert_eq!(captions.segments[1].text, "world"); + assert_eq!(project.timeline.as_ref().unwrap().caption_segments.len(), 2); + + assert!(delete_caption_segments(&mut project, &[1])); + cap_project::synchronize_captions(&mut project, &[10.0]); + assert_eq!(project.captions.as_ref().unwrap().segments.len(), 1); + assert_eq!(project.timeline.as_ref().unwrap().caption_segments.len(), 1); + } + + #[test] + fn caption_split_delete_survives_a_repeated_trimmed_edl() { + let mut project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + segments: vec![ + cap_project::TimelineSegment { + recording_clip: 0, + start: 0.5, + end: 4.5, + timescale: 1.0, + name: None, + speed_audio_mode: None, + }, + cap_project::TimelineSegment { + recording_clip: 0, + start: 0.5, + end: 4.5, + timescale: 1.0, + name: None, + speed_audio_mode: None, + }, + ], + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: Vec::new(), + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: Vec::new(), + }), + captions: Some(cap_project::CaptionsData { + segments: vec![cap_project::CaptionSegment { + id: "spoken".into(), + start: 1.0, + end: 4.0, + text: "hello world".into(), + words: vec![ + cap_project::CaptionWord { + text: "hello".into(), + start: 1.0, + end: 2.0, + }, + cap_project::CaptionWord { + text: "world".into(), + start: 3.0, + end: 4.0, + }, + ], + }], + source_timed: true, + ..Default::default() + }), + ..Default::default() + }; + cap_project::synchronize_captions(&mut project, &[10.0]); + assert_eq!( + project.timeline.as_ref().unwrap().caption_segments[0].id, + "spoken::edl0" + ); + + assert!(split_caption_segment(&mut project, 0, 1.5, &[10.0])); + let tail_source_id = project.captions.as_ref().unwrap().segments[1].id.clone(); + assert_ne!(tail_source_id, "spoken"); + assert_eq!( + cap_project::source_caption_id(&tail_source_id), + tail_source_id + ); + assert_eq!( + project.timeline.as_ref().unwrap().caption_segments[1].id, + tail_source_id + ); + + cap_project::synchronize_captions(&mut project, &[10.0]); + assert_eq!(project.timeline.as_ref().unwrap().caption_segments.len(), 4); + let tail_index = project + .timeline + .as_ref() + .unwrap() + .caption_segments + .iter() + .position(|segment| cap_project::source_caption_id(&segment.id) == tail_source_id) + .unwrap(); + assert!(delete_caption_segments(&mut project, &[tail_index])); + cap_project::synchronize_captions(&mut project, &[10.0]); + + assert_eq!(project.captions.as_ref().unwrap().segments.len(), 1); + assert_eq!(project.captions.as_ref().unwrap().segments[0].id, "spoken"); + let remaining = &project.timeline.as_ref().unwrap().caption_segments; + assert_eq!(remaining.len(), 2); + assert!( + remaining + .iter() + .all(|segment| cap_project::source_caption_id(&segment.id) == "spoken") + ); + } fn open_sidebar_menu_for_test( kind: crate::editor_tabs::SidebarMenu, ) -> Option { diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index 4f27fe5943..c134a70625 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -34,11 +34,13 @@ use std::{ time::Duration, }; +use cap_editor::{TranscriptionAudioSource, TranscriptionAudioTake, append_transcription_audio}; use cap_project::{ CaptionSegment, CaptionSettings, CaptionTrackSegment, CaptionWord, CaptionsData, ProjectConfiguration, RecordingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, }; +use cap_rendering::Video; use ffmpeg::{ ChannelLayout, codec as avcodec, format::{self as avformat}, @@ -1006,24 +1008,6 @@ fn append_resampled_frame( Ok(()) } -/// `convert_to_mono` (`captions.rs:2701-2718`). -fn convert_to_mono(samples: &[f32], channels: usize) -> Vec { - if channels <= 1 { - return samples.to_vec(); - } - samples - .chunks_exact(channels) - .map(|frame| frame.iter().sum::() / channels as f32) - .collect() -} - -/// `mix_samples` (`captions.rs:2720-2726`): average, over the shorter length. -fn mix_samples(dest: &mut [f32], source: &[f32]) { - for (dest_sample, source_sample) in dest.iter_mut().zip(source) { - *dest_sample = (*dest_sample + *source_sample) * 0.5; - } -} - /// `normalize_audio_for_transcription` (`captions.rs:885-915`). fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 { if samples.is_empty() { @@ -1057,12 +1041,6 @@ fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 { gain } -fn push_audio_source(sources: &mut Vec, path: PathBuf) { - if path.exists() && !sources.contains(&path) { - sources.push(path); - } -} - /// The recording-directory arm of `extract_audio_from_video` /// (`captions.rs:250-517`): per segment, decode system audio then mic /// (`captions.rs:281-283`), downmix each to mono, average them together, @@ -1076,64 +1054,97 @@ fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(), return Err("Only studio recordings can be transcribed".to_string()); }; - let mut segment_sources: Vec> = Vec::new(); + studio.ensure_ordinary_media_access(&meta.project_path)?; + let mut final_samples = Vec::new(); + let mut any_audio = false; match studio { StudioRecordingMeta::SingleSegment { segment } => { + let display_duration = Video::new( + segment.display.path.to_path(&meta.project_path), + segment.display.start_time.unwrap_or_default(), + ) + .map_err(|error| format!("Failed to read display video: {error}"))? + .duration; let mut sources = Vec::new(); if let Some(audio) = &segment.audio { - push_audio_source(&mut sources, meta.path(&audio.path)); - } - if !sources.is_empty() { - segment_sources.push(sources); + let path = meta.path(&audio.path); + if path.exists() { + match decode_audio_file(&path) { + Ok((samples, channels)) => { + any_audio = true; + sources.push(TranscriptionAudioSource { + samples, + channels, + offset_secs: 0.0, + }); + } + Err(error) => tracing::warn!( + path = %path.display(), + "Failed to process audio source: {error}" + ), + } + } } + let take = TranscriptionAudioTake { + display_duration_secs: display_duration, + sources, + }; + append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE) + .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?; } StudioRecordingMeta::MultipleSegments { inner } => { - for segment in &inner.segments { + for (segment_idx, segment) in inner.segments.iter().enumerate() { + let display_duration = Video::new( + segment.display.path.to_path(&meta.project_path), + segment.display.start_time.unwrap_or_default(), + ) + .map_err(|error| { + format!("Failed to read display video for segment {segment_idx}: {error}") + })? + .duration; + let offsets = segment.calculate_audio_offsets(); let mut sources = Vec::new(); + let mut push_source = |audio: &cap_project::AudioMeta, offset_secs: f64| { + let path = meta.path(&audio.path); + if !path.exists() { + return; + } + match decode_audio_file(&path) { + Ok((samples, channels)) => { + any_audio = true; + sources.push(TranscriptionAudioSource { + samples, + channels, + offset_secs, + }); + } + Err(error) => tracing::warn!( + path = %path.display(), + "Failed to process audio source: {error}" + ), + } + }; + if let Some(system_audio) = &segment.system_audio { - push_audio_source(&mut sources, meta.path(&system_audio.path)); + push_source(system_audio, f64::from(offsets.system_audio)); } if let Some(mic) = &segment.mic { - push_audio_source(&mut sources, meta.path(&mic.path)); - } - if !sources.is_empty() { - segment_sources.push(sources); + push_source(mic, f64::from(offsets.mic)); } + let take = TranscriptionAudioTake { + display_duration_secs: display_duration, + sources, + }; + append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE) + .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?; } } } - if segment_sources.is_empty() { + if !any_audio { return Err("No audio sources found in the recording metadata".to_string()); } - let mut final_samples: Vec = Vec::new(); - - for sources in &segment_sources { - let mut segment_samples: Vec = Vec::new(); - - for source in sources { - match decode_audio_file(source) { - Ok((samples, channels)) => { - let mono_samples = convert_to_mono(&samples, channels); - if segment_samples.is_empty() { - segment_samples = mono_samples; - } else { - mix_samples(&mut segment_samples, &mono_samples); - } - } - Err(error) => { - tracing::warn!( - path = %source.display(), - "Failed to process audio source: {error}" - ); - } - } - } - - final_samples.extend(segment_samples); - } - if final_samples.is_empty() { return Err("Failed to process any audio sources".to_string()); } @@ -1814,49 +1825,8 @@ fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { // Track derivation -- deriveCaptionTrackSegments, in Rust // --------------------------------------------------------------------------- -/// `CAPTION_EDL_SEPARATOR` (`captions.ts:151`). -const CAPTION_EDL_SEPARATOR: &str = "::edl"; - -/// `sourceCaptionId` (`captions.ts:166-169`). pub fn source_caption_id(track_id: &str) -> &str { - track_id - .find(CAPTION_EDL_SEPARATOR) - .map_or(track_id, |index| &track_id[..index]) -} - -fn mapped_caption_segment_id(base_id: &str, index: usize, total: usize) -> String { - if total == 1 { - base_id.to_string() - } else { - format!("{base_id}{CAPTION_EDL_SEPARATOR}{index}") - } -} - -/// `clampCaptionSegmentWords` (`captions.ts:36-52`). -fn clamp_caption_segment_words(segment: &CaptionSegment) -> CaptionSegment { - if segment.words.is_empty() { - return segment.clone(); - } - - let clamped_words: Vec = segment - .words - .iter() - .map(|word| CaptionWord { - text: word.text.clone(), - start: word.start, - end: word.end.min(word.start + MAX_CAPTION_WORD_DURATION), - }) - .collect(); - - let last_word_end = clamped_words.last().map_or(segment.end, |word| word.end); - - CaptionSegment { - id: segment.id.clone(), - start: segment.start, - end: segment.end.min(last_word_end), - text: segment.text.clone(), - words: clamped_words, - } + cap_project::source_caption_id(track_id) } struct SourceToEditedMapping { @@ -1901,220 +1871,12 @@ fn build_source_to_edited_mappings( .collect() } -/// `mapTimeRangeWithinMapping` (`captions.ts:130-149`). -fn map_time_range_within_mapping( - start: f64, - end: f64, - mapping: &SourceToEditedMapping, -) -> Option<(f64, f64)> { - let overlap_start = start.max(mapping.source_start); - let overlap_end = end.min(mapping.source_end); - if overlap_start >= overlap_end { - return None; - } - Some(( - mapping.edited_start + (overlap_start - mapping.source_start) / mapping.timescale, - mapping.edited_start + (overlap_end - mapping.source_start) / mapping.timescale, - )) -} - -/// `effectiveToOutput` (`timeline-holds.ts:54-64`). -fn effective_to_output(holds: &[(f64, f64)], effective: f64) -> f64 { - let mut output = effective; - for (start, end) in holds { - if output >= *start { - output += end - start; - } else { - break; - } - } - output -} - -/// `effectiveToOutputEnd` (`timeline-holds.ts:70-80`): an end landing exactly -/// on a hold boundary binds to the content before the pause. -fn effective_to_output_end(holds: &[(f64, f64)], effective: f64) -> f64 { - let mut output = effective; - for (start, end) in holds { - if output > *start { - output += end - start; - } else { - break; - } - } - output -} - -struct MappedCaption { - id: String, - start: f64, - end: f64, - text: String, - words: Vec, -} - -/// `mapCaptionsToEditedTimeline` (`captions.ts:171-273`). -fn map_captions_to_edited_timeline( - raw_segments: &[CaptionSegment], - timeline: &TimelineConfiguration, - recording_durations: &[f64], -) -> Vec { - let sanitized: Vec = raw_segments - .iter() - .map(clamp_caption_segment_words) - .collect(); - - if timeline.segments.is_empty() || recording_durations.is_empty() { - return sanitized - .into_iter() - .map(|segment| MappedCaption { - id: segment.id, - start: f64::from(segment.start), - end: f64::from(segment.end), - text: segment.text, - words: segment.words, - }) - .collect(); - } - - let mappings = build_source_to_edited_mappings(timeline, recording_durations); - let holds = timeline.hold_windows(); - let hold_adjusted = |start: f64, end: f64| { - if holds.is_empty() { - (start, end) - } else { - ( - effective_to_output(&holds, start), - effective_to_output_end(&holds, end), - ) - } - }; - - let mut result = Vec::new(); - - for caption in &sanitized { - let mut mapped_caption_segments: Vec = Vec::new(); - - for mapping in &mappings { - if !caption.words.is_empty() { - let mut mapped_words = Vec::new(); - for word in &caption.words { - let Some((start, end)) = map_time_range_within_mapping( - f64::from(word.start), - f64::from(word.end), - mapping, - ) else { - continue; - }; - let (start, end) = hold_adjusted(start, end); - mapped_words.push(CaptionWord { - text: word.text.clone(), - start: start as f32, - end: end as f32, - }); - } - - if mapped_words.is_empty() { - continue; - } - - let start = mapped_words - .first() - .map_or(f64::from(caption.start), |word| f64::from(word.start)); - let end = mapped_words - .last() - .map_or(f64::from(caption.end), |word| f64::from(word.end)); - mapped_caption_segments.push(MappedCaption { - id: caption.id.clone(), - start, - end, - text: caption_text_from_words(&mapped_words), - words: mapped_words, - }); - } else { - let Some((start, end)) = map_time_range_within_mapping( - f64::from(caption.start), - f64::from(caption.end), - mapping, - ) else { - continue; - }; - let (start, end) = hold_adjusted(start, end); - mapped_caption_segments.push(MappedCaption { - id: caption.id.clone(), - start, - end, - text: caption.text.clone(), - words: Vec::new(), - }); - } - } - - let total = mapped_caption_segments.len(); - for (index, mut segment) in mapped_caption_segments.into_iter().enumerate() { - segment.id = mapped_caption_segment_id(&caption.id, index, total); - result.push(segment); - } - } - - result -} - -/// `deriveCaptionTrackSegments` (`captions.ts:323-366`): project the -/// source-time caption master through the current edit list, carrying per -/// source-caption style overrides across by source id. The previous track is -/// read from `timeline.caption_segments` itself. pub fn derive_caption_track_segments( source_segments: &[CaptionSegment], timeline: &TimelineConfiguration, recording_durations: &[f64], ) -> Vec { - struct TrackOverrides { - fade_duration: Option, - linger_duration: Option, - position: Option, - color: Option, - background_color: Option, - font_size: Option, - } - - let mut overrides_by_source_id: HashMap = HashMap::new(); - for segment in &timeline.caption_segments { - overrides_by_source_id - .entry(source_caption_id(&segment.id).to_string()) - .or_insert_with(|| TrackOverrides { - fade_duration: segment.fade_duration_override, - linger_duration: segment.linger_duration_override, - position: segment.position_override.clone(), - color: segment.color_override.clone(), - background_color: segment.background_color_override.clone(), - font_size: segment.font_size_override, - }); - } - - let mut mapped = - map_captions_to_edited_timeline(source_segments, timeline, recording_durations); - mapped.sort_by(|a, b| a.start.total_cmp(&b.start)); - - mapped - .into_iter() - .map(|segment| { - let overrides = overrides_by_source_id.get(source_caption_id(&segment.id)); - CaptionTrackSegment { - id: segment.id.clone(), - start: segment.start, - end: segment.end, - text: segment.text, - words: segment.words, - fade_duration_override: overrides.and_then(|o| o.fade_duration), - linger_duration_override: overrides.and_then(|o| o.linger_duration), - position_override: overrides.and_then(|o| o.position.clone()), - color_override: overrides.and_then(|o| o.color.clone()), - background_color_override: overrides.and_then(|o| o.background_color.clone()), - font_size_override: overrides.and_then(|o| o.font_size), - } - }) - .collect() + cap_project::derive_caption_track_segments(source_segments, timeline, recording_durations) } /// `heldTimeBefore` (`timeline-holds.ts:45-50`): how much hold-extended diff --git a/apps/desktop/src-tauri/src/captions.rs b/apps/desktop/src-tauri/src/captions.rs index cf76122e00..7ea519de11 100644 --- a/apps/desktop/src-tauri/src/captions.rs +++ b/apps/desktop/src-tauri/src/captions.rs @@ -1,5 +1,8 @@ use anyhow::Result; -use cap_audio::AudioData; +use cap_audio::{ + AudioData, TranscriptionAudioSource, TranscriptionAudioTake, append_transcription_audio, +}; +use cap_rendering::Video; use ffmpeg::{ ChannelLayout, codec as avcodec, format::{self as avformat}, @@ -24,7 +27,9 @@ use tokio::sync::{Mutex, Notify}; use tracing::instrument; use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; -pub use cap_project::{CaptionSegment, CaptionSettings, CaptionWord}; +pub use cap_project::{ + CaptionSegment, CaptionSettings, CaptionWord, RecordingMeta, StudioRecordingMeta, +}; use crate::{general_settings::GeneralSettingsStore, http_client}; @@ -211,10 +216,7 @@ pub async fn save_model_file(path: String, data: Vec) -> Result<(), String> } enum AudioExtractionSource { - ProjectDirectory { - base_path: PathBuf, - meta_path: PathBuf, - }, + ProjectDirectory { base_path: PathBuf }, MediaFile(PathBuf), } @@ -229,10 +231,7 @@ fn resolve_audio_extraction_source(video_path: &str) -> Result Re log::info!("Output path: {output_path:?}"); match resolve_audio_extraction_source(video_path)? { - AudioExtractionSource::ProjectDirectory { - base_path, - meta_path, - } => { + AudioExtractionSource::ProjectDirectory { base_path } => { log::info!("Detected recording project directory"); - let meta_content = std::fs::read_to_string(&meta_path) + let recording_meta = RecordingMeta::load_for_project(&base_path) .map_err(|e| format!("Failed to read recording metadata: {e}"))?; - - let meta: serde_json::Value = serde_json::from_str(&meta_content) - .map_err(|e| format!("Failed to parse recording metadata: {e}"))?; - - struct SegmentAudio { - sources: Vec, - } - - let mut segment_audios: Vec = Vec::new(); - - if let Some(segments) = meta["segments"].as_array() { - for segment in segments { + let studio = recording_meta + .studio_meta() + .ok_or_else(|| "Only studio recordings can be transcribed".to_string())?; + studio.ensure_ordinary_media_access(&base_path)?; + + let mut final_samples = Vec::new(); + let mut any_audio = false; + match studio { + StudioRecordingMeta::SingleSegment { segment } => { + let display_duration = Video::new( + segment.display.path.to_path(&base_path), + segment.display.start_time.unwrap_or_default(), + ) + .map_err(|error| format!("Failed to read display video: {error}"))? + .duration; let mut sources = Vec::new(); - let mut push_source = |path: Option<&str>| { - if let Some(path) = path { - let full_path = base_path.join(path); - if full_path.exists() && !sources.contains(&full_path) { - sources.push(full_path); + if let Some(audio) = &segment.audio { + let path = audio.path.to_path(&base_path); + if path.exists() { + match AudioData::from_file(&path) { + Ok(decoded) => { + any_audio = true; + sources.push(TranscriptionAudioSource { + samples: decoded.samples().to_vec(), + channels: decoded.channels() as usize, + offset_secs: 0.0, + }); + } + Err(error) => { + log::warn!("Failed to process audio source {path:?}: {error}") + } } } - }; - - push_source(segment["system_audio"]["path"].as_str()); - push_source(segment["mic"]["path"].as_str()); - push_source(segment["audio"]["path"].as_str()); - - if !sources.is_empty() { - segment_audios.push(SegmentAudio { sources }); } + let take = TranscriptionAudioTake { + display_duration_secs: display_duration, + sources, + }; + append_transcription_audio(&mut final_samples, &take, AudioData::SAMPLE_RATE) + .map_err(|e| format!("Failed to assemble transcription audio: {e}"))?; } - } - - if segment_audios.is_empty() { - return Err("No audio sources found in the recording metadata".to_string()); - } - - log::info!("Found {} segments with audio sources", segment_audios.len()); - - let mut final_samples: Vec = Vec::new(); - - for (segment_idx, segment_audio) in segment_audios.iter().enumerate() { - log::info!( - "Processing segment {} with {} audio sources", - segment_idx, - segment_audio.sources.len() - ); - - let mut segment_samples: Vec = Vec::new(); - - for source in &segment_audio.sources { - match AudioData::from_file(source) { - Ok(audio) => { - log::info!( - "Processing audio source {:?}: {} channels, {} samples", - source, - audio.channels(), - audio.sample_count() - ); - - let mono_samples = if audio.channels() > 1 { - convert_to_mono(audio.samples(), audio.channels() as usize) - } else { - audio.samples().to_vec() - }; - - if segment_samples.is_empty() { - segment_samples = mono_samples; - } else { - mix_samples(&mut segment_samples, &mono_samples); + StudioRecordingMeta::MultipleSegments { inner } => { + for (segment_idx, segment) in inner.segments.iter().enumerate() { + let display_duration = Video::new( + segment.display.path.to_path(&base_path), + segment.display.start_time.unwrap_or_default(), + ) + .map_err(|error| { + format!( + "Failed to read display video for segment {segment_idx}: {error}" + ) + })? + .duration; + let offsets = segment.calculate_audio_offsets(); + let mut sources = Vec::new(); + let mut push_source = |path: &cap_project::AudioMeta, offset_secs: f64| { + let path = path.path.to_path(&base_path); + if !path.exists() { + return; } + match AudioData::from_file(&path) { + Ok(decoded) => { + any_audio = true; + sources.push(TranscriptionAudioSource { + samples: decoded.samples().to_vec(), + channels: decoded.channels() as usize, + offset_secs, + }); + } + Err(error) => { + log::warn!("Failed to process audio source {path:?}: {error}") + } + } + }; + + if let Some(system_audio) = &segment.system_audio { + push_source(system_audio, f64::from(offsets.system_audio)); } - Err(e) => { - log::warn!("Failed to process audio source {source:?}: {e}"); - continue; + if let Some(mic) = &segment.mic { + push_source(mic, f64::from(offsets.mic)); } + + let take = TranscriptionAudioTake { + display_duration_secs: display_duration, + sources, + }; + append_transcription_audio( + &mut final_samples, + &take, + AudioData::SAMPLE_RATE, + ) + .map_err(|e| format!("Failed to assemble transcription audio: {e}"))?; } } + } - if !segment_samples.is_empty() { - log::info!( - "Segment {} produced {} samples, appending to final audio", - segment_idx, - segment_samples.len() - ); - final_samples.extend(segment_samples); - } + if !any_audio { + return Err("No audio sources found in the recording metadata".to_string()); } let mut mixed_samples = final_samples; @@ -2703,33 +2711,6 @@ pub async fn export_captions_srt( } } -fn convert_to_mono(samples: &[f32], channels: usize) -> Vec { - if channels == 1 { - return samples.to_vec(); - } - - let sample_count = samples.len() / channels; - let mut mono_samples = Vec::with_capacity(sample_count); - - for i in 0..sample_count { - let mut sample_sum = 0.0; - for c in 0..channels { - sample_sum += samples[i * channels + c]; - } - mono_samples.push(sample_sum / channels as f32); - } - - mono_samples -} - -fn mix_samples(dest: &mut [f32], source: &[f32]) -> usize { - let length = dest.len().min(source.len()); - for i in 0..length { - dest[i] = (dest[i] + source[i]) * 0.5; - } - length -} - #[cfg(test)] mod tests { use super::{ @@ -2786,12 +2767,8 @@ mod tests { std::fs::write(project_dir.join("recording-meta.json"), "{}").unwrap(); match resolve_audio_extraction_source(project_dir.to_string_lossy().as_ref()).unwrap() { - AudioExtractionSource::ProjectDirectory { - base_path, - meta_path, - } => { + AudioExtractionSource::ProjectDirectory { base_path } => { assert_eq!(base_path, project_dir); - assert_eq!(meta_path, base_path.join("recording-meta.json")); } AudioExtractionSource::MediaFile(_) => panic!("expected project directory"), } diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index aa5c89efa5..1f36a7a296 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -1505,7 +1505,7 @@ async fn generate_export_preview_inner( return Err("Cannot preview non-studio recordings".to_string()); }; - let project_config = + let mut project_config = export_project_config(recording_meta.project_config(), settings.cursor_only); let recordings = Arc::new( @@ -1513,6 +1513,16 @@ async fn generate_export_preview_inner( .map_err(|e| format!("Failed to load recordings: {e}"))?, ); + synchronize_preview_timing( + &recording_meta, + &mut project_config, + &recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect::>(), + ); + let render_constants = Arc::new( RenderVideoConstants::new( &recordings.segments, @@ -1746,6 +1756,39 @@ mod tests { use super::*; use tempfile::tempdir; + #[test] + fn export_preview_projects_captions_after_a_cut() { + let directory = tempdir().unwrap(); + let mut meta: RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name":"preview", "segments":[], "cursors":{} + })) + .unwrap(); + meta.project_path = directory.path().to_path_buf(); + let config: cap_project::ProjectConfiguration = serde_json::from_value(serde_json::json!({ + "timeline": { + "segments":[ + {"start":0.0,"end":5.0,"timescale":1.0}, + {"start":6.0,"end":10.0,"timescale":1.0} + ], "zoomSegments":[] + }, + "captions": { + "sourceTimed":true, "settings":{}, + "segments":[{"id":"word","text":"retained","start":8.0,"end":8.5,"words":[]}] + } + })) + .unwrap(); + config.write(directory.path()).unwrap(); + let mut preview = export_project_config(config.clone(), false); + synchronize_preview_timing(&meta, &mut preview, &[10.0]); + let caption = &preview.timeline.as_ref().unwrap().caption_segments[0]; + assert_eq!(caption.start, 7.0); + assert_eq!(caption.end, 7.5); + assert_eq!(caption.text, "retained"); + let mut cursor_only = export_project_config(config, true); + synchronize_preview_timing(&meta, &mut cursor_only, &[10.0]); + assert!(cursor_only.captions.is_none()); + } + #[test] fn export_estimates_use_source_duration_without_a_timeline() { assert_eq!( @@ -1883,6 +1926,14 @@ pub async fn generate_export_preview_fast( } } +fn synchronize_preview_timing( + meta: &RecordingMeta, + project: &mut cap_project::ProjectConfiguration, + display_durations: &[f64], +) { + cap_project::synchronize_legacy_keyboard(meta, project); + cap_project::synchronize_captions(project, display_durations); +} #[instrument(skip_all)] async fn generate_export_preview_fast_inner( editor: WindowEditorInstance, @@ -1898,10 +1949,26 @@ async fn generate_export_preview_fast_inner( let _preview_guard = ExportPreviewActiveGuard::try_new(&editor.export_preview_active)?; - let project_config = export_project_config( + let mut project_config = export_project_config( editor.project_config.1.borrow().clone(), settings.cursor_only, ); + let meta = editor.meta().clone(); + let recordings = editor.recordings.clone(); + let project_config = tokio::task::spawn_blocking(move || { + synchronize_preview_timing( + &meta, + &mut project_config, + &recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect::>(), + ); + project_config + }) + .await + .map_err(|error| format!("Failed to synchronize export preview timing: {error}"))?; let transition_mapping = project_config.timeline.as_ref().and_then(|timeline| { if timeline.transitions.is_empty() { return None; diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 9611273294..7e15f6962f 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -4313,39 +4313,23 @@ async fn generate_keyboard_segments( show_modifiers: bool, show_special_keys: bool, ) -> Result, String> { - let meta = editor_instance.meta(); - - let RecordingMetaInner::Studio(studio_meta) = &meta.inner else { - return Ok(vec![]); - }; - - let segments = match studio_meta.as_ref() { - StudioRecordingMeta::MultipleSegments { inner, .. } => &inner.segments, - _ => return Ok(vec![]), + let project = editor_instance.project_config.1.borrow().clone(); + let Some(timeline) = project.timeline else { + return Ok(Vec::new()); }; - - let mut all_events = cap_project::KeyboardEvents { presses: vec![] }; - - for segment in segments { - let events = segment.keyboard_events(meta); - all_events.presses.extend(events.presses); - } - - all_events.presses.sort_by(|a, b| { - a.time_ms - .partial_cmp(&b.time_ms) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let grouped = cap_project::group_key_events( - &all_events, + let settings = cap_project::KeyboardSettings { grouping_threshold_ms, - linger_duration_ms, + linger_duration: (linger_duration_ms / 1000.0) as f32, show_modifiers, show_special_keys, - ); - - Ok(grouped) + ..Default::default() + }; + let meta = editor_instance.meta().clone(); + tokio::task::spawn_blocking(move || { + cap_project::generate_project_keyboard_segments(&meta, &timeline, &settings) + }) + .await + .map_err(|error| format!("Keyboard generation failed: {error}"))? } #[tauri::command] diff --git a/apps/desktop/src/routes/editor/ClipsSidebar.tsx b/apps/desktop/src/routes/editor/ClipsSidebar.tsx index 74b5e36117..e64bdc55ac 100644 --- a/apps/desktop/src/routes/editor/ClipsSidebar.tsx +++ b/apps/desktop/src/routes/editor/ClipsSidebar.tsx @@ -65,6 +65,9 @@ import { useEditorContext, } from "./context"; import { getExistingRecordingPickerOptions } from "./existing-recording-picker"; +import { rippleKeyboardTrack } from "./keyboard-timing"; +import { scaleKeyframeTimes } from "./three-d"; +import { effectiveToOutput, holdWindows } from "./timeline-holds"; import { Input } from "./ui"; const findCamera = (cameras: CameraInfo[], id?: DeviceOrModelID | null) => { @@ -662,24 +665,43 @@ function ClipsSidebarInner(props: { open: boolean; class?: string }) { transition.segmentIndex, ); if (!effective) continue; - const boundary = + const boundary = effectiveToOutput( + holdWindows(timeline.textSegments), clipTimelineOffsets(timeline.segments, timeline.transitions)[ transition.segmentIndex - ] + effective.duration; + ] + effective.duration, + ); timeline.transitions = timeline.transitions.filter( (candidate) => candidate.segmentIndex !== transition.segmentIndex, ); + const camera3dSegments = timeline.camera3dSegments ?? []; + const previousCamera3dDurations = camera3dSegments.map( + (segment) => segment.end - segment.start, + ); for (const track of [ timeline.zoomSegments, timeline.sceneSegments ?? [], timeline.maskSegments, timeline.textSegments, timeline.captionSegments ?? [], - timeline.keyboardSegments ?? [], timeline.audioSegments ?? [], + camera3dSegments, ]) { rippleTimelineTrack(track, boundary, effective.duration); } + rippleKeyboardTrack( + timeline.keyboardSegments ?? [], + boundary, + effective.duration, + ); + for (let index = 0; index < camera3dSegments.length; index++) { + const segment = camera3dSegments[index]; + const previousDuration = previousCamera3dDurations[index]; + const nextDuration = segment.end - segment.start; + if (previousDuration <= 0 || previousDuration === nextDuration) + continue; + scaleKeyframeTimes(segment.tracks, nextDuration / previousDuration); + } } timeline.segments = proposedSegments; diff --git a/apps/desktop/src/routes/editor/KeyboardTab.tsx b/apps/desktop/src/routes/editor/KeyboardTab.tsx index 5b5af46fea..d9162485f6 100644 --- a/apps/desktop/src/routes/editor/KeyboardTab.tsx +++ b/apps/desktop/src/routes/editor/KeyboardTab.tsx @@ -2,6 +2,7 @@ import { Button } from "@cap/ui-solid"; import { Select as KSelect } from "@kobalte/core/select"; import { cx } from "cva"; import { batch, createMemo, createSignal, Show } from "solid-js"; +import toast from "solid-toast"; import { Toggle } from "~/components/Toggle"; import { defaultKeyboardSettings, @@ -12,6 +13,10 @@ import { commands } from "~/utils/tauri"; import IconCapChevronDown from "~icons/cap/chevron-down"; import IconCapCircleCheck from "~icons/cap/circle-check"; import { useEditorContext } from "./context"; +import { + generateForStableKeyboardTimeline, + keyboardTimelineSignature, +} from "./keyboard-timing"; import { FONT_OPTIONS, getTextWeightLabel, @@ -33,8 +38,13 @@ import { export function KeyboardTab(props: { brandColorSwatches: OrganizationBrandColorSwatch[]; }) { - const { project, setProject, editorState, setEditorState } = - useEditorContext(); + const { + project, + setProject, + editorState, + setEditorState, + flushProjectConfig, + } = useEditorContext(); const getSetting = ( key: K, @@ -88,24 +98,50 @@ export function KeyboardTab(props: { }; const generateSegments = async () => { + if (!project.timeline || isGenerating()) return; setIsGenerating(true); try { - const segments = await commands.generateKeyboardSegments( - getSetting("groupingThresholdMs"), - getSetting("lingerDuration") * 1000, - getSetting("showModifiers"), - getSetting("showSpecialKeys"), + const segments = await generateForStableKeyboardTimeline( + () => { + const timeline = keyboardTimelineSignature(project.timeline); + if (timeline === null) return null; + return [ + timeline, + getSetting("groupingThresholdMs"), + getSetting("lingerDuration"), + getSetting("showModifiers"), + getSetting("showSpecialKeys"), + ].join("@@"); + }, + async () => { + await flushProjectConfig(); + return commands.generateKeyboardSegments( + getSetting("groupingThresholdMs"), + getSetting("lingerDuration") * 1000, + getSetting("showModifiers"), + getSetting("showSpecialKeys"), + ); + }, ); - if (segments.length > 0) { - batch(() => { + if (!segments) { + toast.error( + "The timeline changed while keyboard events were generated. Try again.", + ); + return; + } + batch(() => { + setProject("timeline", "keyboardSegments", segments); + if (segments.length > 0) { ensureKeyboardSettings(true); - setProject("timeline", "keyboardSegments", segments); setEditorState("timeline", "tracks", "keyboard", true); - }); - } + } else if (editorState.timeline.selection?.type === "keyboard") { + setEditorState("timeline", "selection", null); + } + }); } catch (e) { console.error("Failed to generate keyboard segments:", e); + toast.error("Unable to generate keyboard events"); } finally { setIsGenerating(false); } diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index 30e55787a5..5ec4ab3c01 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -20,7 +20,7 @@ import { onMount, } from "solid-js"; import { createStore, produce, reconcile, unwrap } from "solid-js/store"; - +import toast from "solid-toast"; import { generalSettingsStore } from "~/store"; import { type EditorCaptionSettings, @@ -72,10 +72,16 @@ import { normalizeClipTransitions, rippleTimelineTrack, timelineShiftAfterClipDurationChange, - transitionsAfterClipDelete, transitionsAfterClipSplit, } from "./clip-transitions"; import { normalizeColorCorrection } from "./colorCorrection"; +import { + generateForStableKeyboardTimeline, + keyboardTimelineSignature, + mapKeyboardTrackTimes, + rippleKeyboardTrack, + splitKeyboardSegment, +} from "./keyboard-timing"; import type { MaskSegment } from "./masks"; import type { SnapGuide } from "./snapping"; import type { TextSegment } from "./text"; @@ -94,10 +100,12 @@ import { setMotion, } from "./three-d"; import { + effectiveToOutput, heldTimeBefore, holdWindows, totalHeldDuration, } from "./timeline-holds"; +import { deleteClipAndRippleAllTracks } from "./timeline-utils"; import { getUsedTrackCount, normalizeTrackSegments, @@ -383,9 +391,11 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( const duration = transition ? clampTransitionDuration(transition.duration, previous, segment) : 0; - const boundary = + const boundary = effectiveToOutput( + holdWindows(timeline.textSegments), clipTimelineOffsets(timeline.segments, transitions)[segmentIndex] + - oldDuration; + oldDuration, + ); const shift = oldDuration - duration; timeline.transitions = transitions.filter( @@ -413,13 +423,13 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( timeline.maskSegments, timeline.textSegments, timeline.captionSegments ?? [], - timeline.keyboardSegments ?? [], timeline.audioSegments ?? [], camera3dSegments, ]; for (const track of tracks) { rippleTimelineTrack(track, boundary, shift); } + rippleKeyboardTrack(timeline.keyboardSegments ?? [], boundary, shift); for (let index = 0; index < camera3dSegments.length; index++) { const camera3dSegment = camera3dSegments[index]; const previousDuration = previousCamera3dDurations[index]; @@ -580,19 +590,14 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( }, deleteClipSegment: (segmentIndex: number) => { if (!project.timeline) return; - const segment = project.timeline.segments[segmentIndex]; - if (!segment || project.timeline.segments.length < 2) return; + if (project.timeline.segments.length < 2) return; batch(() => { setProject( produce((project) => { const timeline = project.timeline; if (!timeline) return; - timeline.segments.splice(segmentIndex, 1); - timeline.transitions = transitionsAfterClipDelete( - timeline.transitions ?? [], - segmentIndex, - ); + deleteClipAndRippleAllTracks(timeline, segmentIndex); }), ); setEditorState("timeline", "selection", null); @@ -981,14 +986,12 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( const duration = segment.end - segment.start; const remaining = duration - time; if (time < 0.3 || remaining < 0.3) return; - - segments.splice(index + 1, 0, { - ...segment, - id: `kb-split-${Date.now()}-${Math.random().toString(36).slice(2)}`, - start: segment.start + time, - end: segment.end, - }); - segments[index].end = segment.start + time; + const parts = splitKeyboardSegment( + structuredClone(unwrap(segment)), + segment.start + time, + `kb-split-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + if (parts) segments.splice(index, 1, ...parts); }), ); }, @@ -1101,6 +1104,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( timeline.segments, timeline.transitions ?? [], ); + const oldHolds = holdWindows(timeline.textSegments); const incomingDuration = getClipTransition( timeline.segments, @@ -1133,47 +1137,51 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( oldNextBoundary, newNextBoundary, ); + const mapOutputTime = (value: number) => { + const held = heldTimeBefore(oldHolds, value); + return value + diff(value - held); + }; for (const zoomSegment of timeline.zoomSegments) { - zoomSegment.start += diff(zoomSegment.start); - zoomSegment.end += diff(zoomSegment.end); + zoomSegment.start = mapOutputTime(zoomSegment.start); + zoomSegment.end = mapOutputTime(zoomSegment.end); } for (const sceneSegment of timeline.sceneSegments ?? []) { - sceneSegment.start += diff(sceneSegment.start); - sceneSegment.end += diff(sceneSegment.end); + sceneSegment.start = mapOutputTime(sceneSegment.start); + sceneSegment.end = mapOutputTime(sceneSegment.end); } for (const maskSegment of timeline.maskSegments) { - maskSegment.start += diff(maskSegment.start); - maskSegment.end += diff(maskSegment.end); + maskSegment.start = mapOutputTime(maskSegment.start); + maskSegment.end = mapOutputTime(maskSegment.end); } for (const textSegment of timeline.textSegments) { - textSegment.start += diff(textSegment.start); - textSegment.end += diff(textSegment.end); + textSegment.start = mapOutputTime(textSegment.start); + textSegment.end = mapOutputTime(textSegment.end); } for (const audioSegment of timeline.audioSegments ?? []) { - audioSegment.start += diff(audioSegment.start); - audioSegment.end += diff(audioSegment.end); + audioSegment.start = mapOutputTime(audioSegment.start); + audioSegment.end = mapOutputTime(audioSegment.end); } for (const captionSegment of timeline.captionSegments ?? []) { - captionSegment.start += diff(captionSegment.start); - captionSegment.end += diff(captionSegment.end); + captionSegment.start = mapOutputTime(captionSegment.start); + captionSegment.end = mapOutputTime(captionSegment.end); } - for (const keyboardSegment of timeline.keyboardSegments ?? []) { - keyboardSegment.start += diff(keyboardSegment.start); - keyboardSegment.end += diff(keyboardSegment.end); - } + mapKeyboardTrackTimes( + timeline.keyboardSegments ?? [], + mapOutputTime, + ); for (const camera3dSegment of timeline.camera3dSegments ?? []) { const previousDuration = camera3dSegment.end - camera3dSegment.start; - camera3dSegment.start += diff(camera3dSegment.start); - camera3dSegment.end += diff(camera3dSegment.end); + camera3dSegment.start = mapOutputTime(camera3dSegment.start); + camera3dSegment.end = mapOutputTime(camera3dSegment.end); // Keyframe times are relative to the segment start, so they // have to follow the segment's new length rather than the // absolute shift the other tracks use. @@ -1203,53 +1211,48 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( }; let projectSaveTimeout: number | undefined; - let saveInFlight = false; - let shouldResave = false; - let hasPendingProjectSave = false; + let saveInFlight: ReturnType | undefined; + let persistedProject: string | undefined; const flushProjectConfig = async () => { - if (!hasPendingProjectSave && !saveInFlight) return; - if (saveInFlight) { - if (hasPendingProjectSave) { - shouldResave = true; - } - return; + if (projectSaveTimeout !== undefined) { + clearTimeout(projectSaveTimeout); + projectSaveTimeout = undefined; } - saveInFlight = true; - shouldResave = false; - hasPendingProjectSave = false; - try { + while (true) { + if (saveInFlight) { + await saveInFlight; + continue; + } const config = serializeProjectConfiguration(project); - await commands.setProjectConfig(config); - } catch (error) { - console.error("Failed to persist project config", error); - } finally { - saveInFlight = false; - if (shouldResave) { - shouldResave = false; - void flushProjectConfig(); + const serialized = JSON.stringify(config); + if (serialized === persistedProject) return; + const saving = commands.setProjectConfig(config); + saveInFlight = saving; + try { + await saving; + persistedProject = serialized; + } finally { + if (saveInFlight === saving) saveInFlight = undefined; } } }; + const saveProjectConfig = () => { + void flushProjectConfig().catch((error) => + console.error("Failed to persist project config", error), + ); + }; + const scheduleProjectConfigSave = () => { - hasPendingProjectSave = true; - if (projectSaveTimeout) { - clearTimeout(projectSaveTimeout); - } - projectSaveTimeout = window.setTimeout(() => { - projectSaveTimeout = undefined; - void flushProjectConfig(); - }, PROJECT_SAVE_DEBOUNCE_MS); + if (projectSaveTimeout !== undefined) clearTimeout(projectSaveTimeout); + projectSaveTimeout = window.setTimeout( + saveProjectConfig, + PROJECT_SAVE_DEBOUNCE_MS, + ); }; - onCleanup(() => { - if (projectSaveTimeout) { - clearTimeout(projectSaveTimeout); - projectSaveTimeout = undefined; - } - void flushProjectConfig(); - }); + onCleanup(saveProjectConfig); createEffect( on( @@ -1562,13 +1565,25 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( void (async () => { try { - const segments = await commands.generateKeyboardSegments( - defaultKeyboardSettings.groupingThresholdMs, - defaultKeyboardSettings.lingerDuration * 1000, - defaultKeyboardSettings.showModifiers, - defaultKeyboardSettings.showSpecialKeys, + const segments = await generateForStableKeyboardTimeline( + () => keyboardTimelineSignature(project.timeline), + async () => { + await flushProjectConfig(); + return commands.generateKeyboardSegments( + defaultKeyboardSettings.groupingThresholdMs, + defaultKeyboardSettings.lingerDuration * 1000, + defaultKeyboardSettings.showModifiers, + defaultKeyboardSettings.showSpecialKeys, + ); + }, ); + if (!segments) { + toast.error( + "The timeline changed while keyboard events were generated. Try again.", + ); + return; + } if (segments.length < 1) return; batch(() => { @@ -1581,6 +1596,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( }); } catch (error) { console.error("Failed to initialize keyboard segments", error); + toast.error("Unable to generate keyboard events"); } })(); }); @@ -1604,6 +1620,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( timeline.transitions ?? [], undefined, "incoming", + timeline.textSegments, ); const inverted = segments.flatMap((segment) => { const start = toSource(segment.start); @@ -1709,6 +1726,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( return { ...editorInstanceContext, + flushProjectConfig, meta() { return props.meta(); }, diff --git a/apps/desktop/src/routes/editor/keyboard-timing.test.ts b/apps/desktop/src/routes/editor/keyboard-timing.test.ts new file mode 100644 index 0000000000..bcf37402eb --- /dev/null +++ b/apps/desktop/src/routes/editor/keyboard-timing.test.ts @@ -0,0 +1,332 @@ +import { describe, expect, it } from "vitest"; +import type { KeyboardTrackSegment, SegmentRecordings } from "~/utils/tauri"; +import { mapEditedTimeToSource } from "./captions"; +import { timelineShiftAfterClipDurationChange } from "./clip-transitions"; +import { + generateForStableKeyboardTimeline, + mapKeyboardTrackTimes, + rippleDeleteKeyboardTrack, + rippleKeyboardTrack, + splitKeyboardSegment, +} from "./keyboard-timing"; +import { defaultTextSegment } from "./text"; +import { defaultCamera3DTracks } from "./three-d"; +import { + deleteClipAndRippleAllTracks, + rippleDeleteAllTracks, + rippleDeleteFromTrack, +} from "./timeline-utils"; + +function keyboardSegment( + start: number, + end: number, + displayText: string, + offsets: number[], +): KeyboardTrackSegment { + return { + id: "keyboard-1", + start, + end, + displayText, + keys: offsets.map((timeOffset, index) => ({ + key: displayText[index] ?? "Key", + timeOffset, + })), + }; +} + +describe("keyboard output timing", () => { + it("moves a source-time key at 8s to output time 7s after deleting 5s to 6s", () => { + const timeline = { + segments: [{ start: 0, end: 10, timescale: 1 }], + transitions: [], + keyboardSegments: [keyboardSegment(8, 9, "a", [0])], + }; + + rippleDeleteAllTracks(timeline, 5, 6); + + expect(timeline.keyboardSegments[0]).toMatchObject({ + start: 7, + end: 8, + keys: [{ timeOffset: 0 }], + }); + }); + + it("filters a cut from a one-character-per-key group and rebases later keys", () => { + const segments = [keyboardSegment(4, 8, "abc", [0, 1500, 2500])]; + + rippleDeleteKeyboardTrack(segments, 5, 6); + + expect(segments).toHaveLength(1); + expect(segments[0]).toMatchObject({ + start: 4, + end: 7, + displayText: "ac", + keys: [{ timeOffset: 0 }, { timeOffset: 1500 }], + }); + }); + + it("removes an atomic shortcut instead of corrupting its display text", () => { + const segment = keyboardSegment(4, 8, "ab", [0, 1500, 2500]); + segment.displayText = "⌘K"; + const segments = [segment]; + + rippleDeleteKeyboardTrack(segments, 5, 6); + + expect(segments).toEqual([]); + }); + + it("scales per-key offsets with a 2x clip speed change", () => { + const segments = [keyboardSegment(0, 4, "abc", [0, 1000, 2000])]; + const shift = (time: number) => + timelineShiftAfterClipDurationChange(time, 0, 0, 0, 4, 2); + + mapKeyboardTrackTimes(segments, (time) => time + shift(time)); + + expect(segments[0]).toMatchObject({ + start: 0, + end: 2, + keys: [{ timeOffset: 0 }, { timeOffset: 500 }, { timeOffset: 1000 }], + }); + }); + + it("rebases key offsets when a transition changes", () => { + const segments = [keyboardSegment(4, 7, "ab", [0, 2000])]; + + rippleKeyboardTrack(segments, 5, -1); + + expect(segments[0]).toMatchObject({ + start: 4, + end: 6, + keys: [{ timeOffset: 0 }, { timeOffset: 1000 }], + }); + }); + + it("leaves a keyboard segment ending at a transition boundary unchanged", () => { + const segments = [keyboardSegment(4, 5, "a", [0])]; + + rippleKeyboardTrack(segments, 5, -1); + + expect(segments[0]).toMatchObject({ + start: 4, + end: 5, + keys: [{ timeOffset: 0 }], + }); + }); + + it("maps a retained right tail to the shifted cut end", () => { + const segments = [keyboardSegment(5.5, 8, "a", [500])]; + const overlays = [{ start: 5.5, end: 8 }]; + + rippleDeleteKeyboardTrack(segments, 5, 6, 0.5); + rippleDeleteFromTrack(overlays, 5, 6, 0.5); + + expect(segments[0]).toMatchObject({ + start: 5.5, + end: 7.5, + keys: [{ timeOffset: 0 }], + }); + expect(overlays).toEqual([{ start: 5.5, end: 7.5 }]); + }); + + it("whole-clip deletion removes holds and uses the actual transition duration", () => { + const timeline = { + segments: [ + { start: 0, end: 4, timescale: 1 }, + { start: 0, end: 4, timescale: 1 }, + { start: 0, end: 4, timescale: 1 }, + ], + transitions: [ + { segmentIndex: 1, type: "cross-fade" as const, duration: 1 }, + { segmentIndex: 2, type: "cross-fade" as const, duration: 1 }, + ], + textSegments: [ + { start: 4, end: 5, enabled: true, layout: "fullscreen" as const }, + { start: 7, end: 8, enabled: true, layout: "fullscreen" as const }, + ], + zoomSegments: [{ start: 10, end: 11 }], + keyboardSegments: [keyboardSegment(3, 9, "abc", [500, 2500, 5500])], + audioSegments: [{ start: 5, end: 10, trimStart: 2, fadeIn: 1 }], + maskSegments: [ + { + start: 5, + end: 10, + keyframes: { + position: [{ time: 1 }, { time: 4 }, { time: 4.5 }], + }, + }, + ], + camera3dSegments: [ + { start: 4, end: 5, tracks: defaultCamera3DTracks() }, + { + start: 10, + end: 14, + tracks: { + ...defaultCamera3DTracks(), + zoom: [{ time: 2, value: 1, outEasing: null, inEasing: null }], + }, + }, + ], + }; + + expect(deleteClipAndRippleAllTracks(timeline, 1)).toBe(true); + expect(timeline.segments).toHaveLength(2); + expect(timeline.transitions).toEqual([]); + expect(timeline.textSegments).toEqual([ + { start: 4, end: 5, enabled: true, layout: "fullscreen" }, + ]); + expect(timeline.zoomSegments).toEqual([{ start: 7, end: 8 }]); + expect(timeline.keyboardSegments[0]).toMatchObject({ + start: 3, + end: 6, + displayText: "ac", + keys: [{ timeOffset: 500 }, { timeOffset: 2500 }], + }); + expect(timeline.audioSegments).toEqual([ + { start: 4, end: 7, trimStart: 4, fadeIn: 0 }, + ]); + expect(timeline.maskSegments[0]).toMatchObject({ + start: 4, + end: 7, + keyframes: { position: [{ time: 2 }, { time: 2.5 }] }, + }); + expect(timeline.camera3dSegments).toHaveLength(1); + expect(timeline.camera3dSegments[0]).toMatchObject({ + start: 7, + end: 11, + tracks: { zoom: [{ time: 2, value: 1 }] }, + }); + }); + + it("cuts camera keyframes without rescaling retained source timing", () => { + const tracks = defaultCamera3DTracks(); + tracks.zoom = [ + { + time: 2, + value: 2, + outEasing: [0, 0], + inEasing: null, + }, + { + time: 5, + value: 5, + outEasing: [0, 0], + inEasing: [1, 1], + }, + { + time: 8, + value: 8, + outEasing: null, + inEasing: [1, 1], + }, + ]; + const timeline = { + segments: [{ start: 0, end: 10, timescale: 1 }], + camera3dSegments: [ + { + start: 0, + end: 10, + tracks, + transitionIn: 0.2, + transitionOut: 0.3, + }, + ], + }; + + rippleDeleteAllTracks(timeline, 3, 6); + + expect(timeline.camera3dSegments[0]).toMatchObject({ + start: 0, + end: 7, + transitionIn: 0.2, + transitionOut: 0.3, + tracks: { + zoom: [ + { time: 2, value: 2, outEasing: [0, 0], inEasing: null }, + { time: 3, value: 3, outEasing: null, inEasing: [1, 1] }, + { time: 3, value: 6, outEasing: [0, 0], inEasing: null }, + { time: 5, value: 8, outEasing: null, inEasing: [1, 1] }, + ], + }, + }); + }); + + it("retains the last clip", () => { + const timeline = { + segments: [{ start: 0, end: 4, timescale: 1 }], + keyboardSegments: [keyboardSegment(1, 2, "a", [0])], + }; + + expect(deleteClipAndRippleAllTracks(timeline, 0)).toBe(false); + expect(timeline.segments).toHaveLength(1); + expect(timeline.keyboardSegments).toHaveLength(1); + + rippleDeleteAllTracks(timeline, 0, 4, 0); + expect(timeline.segments).toHaveLength(1); + expect(timeline.keyboardSegments).toHaveLength(1); + }); + + it("manual split partitions generated keys and preserves static segments", () => { + const generated = keyboardSegment(10, 13, "abc", [0, 1000, 2000]); + const generatedParts = splitKeyboardSegment(generated, 11, "keyboard-2"); + expect(generatedParts?.[0]).toMatchObject({ + id: "kb-edit-keyboard-1", + end: 11, + displayText: "a", + keys: [{ timeOffset: 0 }], + }); + expect(generatedParts?.[1]).toMatchObject({ + id: "kb-edit-keyboard-2", + start: 11, + displayText: "bc", + keys: [{ timeOffset: 0 }, { timeOffset: 1000 }], + }); + + const manual = keyboardSegment(10, 13, "Custom", []); + const manualParts = splitKeyboardSegment(manual, 11, "kb-edit-keyboard-3"); + expect(manualParts?.map((part) => part.id)).toEqual([ + "kb-edit-keyboard-1", + "kb-edit-keyboard-3", + ]); + expect(manualParts?.map((part) => part.displayText)).toEqual([ + "Custom", + "Custom", + ]); + }); + + it("retries generation once when the timeline changes during the command", async () => { + let signature = "first"; + let callCount = 0; + const result = await generateForStableKeyboardTimeline( + () => signature, + async () => { + callCount++; + if (callCount === 1) signature = "second"; + return callCount; + }, + ); + + expect(result).toBe(2); + expect(callCount).toBe(2); + }); + + it("inverts legacy caption output time through fullscreen holds", () => { + const hold = { + ...defaultTextSegment(2, 4), + layout: "fullscreen" as const, + }; + const recordings = [{ display: { duration: 10 } } as SegmentRecordings]; + + expect( + mapEditedTimeToSource( + 5, + [{ start: 0, end: 10, timescale: 1 }], + recordings, + [], + undefined, + "incoming", + [hold], + ), + ).toBe(3); + }); +}); diff --git a/apps/desktop/src/routes/editor/keyboard-timing.ts b/apps/desktop/src/routes/editor/keyboard-timing.ts new file mode 100644 index 0000000000..cd1c18fe6c --- /dev/null +++ b/apps/desktop/src/routes/editor/keyboard-timing.ts @@ -0,0 +1,206 @@ +import type { KeyboardTrackSegment } from "~/utils/tauri"; + +type TimelineSignatureInput = { + segments: Array<{ + start: number; + end: number; + timescale: number; + recordingSegment?: number | null; + }>; + transitions?: Array<{ + segmentIndex: number; + type: string; + duration: number; + }> | null; + textSegments?: Array<{ + start: number; + end: number; + enabled?: boolean; + layout?: string; + }> | null; +}; + +function rebaseKeys( + segment: KeyboardTrackSegment, + oldStart: number, + mapTime: (time: number) => number, +) { + const newStart = mapTime(oldStart); + for (const key of segment.keys ?? []) { + key.timeOffset = Math.max( + 0, + (mapTime(oldStart + key.timeOffset / 1000) - newStart) * 1000, + ); + } + segment.start = newStart; + segment.end = Math.max(newStart, mapTime(segment.end)); +} + +export function mapKeyboardTrackTimes( + segments: KeyboardTrackSegment[], + mapTime: (time: number) => number, +) { + for (const segment of segments) { + rebaseKeys(segment, segment.start, mapTime); + } +} + +export function rippleKeyboardTrack( + segments: KeyboardTrackSegment[], + boundary: number, + shift: number, +) { + for (const segment of segments) { + if (segment.end <= boundary) continue; + rebaseKeys(segment, segment.start, (time) => + time >= boundary ? time + shift : time, + ); + } +} + +export function rippleDeleteKeyboardTrack( + segments: KeyboardTrackSegment[], + cutStart: number, + cutEnd: number, + shift = cutEnd - cutStart, +) { + for ( + let segmentIndex = segments.length - 1; + segmentIndex >= 0; + segmentIndex-- + ) { + const segment = segments[segmentIndex]; + if (segment.end <= cutStart) continue; + if (segment.start >= cutStart && segment.end <= cutEnd) { + segments.splice(segmentIndex, 1); + continue; + } + + const oldStart = segment.start; + const retained = (segment.keys ?? []).map((key) => { + const time = oldStart + key.timeOffset / 1000; + return time < cutStart || time >= cutEnd; + }); + const removesKeys = retained.some((keep) => !keep); + const chars = Array.from(segment.displayText); + if (removesKeys && chars.length !== (segment.keys?.length ?? 0)) { + segments.splice(segmentIndex, 1); + continue; + } + if (removesKeys) { + segment.displayText = chars + .filter((_, index) => retained[index]) + .join(""); + } + + if (segment.start >= cutEnd) { + segment.start -= shift; + segment.end -= shift; + } else if (segment.start < cutStart && segment.end > cutEnd) { + segment.end -= shift; + } else if (segment.start < cutStart) { + segment.end = cutStart; + } else { + segment.start = cutEnd - shift; + segment.end = Math.max(segment.start, segment.end - shift); + } + + const newStart = segment.start; + segment.keys = (segment.keys ?? []).flatMap((key, index) => { + if (!retained[index]) return []; + const time = oldStart + key.timeOffset / 1000; + const mapped = time >= cutEnd ? time - shift : time; + return [{ ...key, timeOffset: Math.max(0, (mapped - newStart) * 1000) }]; + }); + + if ( + segment.end <= segment.start || + (removesKeys && segment.keys.length === 0) + ) { + segments.splice(segmentIndex, 1); + } + } +} + +export function splitKeyboardSegment( + segment: KeyboardTrackSegment, + at: number, + rightId: string, +): [KeyboardTrackSegment, KeyboardTrackSegment] | null { + if (!Number.isFinite(at) || at <= segment.start || at >= segment.end) { + return null; + } + + const left = structuredClone(segment); + const right = structuredClone(segment); + if (!left.id.startsWith("kb-edit-")) left.id = `kb-edit-${left.id}`; + left.end = at; + right.id = rightId.startsWith("kb-edit-") ? rightId : `kb-edit-${rightId}`; + right.start = at; + + if (!segment.keys?.length) return [left, right]; + + const chars = Array.from(segment.displayText); + if (chars.length !== segment.keys.length) return null; + + left.keys = []; + left.displayText = ""; + right.keys = []; + right.displayText = ""; + for (let index = 0; index < segment.keys.length; index++) { + const key = segment.keys[index]; + const absolute = segment.start + key.timeOffset / 1000; + if (absolute < at) { + left.keys.push({ ...key }); + left.displayText += chars[index]; + } else { + right.keys.push({ + ...key, + timeOffset: (absolute - at) * 1000, + }); + right.displayText += chars[index]; + } + } + + if (left.keys.length === 0 || right.keys.length === 0) return null; + return [left, right]; +} + +export function keyboardTimelineSignature( + timeline: TimelineSignatureInput | null | undefined, +) { + if (!timeline) return null; + const segments = timeline.segments + .map( + (segment) => + `${segment.start}|${segment.end}|${segment.timescale}|${segment.recordingSegment ?? 0}`, + ) + .join(","); + const transitions = (timeline.transitions ?? []) + .map( + (transition) => + `${transition.segmentIndex}|${transition.type}|${transition.duration}`, + ) + .join(","); + const holds = (timeline.textSegments ?? []) + .filter( + (segment) => segment.enabled !== false && segment.layout === "fullscreen", + ) + .map((segment) => `${segment.start}|${segment.end}`) + .join(","); + return `${segments}@@${transitions}@@${holds}`; +} + +export async function generateForStableKeyboardTimeline( + getTimelineSignature: () => string | null, + generate: () => Promise, + maxAttempts = 2, +): Promise { + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const signature = getTimelineSignature(); + if (signature === null) return null; + const result = await generate(); + if (getTimelineSignature() === signature) return result; + } + return null; +} diff --git a/apps/desktop/src/routes/editor/timeline-utils.ts b/apps/desktop/src/routes/editor/timeline-utils.ts index 31ca6a616d..dd82410edc 100644 --- a/apps/desktop/src/routes/editor/timeline-utils.ts +++ b/apps/desktop/src/routes/editor/timeline-utils.ts @@ -1,10 +1,19 @@ +import type { KeyboardTrackSegment } from "~/utils/tauri"; import { type ClipTransition, + clipDuration, clipTimelineDuration, clipTimelineOffsets, + getClipTransition, transitionsAfterClipDelete, transitionsAfterClipSplit, } from "./clip-transitions"; +import { rippleDeleteKeyboardTrack } from "./keyboard-timing"; +import { + CAMERA3D_TRACK_KEYS, + type Camera3DTracks, + sampleTrack, +} from "./three-d"; import { effectiveToOutput, effectiveToOutputEnd, @@ -65,12 +74,176 @@ export function rippleDeleteFromTrack( } else if (seg.start < cutStart) { seg.end = cutStart; } else { - seg.start = cutStart; + seg.start = cutEnd - shiftDuration; seg.end = Math.max(seg.start, seg.end - shiftDuration); } } } +type RippleMaskSegment = { + start: number; + end: number; + keyframes?: { + position?: Array<{ time: number }>; + size?: Array<{ time: number }>; + intensity?: Array<{ time: number }>; + }; +}; + +function rippleDeleteMaskTrack( + segments: RippleMaskSegment[], + cutStart: number, + cutEnd: number, + shift: number, +) { + const previousStarts = new Map( + segments.map((segment) => [segment, segment.start]), + ); + rippleDeleteFromTrack(segments, cutStart, cutEnd, shift); + for (const segment of segments) { + const oldStart = previousStarts.get(segment); + if (oldStart === undefined || !segment.keyframes) continue; + const duration = segment.end - segment.start; + const rebase = (keyframes: T[] | undefined) => + keyframes?.flatMap((keyframe) => { + const absolute = oldStart + keyframe.time; + if (absolute >= cutStart && absolute < cutEnd) return []; + const mapped = absolute >= cutEnd ? absolute - shift : absolute; + const time = mapped - segment.start; + return time >= 0 && time <= duration ? [{ ...keyframe, time }] : []; + }); + segment.keyframes.position = rebase(segment.keyframes.position); + segment.keyframes.size = rebase(segment.keyframes.size); + segment.keyframes.intensity = rebase(segment.keyframes.intensity); + } +} + +type RippleAudioSegment = { + start: number; + end: number; + trimStart?: number; + fadeIn?: number; +}; + +function rippleDeleteAudioTrack( + segments: RippleAudioSegment[], + cutStart: number, + cutEnd: number, + shift: number, +) { + for (const segment of segments) { + if ( + segment.start >= cutStart && + segment.start < cutEnd && + segment.end > cutEnd && + segment.trimStart !== undefined + ) { + segment.trimStart += cutEnd - segment.start; + if (segment.fadeIn !== undefined) segment.fadeIn = 0; + } + } + rippleDeleteFromTrack(segments, cutStart, cutEnd, shift); +} + +type RippleCamera3DSegment = { + start: number; + end: number; + tracks: Camera3DTracks; + transitionIn?: number; + transitionOut?: number; +}; + +function rippleDeleteCamera3DTrack( + segments: RippleCamera3DSegment[], + cutStart: number, + cutEnd: number, + shift: number, +) { + for ( + let segmentIndex = segments.length - 1; + segmentIndex >= 0; + segmentIndex-- + ) { + const segment = segments[segmentIndex]; + if (segment.end <= cutStart) continue; + if (segment.start >= cutEnd) { + segment.start -= shift; + segment.end -= shift; + continue; + } + if (segment.start >= cutStart && segment.end <= cutEnd) { + segments.splice(segmentIndex, 1); + continue; + } + + const oldStart = segment.start; + const oldEnd = segment.end; + const keepsLeft = oldStart < cutStart; + const keepsRight = oldEnd > cutEnd; + const newStart = keepsLeft ? oldStart : cutEnd - shift; + const newEnd = keepsRight ? oldEnd - shift : cutStart; + const leftCutTime = cutStart - oldStart; + const rightCutTime = cutEnd - oldStart; + + for (const trackKey of CAMERA3D_TRACK_KEYS) { + const keyframes = segment.tracks[trackKey]; + if (keyframes.length === 0) continue; + const before = keepsLeft + ? keyframes + .filter((keyframe) => keyframe.time < leftCutTime) + .map((keyframe) => ({ ...keyframe })) + : []; + const after = keepsRight + ? keyframes + .filter((keyframe) => keyframe.time > rightCutTime) + .map((keyframe) => ({ + ...keyframe, + time: oldStart + keyframe.time - shift - newStart, + })) + : []; + const nextKeyframe = keyframes.find( + (keyframe) => keyframe.time >= leftCutTime, + ); + const previousKeyframe = [...keyframes] + .reverse() + .find((keyframe) => keyframe.time <= rightCutTime); + segment.tracks[trackKey] = [ + ...before, + ...(keepsLeft + ? [ + { + time: cutStart - newStart, + value: sampleTrack(0, keyframes, leftCutTime), + outEasing: null, + inEasing: nextKeyframe?.inEasing ?? null, + }, + ] + : []), + ...(keepsRight + ? [ + { + time: cutEnd - shift - newStart, + value: sampleTrack(0, keyframes, rightCutTime), + outEasing: previousKeyframe?.outEasing ?? null, + inEasing: null, + }, + ] + : []), + ...after, + ]; + } + + segment.start = newStart; + segment.end = Math.max(newStart, newEnd); + if (keepsLeft && !keepsRight && segment.transitionOut !== undefined) { + segment.transitionOut = 0; + } + if (!keepsLeft && keepsRight && segment.transitionIn !== undefined) { + segment.transitionIn = 0; + } + } +} + export function cutClipSegmentsForRange( segments: Array<{ timescale: number; @@ -121,6 +294,7 @@ export function cutClipSegmentsForRange( newSegs.push({ ...seg, start: afterStart }); } + if (segments.length === 1 && newSegs.length === 0) return transitions; segments.splice(startSegIdx, 1, ...newSegs); if (newSegs.length === 2) { return transitionsAfterClipSplit(transitions, startSegIdx); @@ -156,15 +330,21 @@ export function rippleDeleteAllTracks( transitions?: ClipTransition[] | null; zoomSegments?: Array<{ start: number; end: number }> | null; sceneSegments?: Array<{ start: number; end: number }> | null; - maskSegments?: Array<{ start: number; end: number }> | null; + maskSegments?: RippleMaskSegment[] | null; textSegments?: Array | null; captionSegments?: Array<{ start: number; end: number }> | null; - keyboardSegments?: Array<{ start: number; end: number }> | null; - audioSegments?: Array<{ start: number; end: number }> | null; + keyboardSegments?: KeyboardTrackSegment[] | null; + audioSegments?: RippleAudioSegment[] | null; + camera3dSegments?: RippleCamera3DSegment[] | null; }, cutStart: number, cutEnd: number, requestedSegmentIndex?: number, + trackCutRange?: { + start: number; + end: number; + removeHoldAtStart?: boolean; + }, ) { // The clip cut below works in the gapless recording-flow domain, but the // overlay tracks live in output time, which includes fullscreen-text @@ -172,27 +352,60 @@ export function rippleDeleteAllTracks( // time inside the cut leave with the text segments it belongs to (they // sit inside the converted range, so the overlay pass deletes them). const holds = holdWindows(timeline.textSegments); - const overlayCutStart = effectiveToOutput(holds, cutStart); - const overlayCutEnd = effectiveToOutputEnd(holds, cutEnd); + const trackCutStart = trackCutRange?.start ?? cutStart; + const trackCutEnd = trackCutRange?.end ?? cutEnd; + const overlayCutStart = trackCutRange?.removeHoldAtStart + ? effectiveToOutputEnd(holds, trackCutStart) + : effectiveToOutput(holds, trackCutStart); + const overlayCutEnd = effectiveToOutputEnd(holds, trackCutEnd); const durationBefore = clipTimelineDuration( timeline.segments, timeline.transitions ?? [], ); - timeline.transitions = cutClipSegmentsForRange( + const previousSegments = timeline.segments.map((segment) => ({ ...segment })); + const previousTransitions = (timeline.transitions ?? []).map( + (transition) => ({ + ...transition, + }), + ); + const nextTransitions = cutClipSegmentsForRange( timeline.segments, timeline.transitions ?? [], cutStart, cutEnd, requestedSegmentIndex, ); + timeline.transitions = nextTransitions; + const clipChanged = + previousSegments.length !== timeline.segments.length || + previousSegments.some((segment, index) => { + const current = timeline.segments[index]; + return ( + !current || + segment.start !== current.start || + segment.end !== current.end || + segment.timescale !== current.timescale + ); + }) || + previousTransitions.length !== nextTransitions.length || + previousTransitions.some((transition, index) => { + const current = nextTransitions[index]; + return ( + !current || + transition.segmentIndex !== current.segmentIndex || + transition.type !== current.type || + transition.duration !== current.duration + ); + }); + if (!clipChanged) return; const shiftDuration = Math.max( 0, - durationBefore - - clipTimelineDuration(timeline.segments, timeline.transitions), + durationBefore - clipTimelineDuration(timeline.segments, nextTransitions), ); const overlayShift = - shiftDuration + (overlayCutEnd - overlayCutStart - (cutEnd - cutStart)); + shiftDuration + + (overlayCutEnd - overlayCutStart - (trackCutEnd - trackCutStart)); if (timeline.zoomSegments) rippleDeleteFromTrack( timeline.zoomSegments, @@ -208,7 +421,7 @@ export function rippleDeleteAllTracks( overlayShift, ); if (timeline.maskSegments) - rippleDeleteFromTrack( + rippleDeleteMaskTrack( timeline.maskSegments, overlayCutStart, overlayCutEnd, @@ -229,19 +442,58 @@ export function rippleDeleteAllTracks( overlayShift, ); if (timeline.keyboardSegments) - rippleDeleteFromTrack( + rippleDeleteKeyboardTrack( timeline.keyboardSegments, overlayCutStart, overlayCutEnd, overlayShift, ); if (timeline.audioSegments) - rippleDeleteFromTrack( + rippleDeleteAudioTrack( timeline.audioSegments, overlayCutStart, overlayCutEnd, overlayShift, ); + if (timeline.camera3dSegments) { + rippleDeleteCamera3DTrack( + timeline.camera3dSegments, + overlayCutStart, + overlayCutEnd, + overlayShift, + ); + } +} + +export function deleteClipAndRippleAllTracks( + timeline: Parameters[0], + segmentIndex: number, +) { + const segment = timeline.segments[segmentIndex]; + if (!segment || timeline.segments.length < 2) return false; + const start = clipTimelineOffsets( + timeline.segments, + timeline.transitions ?? [], + )[segmentIndex]; + const incomingDuration = + getClipTransition( + timeline.segments, + timeline.transitions ?? [], + segmentIndex, + )?.duration ?? 0; + const outgoingDuration = + getClipTransition( + timeline.segments, + timeline.transitions ?? [], + segmentIndex + 1, + )?.duration ?? 0; + const end = start + clipDuration(segment); + rippleDeleteAllTracks(timeline, start, end, segmentIndex, { + start: start + incomingDuration, + end: end - outgoingDuration, + removeHoldAtStart: true, + }); + return true; } if (import.meta.vitest) { @@ -259,7 +511,15 @@ if (import.meta.vitest) { // Covers recording content 3.5..4.5 — entirely before the cut. zoomSegments: [{ start: 5.5, end: 6.5 }], // Covers recording content 6..7 — entirely after the cut. - keyboardSegments: [{ start: 8, end: 9 }], + keyboardSegments: [ + { + id: "keyboard-1", + start: 8, + end: 9, + displayText: "a", + keys: [{ key: "a", timeOffset: 0 }], + }, + ], }; // Delete recording content [5,6], which plays at output [7,8]. @@ -272,7 +532,15 @@ if (import.meta.vitest) { // Before the fix the gapless cut range [5,6] was compared against // these output-time positions and mangled the zoom to [5,5.5]. expect(timeline.zoomSegments).toEqual([{ start: 5.5, end: 6.5 }]); - expect(timeline.keyboardSegments).toEqual([{ start: 7, end: 8 }]); + expect(timeline.keyboardSegments).toEqual([ + { + id: "keyboard-1", + start: 7, + end: 8, + displayText: "a", + keys: [{ key: "a", timeOffset: 0 }], + }, + ]); expect(timeline.textSegments).toHaveLength(1); }); diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index f774c7252c..2d99265187 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -4,6 +4,7 @@ mod latency; mod renderer; mod streaming; mod sync_analysis; +mod transcription_timing; pub use audio_data::*; pub use calibration_store::*; @@ -11,6 +12,7 @@ pub use latency::*; pub use renderer::*; pub use streaming::*; pub use sync_analysis::*; +pub use transcription_timing::*; pub trait FromSampleBytes: cpal::SizedSample + std::fmt::Debug + Send + 'static { const BYTE_SIZE: usize; diff --git a/crates/audio/src/transcription_timing.rs b/crates/audio/src/transcription_timing.rs new file mode 100644 index 0000000000..6bb981332e --- /dev/null +++ b/crates/audio/src/transcription_timing.rs @@ -0,0 +1,199 @@ +#[derive(Debug, Clone, PartialEq)] +pub struct TranscriptionAudioSource { + pub samples: Vec, + pub channels: usize, + pub offset_secs: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TranscriptionAudioTake { + pub display_duration_secs: f64, + pub sources: Vec, +} + +fn sample_count(duration_secs: f64, sample_rate: u32, label: &str) -> Result { + if !duration_secs.is_finite() || duration_secs < 0.0 { + return Err(format!("{label} must be finite and non-negative")); + } + + let count = duration_secs * f64::from(sample_rate); + if !count.is_finite() || count > usize::MAX as f64 { + return Err(format!("{label} is too large")); + } + + Ok(count.round() as usize) +} + +fn offset_samples(offset_secs: f64, sample_rate: u32) -> Result { + if !offset_secs.is_finite() { + return Err("Audio source offset must be finite".to_string()); + } + + let samples = offset_secs * f64::from(sample_rate); + if !samples.is_finite() || samples < isize::MIN as f64 || samples > isize::MAX as f64 { + return Err("Audio source offset is too large".to_string()); + } + + Ok(samples.round() as isize) +} + +fn try_zeroed(len: usize, value: T, label: &str) -> Result, String> { + let mut values = Vec::new(); + values + .try_reserve_exact(len) + .map_err(|_| format!("{label} is too large"))?; + values.resize(len, value); + Ok(values) +} + +pub fn assemble_transcription_audio( + takes: &[TranscriptionAudioTake], + sample_rate: u32, +) -> Result, String> { + let mut output = Vec::new(); + for take in takes { + append_transcription_audio(&mut output, take, sample_rate)?; + } + Ok(output) +} + +pub fn append_transcription_audio( + output: &mut Vec, + take: &TranscriptionAudioTake, + sample_rate: u32, +) -> Result<(), String> { + if sample_rate == 0 { + return Err("Audio sample rate must be positive".to_string()); + } + + let frames = sample_count(take.display_duration_secs, sample_rate, "Display duration")?; + for source in &take.sources { + if source.channels == 0 { + return Err("Audio source must have at least one channel".to_string()); + } + if source.samples.len() % source.channels != 0 { + return Err("Audio source samples are not channel aligned".to_string()); + } + offset_samples(source.offset_secs, sample_rate)?; + } + + output + .try_reserve_exact(frames) + .map_err(|_| "Transcription audio is too large".to_string())?; + let mut sums = try_zeroed(frames, 0.0_f32, "Transcription audio")?; + let mut counts = try_zeroed(frames, 0_u32, "Transcription audio")?; + + for source in &take.sources { + let offset = offset_samples(source.offset_secs, sample_rate)?; + let source_frames = source.samples.len() / source.channels; + for source_frame in 0..source_frames { + let destination = source_frame as i128 - offset as i128; + if destination < 0 || destination >= frames as i128 { + continue; + } + + let first_sample = source_frame * source.channels; + let mono = source.samples[first_sample..first_sample + source.channels] + .iter() + .copied() + .sum::() + / source.channels as f32; + let destination = destination as usize; + sums[destination] += mono; + counts[destination] += 1; + } + } + + output.extend( + sums.into_iter() + .zip(counts) + .map(|(sum, count)| if count == 0 { 0.0 } else { sum / count as f32 }), + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{TranscriptionAudioSource, TranscriptionAudioTake, assemble_transcription_audio}; + + fn take(duration: f64, sources: Vec) -> TranscriptionAudioTake { + TranscriptionAudioTake { + display_duration_secs: duration, + sources, + } + } + + fn source(samples: Vec, channels: usize, offset_secs: f64) -> TranscriptionAudioSource { + TranscriptionAudioSource { + samples, + channels, + offset_secs, + } + } + + #[test] + fn silent_first_take_preserves_display_time() { + let takes = [ + take(2.0, Vec::new()), + take(1.0, vec![source(vec![1.0; 48_000], 1, 0.0)]), + ]; + + let output = assemble_transcription_audio(&takes, 48_000).unwrap(); + + assert_eq!(output.len(), 144_000); + assert!(output[..96_000].iter().all(|sample| *sample == 0.0)); + assert!(output[96_000..].iter().all(|sample| *sample == 1.0)); + } + + #[test] + fn short_audio_is_padded_to_display_duration() { + let takes = [take(2.0, vec![source(vec![1.0; 24_000], 1, 0.0)])]; + + let output = assemble_transcription_audio(&takes, 48_000).unwrap(); + + assert_eq!(output.len(), 96_000); + assert!(output[..24_000].iter().all(|sample| *sample == 1.0)); + assert!(output[24_000..].iter().all(|sample| *sample == 0.0)); + } + + #[test] + fn staggered_starts_trim_and_prepend() { + let takes = [take( + 2.0, + vec![ + source(vec![1.0; 96_000], 1, 0.5), + source(vec![2.0; 48_000], 1, -0.5), + ], + )]; + + let output = assemble_transcription_audio(&takes, 48_000).unwrap(); + + assert!(output[..24_000].iter().all(|sample| *sample == 1.0)); + assert!(output[24_000..72_000].iter().all(|sample| *sample == 1.5)); + assert!(output[72_000..].iter().all(|sample| *sample == 0.0)); + } + + #[test] + fn multitrack_mix_downmixes_and_averages_overlaps() { + let takes = [take( + 1.0, + vec![ + source(vec![1.0, 3.0, 1.0, 3.0], 2, 0.0), + source(vec![0.5, 0.5], 1, 0.0), + ], + )]; + + let output = assemble_transcription_audio(&takes, 2).unwrap(); + + assert_eq!(output, vec![1.25, 1.25]); + } + + #[test] + fn invalid_timing_is_rejected_before_allocation() { + let invalid_duration = [take(f64::NAN, Vec::new())]; + let invalid_offset = [take(1.0, vec![source(vec![1.0], 1, f64::INFINITY)])]; + + assert!(assemble_transcription_audio(&invalid_duration, 48_000).is_err()); + assert!(assemble_transcription_audio(&invalid_offset, 48_000).is_err()); + } +} diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index f2fa632a0a..cba0b63e35 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -494,6 +494,16 @@ impl EditorInstance { )?), }; + cap_project::synchronize_legacy_keyboard(&recording_meta, &mut project); + cap_project::synchronize_captions( + &mut project, + &recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect::>(), + ); + let render_constants = if let Some(shared) = shared_device { let rc = RenderVideoConstants::new_with_device( shared, diff --git a/crates/editor/src/lib.rs b/crates/editor/src/lib.rs index 745ae78189..6edd1dd8d0 100644 --- a/crates/editor/src/lib.rs +++ b/crates/editor/src/lib.rs @@ -11,6 +11,10 @@ pub use audio::{AudioRenderer, MusicTracks}; pub use audio_output::{ AudioOutput, HEADLESS_BLOCK_FRAMES, HEADLESS_CHANNELS, HEADLESS_SAMPLE_RATE, HeadlessAudioTap, }; +pub use cap_audio::{ + TranscriptionAudioSource, TranscriptionAudioTake, append_transcription_audio, + assemble_transcription_audio, +}; pub use cap_rendering::FrameLayout; pub use editor::{ EditorFrameCallback, EditorFrameFormat, EditorFrameOutput, Renderer, RendererHandle, diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index efe3bcf3f3..879fd57d0c 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -159,6 +159,16 @@ impl ExporterBuilder { } } + cap_project::synchronize_legacy_keyboard(&recording_meta, &mut project_config); + cap_project::synchronize_captions( + &mut project_config, + &recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect::>(), + ); + let output_path = self .output_path .unwrap_or_else(|| recording_meta.output_path()); diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rs new file mode 100644 index 0000000000..aa7661afb4 --- /dev/null +++ b/crates/project/src/caption_timing.rs @@ -0,0 +1,504 @@ +use crate::{ + CaptionSegment, CaptionTrackSegment, CaptionWord, ProjectConfiguration, TimelineConfiguration, +}; +use std::collections::HashMap; + +const MAX_CAPTION_WORD_DURATION: f32 = 2.5; + +pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec { + let mut offset = 0.0; + timeline + .segments + .iter() + .enumerate() + .map(|(index, segment)| { + offset -= timeline + .effective_transition(index) + .map_or(0.0, |transition| transition.duration); + let start = offset; + offset += segment.duration(); + start + }) + .collect() +} + +fn caption_char_attaches_to_previous(value: char) -> bool { + matches!( + value, + ',' | '.' + | '!' + | '?' + | ';' + | ':' + | '%' + | ')' + | ']' + | '}' + | '\'' + | '’' + | '、' + | '。' + | '！' + | '？' + | '；' + | '：' + | '，' + ) +} + +fn caption_token_attaches_to_previous(text: &str) -> bool { + text.trim() + .chars() + .next() + .is_some_and(caption_char_attaches_to_previous) +} + +fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { + let mut text = String::new(); + + for word in words { + let word_text = word.text.trim(); + if word_text.is_empty() { + continue; + } + + if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { + text.push(' '); + } + text.push_str(word_text); + } + + text +} + +const CAPTION_EDL_SEPARATOR: &str = "::edl"; + +pub fn source_caption_id(track_id: &str) -> &str { + track_id + .find(CAPTION_EDL_SEPARATOR) + .map_or(track_id, |index| &track_id[..index]) +} + +fn mapped_caption_segment_id(base_id: &str, index: usize, total: usize) -> String { + if total == 1 { + base_id.to_string() + } else { + format!("{base_id}{CAPTION_EDL_SEPARATOR}{index}") + } +} + +fn clamp_caption_segment_words(segment: &CaptionSegment) -> CaptionSegment { + if segment.words.is_empty() { + return segment.clone(); + } + + let clamped_words: Vec = segment + .words + .iter() + .map(|word| CaptionWord { + text: word.text.clone(), + start: word.start, + end: word.end.min(word.start + MAX_CAPTION_WORD_DURATION), + }) + .collect(); + + let last_word_end = clamped_words.last().map_or(segment.end, |word| word.end); + + CaptionSegment { + id: segment.id.clone(), + start: segment.start, + end: segment.end.min(last_word_end), + text: segment.text.clone(), + words: clamped_words, + } +} + +struct SourceToEditedMapping { + source_start: f64, + source_end: f64, + edited_start: f64, + timescale: f64, +} + +fn build_source_to_edited_mappings( + timeline: &TimelineConfiguration, + recording_durations: &[f64], +) -> Vec { + let mut recording_offsets = Vec::with_capacity(recording_durations.len()); + let mut cumulative = 0.0; + for duration in recording_durations { + recording_offsets.push(cumulative); + cumulative += duration; + } + + let edited_offsets = clip_timeline_offsets(timeline); + + timeline + .segments + .iter() + .zip(edited_offsets) + .map(|(segment, edited_start)| { + let recording_offset = recording_offsets + .get(segment.recording_clip as usize) + .copied() + .unwrap_or(0.0); + SourceToEditedMapping { + source_start: recording_offset + segment.start, + source_end: recording_offset + segment.end, + edited_start, + timescale: segment.timescale, + } + }) + .collect() +} + +fn map_time_range_within_mapping( + start: f64, + end: f64, + mapping: &SourceToEditedMapping, +) -> Option<(f64, f64)> { + let overlap_start = start.max(mapping.source_start); + let overlap_end = end.min(mapping.source_end); + if overlap_start >= overlap_end { + return None; + } + Some(( + mapping.edited_start + (overlap_start - mapping.source_start) / mapping.timescale, + mapping.edited_start + (overlap_end - mapping.source_start) / mapping.timescale, + )) +} + +fn effective_to_output(holds: &[(f64, f64)], effective: f64) -> f64 { + let mut output = effective; + for (start, end) in holds { + if output >= *start { + output += end - start; + } else { + break; + } + } + output +} + +fn effective_to_output_end(holds: &[(f64, f64)], effective: f64) -> f64 { + let mut output = effective; + for (start, end) in holds { + if output > *start { + output += end - start; + } else { + break; + } + } + output +} + +struct MappedCaption { + id: String, + start: f64, + end: f64, + text: String, + words: Vec, +} + +fn map_captions_to_edited_timeline( + raw_segments: &[CaptionSegment], + timeline: &TimelineConfiguration, + recording_durations: &[f64], +) -> Vec { + let sanitized: Vec = raw_segments + .iter() + .map(clamp_caption_segment_words) + .collect(); + + if timeline.segments.is_empty() || recording_durations.is_empty() { + return sanitized + .into_iter() + .map(|segment| MappedCaption { + id: segment.id, + start: f64::from(segment.start), + end: f64::from(segment.end), + text: segment.text, + words: segment.words, + }) + .collect(); + } + + let mappings = build_source_to_edited_mappings(timeline, recording_durations); + let holds = timeline.hold_windows(); + let hold_adjusted = |start: f64, end: f64| { + if holds.is_empty() { + (start, end) + } else { + ( + effective_to_output(&holds, start), + effective_to_output_end(&holds, end), + ) + } + }; + + let mut result = Vec::new(); + + for caption in &sanitized { + let mut mapped_caption_segments: Vec = Vec::new(); + + for mapping in &mappings { + if !caption.words.is_empty() { + let mut mapped_words = Vec::new(); + for word in &caption.words { + let Some((start, end)) = map_time_range_within_mapping( + f64::from(word.start), + f64::from(word.end), + mapping, + ) else { + continue; + }; + let (start, end) = hold_adjusted(start, end); + mapped_words.push(CaptionWord { + text: word.text.clone(), + start: start as f32, + end: end as f32, + }); + } + + if mapped_words.is_empty() { + continue; + } + + let start = mapped_words + .first() + .map_or(f64::from(caption.start), |word| f64::from(word.start)); + let end = mapped_words + .last() + .map_or(f64::from(caption.end), |word| f64::from(word.end)); + mapped_caption_segments.push(MappedCaption { + id: caption.id.clone(), + start, + end, + text: caption_text_from_words(&mapped_words), + words: mapped_words, + }); + } else { + let Some((start, end)) = map_time_range_within_mapping( + f64::from(caption.start), + f64::from(caption.end), + mapping, + ) else { + continue; + }; + let (start, end) = hold_adjusted(start, end); + mapped_caption_segments.push(MappedCaption { + id: caption.id.clone(), + start, + end, + text: caption.text.clone(), + words: Vec::new(), + }); + } + } + + let total = mapped_caption_segments.len(); + for (index, mut segment) in mapped_caption_segments.into_iter().enumerate() { + segment.id = mapped_caption_segment_id(&caption.id, index, total); + result.push(segment); + } + } + + result +} + +pub fn derive_caption_track_segments( + source_segments: &[CaptionSegment], + timeline: &TimelineConfiguration, + recording_durations: &[f64], +) -> Vec { + struct TrackOverrides { + fade_duration: Option, + linger_duration: Option, + position: Option, + color: Option, + background_color: Option, + font_size: Option, + } + + let mut overrides_by_source_id: HashMap = HashMap::new(); + for segment in &timeline.caption_segments { + overrides_by_source_id + .entry(source_caption_id(&segment.id).to_string()) + .or_insert_with(|| TrackOverrides { + fade_duration: segment.fade_duration_override, + linger_duration: segment.linger_duration_override, + position: segment.position_override.clone(), + color: segment.color_override.clone(), + background_color: segment.background_color_override.clone(), + font_size: segment.font_size_override, + }); + } + + let mut mapped = + map_captions_to_edited_timeline(source_segments, timeline, recording_durations); + mapped.sort_by(|a, b| a.start.total_cmp(&b.start)); + + mapped + .into_iter() + .map(|segment| { + let overrides = overrides_by_source_id.get(source_caption_id(&segment.id)); + CaptionTrackSegment { + id: segment.id.clone(), + start: segment.start, + end: segment.end, + text: segment.text, + words: segment.words, + fade_duration_override: overrides.and_then(|o| o.fade_duration), + linger_duration_override: overrides.and_then(|o| o.linger_duration), + position_override: overrides.and_then(|o| o.position.clone()), + color_override: overrides.and_then(|o| o.color.clone()), + background_color_override: overrides.and_then(|o| o.background_color.clone()), + font_size_override: overrides.and_then(|o| o.font_size), + } + }) + .collect() +} + +pub fn synchronize_captions(project: &mut ProjectConfiguration, recording_durations: &[f64]) { + let (Some(captions), Some(timeline)) = (&mut project.captions, &mut project.timeline) else { + return; + }; + if recording_durations.is_empty() || timeline.segments.is_empty() { + return; + } + if !captions.source_timed { + let holds = timeline.hold_windows(); + let mappings = build_source_to_edited_mappings(timeline, recording_durations); + let to_source = |output: f64| -> Option { + let effective = output + - holds + .iter() + .map(|(start, end)| (output.min(*end) - start).max(0.0)) + .sum::(); + mappings.iter().rev().find_map(|mapping| { + let end = mapping.edited_start + + (mapping.source_end - mapping.source_start) / mapping.timescale; + (effective >= mapping.edited_start && effective <= end).then_some({ + (mapping.source_start + (effective - mapping.edited_start) * mapping.timescale) + as f32 + }) + }) + }; + captions.segments = captions + .segments + .iter() + .filter_map(|caption| { + let start = to_source(f64::from(caption.start))?; + let end = to_source(f64::from(caption.end))?; + let words = caption + .words + .iter() + .filter_map(|word| { + Some(CaptionWord { + text: word.text.clone(), + start: to_source(f64::from(word.start))?, + end: to_source(f64::from(word.end))?, + }) + }) + .collect(); + Some(CaptionSegment { + id: caption.id.clone(), + start, + end, + text: caption.text.clone(), + words, + }) + }) + .collect(); + captions + .segments + .sort_by(|a, b| a.start.total_cmp(&b.start)); + captions.source_timed = true; + } + timeline.caption_segments = + derive_caption_track_segments(&captions.segments, timeline, recording_durations); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CaptionsData; + + fn project(source_timed: bool, start: f32, end: f32) -> ProjectConfiguration { + ProjectConfiguration { + timeline: Some(serde_json::from_value(serde_json::json!({ + "segments":[{"start":0.0,"end":5.0,"timescale":1.0},{"start":6.0,"end":10.0,"timescale":1.0}], + "zoomSegments":[] + })).unwrap()), + captions: Some(CaptionsData { source_timed, segments:vec![CaptionSegment { id:"spoken".into(),start,end,text:"hello".into(),words:vec![CaptionWord { text:"hello".into(),start,end }] }], ..Default::default() }), + ..Default::default() + } + } + + #[test] + fn source_captions_follow_a_cut_on_load_and_export() { + let mut project = project(true, 8.0, 9.0); + synchronize_captions(&mut project, &[10.0]); + let track = &project.timeline.as_ref().unwrap().caption_segments; + assert_eq!(track[0].start, 7.0); + assert_eq!(track[0].end, 8.0); + assert_eq!(track[0].words[0].start, 7.0); + let saved = serde_json::to_string(&project).unwrap(); + let mut reopened: ProjectConfiguration = serde_json::from_str(&saved).unwrap(); + synchronize_captions(&mut reopened, &[10.0]); + assert_eq!(serde_json::to_string(&reopened).unwrap(), saved); + } + + #[test] + fn legacy_caption_migration_accounts_for_fullscreen_holds() { + let mut project = project(false, 9.0, 10.0); + project.timeline.as_mut().unwrap().text_segments.push( + serde_json::from_value( + serde_json::json!({"start":2.0,"end":4.0,"layout":"fullscreen"}), + ) + .unwrap(), + ); + synchronize_captions(&mut project, &[10.0]); + assert!(project.captions.as_ref().unwrap().source_timed); + assert_eq!(project.captions.as_ref().unwrap().segments[0].start, 8.0); + let track = &project.timeline.as_ref().unwrap().caption_segments; + assert_eq!(track[0].start, 9.0); + assert_eq!(track[0].end, 10.0); + } + + #[test] + fn stale_caption_tracks_are_replaced_and_styles_preserved() { + let mut project = project(true, 8.0, 9.0); + project.timeline.as_mut().unwrap().caption_segments.push(serde_json::from_value(serde_json::json!({"id":"spoken::edl0","start":100.0,"end":101.0,"text":"hello","colorOverride":"#123456"})).unwrap()); + synchronize_captions(&mut project, &[10.0]); + let track = &project.timeline.as_ref().unwrap().caption_segments; + assert_eq!(track.len(), 1); + assert_eq!(track[0].start, 7.0); + assert_eq!(track[0].color_override.as_deref(), Some("#123456")); + project.captions.as_mut().unwrap().segments.clear(); + synchronize_captions(&mut project, &[10.0]); + assert!( + project + .timeline + .as_ref() + .unwrap() + .caption_segments + .is_empty() + ); + } + + #[test] + fn source_captions_keep_display_offsets_across_silent_takes() { + let mut project = project(true, 6.0, 7.0); + project.timeline=Some(serde_json::from_value(serde_json::json!({"segments":[{"recordingSegment":1,"start":0.0,"end":5.0,"timescale":2.0}],"zoomSegments":[]})).unwrap()); + synchronize_captions(&mut project, &[5.0, 5.0]); + assert_eq!( + project.timeline.as_ref().unwrap().caption_segments[0].start, + 0.5 + ); + assert_eq!( + project.timeline.as_ref().unwrap().caption_segments[0].end, + 1.0 + ); + } +} diff --git a/crates/project/src/keyboard.rs b/crates/project/src/keyboard.rs index e5b0a6544e..b254b4540c 100644 --- a/crates/project/src/keyboard.rs +++ b/crates/project/src/keyboard.rs @@ -166,6 +166,516 @@ pub struct KeyboardTrackSegment { pub uppercase_override: Option, } +impl KeyboardTrackSegment { + pub fn remap_times(&mut self, map: impl Fn(f64) -> f64) { + let old_start = self.start; + let new_start = map(old_start); + for key in &mut self.keys { + key.time_offset = + ((map(old_start + key.time_offset / 1000.0) - new_start) * 1000.0).max(0.0); + } + self.start = new_start; + self.end = map(self.end).max(new_start); + } + + pub fn ripple_delete(&mut self, cut_start: f64, cut_end: f64, shift: f64) -> bool { + if self.end <= cut_start { + return true; + } + if self.start >= cut_start && self.end <= cut_end { + return false; + } + let old_start = self.start; + let retained: Vec = self + .keys + .iter() + .map(|key| { + let time = old_start + key.time_offset / 1000.0; + time < cut_start || time >= cut_end + }) + .collect(); + let removes_keys = retained.iter().any(|keep| !keep); + let chars: Vec = self.display_text.chars().collect(); + if removes_keys && chars.len() != self.keys.len() { + return false; + } + if removes_keys { + self.display_text = chars + .into_iter() + .zip(&retained) + .filter_map(|(ch, keep)| keep.then_some(ch)) + .collect(); + } + if self.start >= cut_end { + self.start -= shift; + self.end -= shift; + } else if self.start < cut_start && self.end > cut_end { + self.end -= shift; + } else if self.start < cut_start { + self.end = cut_start; + } else { + self.start = cut_end - shift; + self.end = (self.end - shift).max(self.start); + } + let new_start = self.start; + let mut index = 0; + self.keys.retain_mut(|key| { + let keep = retained[index]; + index += 1; + if keep { + let time = old_start + key.time_offset / 1000.0; + let mapped = if time >= cut_end { time - shift } else { time }; + key.time_offset = ((mapped - new_start) * 1000.0).max(0.0); + } + keep + }); + self.end > self.start && (!removes_keys || !self.keys.is_empty()) + } + + pub fn split_at(&self, at: f64) -> Option<(Self, Self)> { + if !at.is_finite() || at <= self.start || at >= self.end { + return None; + } + let mut left = self.clone(); + let mut right = self.clone(); + left.end = at; + right.start = at; + if self.keys.is_empty() { + return Some((left, right)); + } + let chars: Vec = self.display_text.chars().collect(); + if chars.len() != self.keys.len() { + return None; + } + left.keys.clear(); + right.keys.clear(); + left.display_text.clear(); + right.display_text.clear(); + for (key, ch) in self.keys.iter().zip(chars) { + let absolute = self.start + key.time_offset / 1000.0; + if absolute < at { + left.keys.push(key.clone()); + left.display_text.push(ch); + } else { + right.keys.push(KeyPressDisplay { + key: key.key.clone(), + time_offset: (absolute - at) * 1000.0, + }); + right.display_text.push(ch); + } + } + if left.keys.is_empty() || right.keys.is_empty() { + return None; + } + Some((left, right)) + } +} + +fn load_project_keyboard_events( + meta: &crate::RecordingMeta, +) -> Result, String> { + let Some(crate::StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + return Ok(Vec::new()); + }; + inner + .segments + .iter() + .map(|take| { + let path = take.keyboard.clone().or_else(|| { + let directory = take.display.path.parent()?; + [KEYBOARD_EVENTS_FILE_NAME, LEGACY_KEYBOARD_EVENTS_FILE_NAME] + .into_iter() + .map(|name| directory.join(name)) + .find(|path| meta.path(path).exists()) + }); + let events = match path { + Some(path) => KeyboardEvents::load_from_file(&meta.path(&path))?, + None => KeyboardEvents::default(), + }; + Ok((events, take.latest_start_time().unwrap_or(0.0))) + }) + .collect() +} + +pub fn generate_project_keyboard_segments( + meta: &crate::RecordingMeta, + timeline: &crate::TimelineConfiguration, + settings: &crate::KeyboardSettings, +) -> Result, String> { + let takes = load_project_keyboard_events(meta)?; + Ok(project_keyboard_events(&takes, timeline, settings)) +} + +fn legacy_keyboard_capture_offset(meta: &crate::RecordingMeta, recording_clip: u32) -> f64 { + match meta.studio_meta() { + Some(crate::StudioRecordingMeta::MultipleSegments { inner }) => inner + .segments + .get(recording_clip as usize) + .and_then(|take| take.latest_start_time()) + .unwrap_or(0.0), + _ => 0.0, + } +} + +fn project_legacy_keyboard_track( + meta: &crate::RecordingMeta, + timeline: &crate::TimelineConfiguration, + settings: &crate::KeyboardSettings, +) -> Vec { + let offsets = crate::caption_timing::clip_timeline_offsets(timeline); + let holds = timeline.hold_windows(); + let to_output = |effective: f64, end: bool| { + let mut output = effective; + for (start, finish) in &holds { + if output > *start || (!end && output == *start) { + output += finish - start; + } else { + break; + } + } + output + }; + let mut result: Vec<_> = timeline + .keyboard_segments + .iter() + .filter(|segment| segment.id.starts_with("kb-edit-")) + .cloned() + .collect(); + for (index, clip) in timeline.segments.iter().enumerate() { + if !clip.timescale.is_finite() || clip.timescale <= 0.0 { + continue; + } + let offset = legacy_keyboard_capture_offset(meta, clip.recording_clip); + let source_start = clip.start + offset; + let source_end = clip.end + offset; + for previous in &timeline.keyboard_segments { + if previous.id.starts_with("kb-edit-") { + continue; + } + let start = previous.start.max(source_start); + let end = previous.end.min(source_end); + if !start.is_finite() || !end.is_finite() || end <= start { + continue; + } + let map = |time: f64, end: bool| { + to_output(offsets[index] + (time - source_start) / clip.timescale, end) + }; + let mut segment = previous.clone(); + segment.id = format!("kb-edit-legacy-{index}-{}", previous.id); + segment.start = map(start, false); + segment.end = map(end, true); + let retained: Vec = previous + .keys + .iter() + .map(|key| { + let time = previous.start + key.time_offset / 1000.0; + time >= source_start && time < source_end + }) + .collect(); + if retained.iter().any(|keep| !keep) { + let chars: Vec = previous.display_text.chars().collect(); + if chars.len() != previous.keys.len() || !retained.iter().any(|keep| *keep) { + continue; + } + segment.display_text = chars + .into_iter() + .zip(&retained) + .filter_map(|(ch, keep)| keep.then_some(ch)) + .collect(); + } + segment.keys = previous + .keys + .iter() + .zip(retained) + .filter(|(_, keep)| *keep) + .map(|(key, _)| KeyPressDisplay { + key: key.key.clone(), + time_offset: ((map(previous.start + key.time_offset / 1000.0, false) + - segment.start) + * 1000.0) + .max(0.0), + }) + .collect(); + segment.fade_duration_override = Some( + previous + .fade_duration_override + .unwrap_or(settings.fade_duration) + / clip.timescale as f32, + ); + result.push(segment); + } + } + result.sort_by(|a, b| a.start.total_cmp(&b.start)); + result +} + +fn legacy_keyboard_tracks_match( + generated: &[KeyboardTrackSegment], + saved: &[KeyboardTrackSegment], +) -> bool { + generated.len() == saved.len() + && generated.iter().zip(saved).all(|(generated, saved)| { + generated.id == saved.id + && (generated.start - saved.start).abs() <= 1e-6 + && (generated.end - saved.end).abs() <= 1e-6 + && generated.display_text == saved.display_text + && generated.keys.len() == saved.keys.len() + && generated + .keys + .iter() + .zip(&saved.keys) + .all(|(a, b)| a.key == b.key && (a.time_offset - b.time_offset).abs() <= 1e-3) + }) +} + +fn legacy_keyboard_after_cuts( + legacy: &[KeyboardTrackSegment], + timeline: &crate::TimelineConfiguration, +) -> Option> { + if timeline.segments.is_empty() + || !timeline.transitions.is_empty() + || !timeline.hold_windows().is_empty() + || timeline.segments.iter().any(|clip| { + clip.recording_clip != 0 + || !clip.timescale.is_finite() + || (clip.timescale - 1.0).abs() > 1e-6 + || !clip.start.is_finite() + || clip.start < 0.0 + || !clip.end.is_finite() + || clip.end <= clip.start + }) + || timeline + .segments + .windows(2) + .any(|pair| pair[0].end > pair[1].start) + { + return None; + } + let mut projected = legacy.to_vec(); + for index in (0..timeline.segments.len()).rev() { + let cut_start = index + .checked_sub(1) + .map(|previous| timeline.segments[previous].end) + .unwrap_or(0.0); + let cut_end = timeline.segments[index].start; + let shift = cut_end - cut_start; + if shift <= 1e-6 { + continue; + } + projected.retain_mut(|segment| { + if segment.end <= cut_start { + return true; + } + if segment.start >= cut_end { + segment.start -= shift; + segment.end -= shift; + } else if segment.start >= cut_start && segment.end <= cut_end { + return false; + } else if segment.start < cut_start && segment.end > cut_end { + segment.end -= shift; + } else if segment.start < cut_start { + segment.end = cut_start; + } else { + segment.start = cut_start; + segment.end = (segment.end - shift).max(cut_start); + } + true + }); + } + Some(projected) +} + +pub fn synchronize_legacy_keyboard( + meta: &crate::RecordingMeta, + project: &mut crate::ProjectConfiguration, +) { + let Some(timeline) = &mut project.timeline else { + return; + }; + if timeline.keyboard_segments.is_empty() + || timeline + .keyboard_segments + .iter() + .all(|segment| segment.id.starts_with("kb-edit-")) + { + return; + } + let settings = project + .keyboard + .as_ref() + .map(|keyboard| keyboard.settings.clone()) + .unwrap_or_default(); + let takes = load_project_keyboard_events(meta).ok(); + let mut events = KeyboardEvents { + presses: takes + .as_ref() + .into_iter() + .flatten() + .flat_map(|(events, _)| events.presses.clone()) + .collect(), + }; + events + .presses + .sort_by(|a, b| a.time_ms.total_cmp(&b.time_ms)); + let legacy = group_key_events( + &events, + settings.grouping_threshold_ms, + f64::from(settings.linger_duration) * 1000.0, + settings.show_modifiers, + settings.show_special_keys, + ); + let unchanged = + takes.is_some() && legacy_keyboard_tracks_match(&legacy, &timeline.keyboard_segments); + let stale_cut_track = takes.as_ref().is_some_and(|takes| takes.len() == 1) + && legacy_keyboard_after_cuts(&legacy, timeline).is_some_and(|projected| { + legacy_keyboard_tracks_match(&projected, &timeline.keyboard_segments) + }); + if !unchanged && !stale_cut_track { + timeline.keyboard_segments = project_legacy_keyboard_track(meta, timeline, &settings); + return; + } + let mut generated = + project_keyboard_events(takes.as_deref().unwrap_or_default(), timeline, &settings); + let clip_offsets = crate::caption_timing::clip_timeline_offsets(timeline); + let holds = timeline.hold_windows(); + for segment in &mut generated { + let Some(index) = segment + .id + .strip_prefix("kb-edit-") + .and_then(|id| id.split_once('-')) + .and_then(|(index, _)| index.parse::().ok()) + else { + continue; + }; + let Some(clip) = timeline.segments.get(index) else { + continue; + }; + let held_time: f64 = holds + .iter() + .map(|(start, end)| (segment.start - start).clamp(0.0, end - start)) + .sum(); + let source_time = + clip.start + (segment.start - held_time - clip_offsets[index]) * clip.timescale; + let captured = source_time + legacy_keyboard_capture_offset(meta, clip.recording_clip); + let Some(source) = legacy + .iter() + .rev() + .find(|source| captured >= source.start && captured < source.end) + else { + continue; + }; + let Some(previous) = timeline + .keyboard_segments + .iter() + .find(|previous| previous.id == source.id) + else { + continue; + }; + segment.fade_duration_override = previous.fade_duration_override; + segment.position_override = previous.position_override.clone(); + segment.color_override = previous.color_override.clone(); + segment.background_color_override = previous.background_color_override.clone(); + segment.font_size_override = previous.font_size_override; + segment.uppercase_override = previous.uppercase_override; + } + timeline.keyboard_segments = generated; +} + +fn project_keyboard_events( + takes: &[(KeyboardEvents, f64)], + timeline: &crate::TimelineConfiguration, + settings: &crate::KeyboardSettings, +) -> Vec { + let holds = timeline.hold_windows(); + let output_time = |time: f64, end: bool| { + let mut output = time; + for (start, finish) in &holds { + if output > *start || (!end && output == *start) { + output += finish - start; + } else { + break; + } + } + output + }; + let mut edited_start = 0.0; + let mut result = Vec::new(); + for (index, clip) in timeline.segments.iter().enumerate() { + if !clip.start.is_finite() + || !clip.end.is_finite() + || !clip.timescale.is_finite() + || clip.timescale <= 0.0 + || clip.end <= clip.start + { + continue; + } + edited_start -= timeline + .effective_transition(index) + .map_or(0.0, |transition| transition.duration); + let edited_end = edited_start + clip.duration(); + if let Some((events, capture_offset)) = takes.get(clip.recording_clip as usize) + && capture_offset.is_finite() + { + let source_start = clip.start + capture_offset; + let source_end = clip.end + capture_offset; + let mut presses = Vec::new(); + let mut modifiers = Vec::new(); + let mut ordered: Vec<_> = events + .presses + .iter() + .filter(|event| event.time_ms.is_finite()) + .collect(); + ordered.sort_by(|a, b| a.time_ms.total_cmp(&b.time_ms)); + for event in ordered { + let source = event.time_ms / 1000.0; + if source < source_start { + if is_modifier_key(&event.key) { + modifiers.retain(|key: &KeyPressEvent| key.key != event.key); + if event.down { + modifiers.push(event.clone()); + } + } + continue; + } + if source >= source_end { + break; + } + if presses.is_empty() { + for mut modifier in modifiers.drain(..) { + modifier.time_ms = output_time(edited_start, false) * 1000.0; + presses.push(modifier); + } + } + let mut projected = event.clone(); + projected.time_ms = output_time( + edited_start + (source - source_start) / clip.timescale, + false, + ) * 1000.0; + presses.push(projected); + } + let mut grouped = group_key_events( + &KeyboardEvents { presses }, + settings.grouping_threshold_ms, + f64::from(settings.linger_duration) * 1000.0, + settings.show_modifiers, + settings.show_special_keys, + ); + for segment in &mut grouped { + segment.id = format!("kb-edit-{index}-{}", segment.id); + segment.end = segment.end.min(output_time(edited_end, true)); + } + result.extend( + grouped + .into_iter() + .filter(|segment| segment.end > segment.start), + ); + } + edited_start = edited_end; + } + result.sort_by(|a, b| a.start.total_cmp(&b.start)); + result +} + pub fn group_key_events( events: &KeyboardEvents, grouping_threshold_ms: f64, @@ -737,3 +1247,460 @@ mod tests { assert_eq!(segments[0].display_text, "⌘W"); } } + +#[cfg(test)] +mod timing_tests { + use super::*; + use crate::{KeyboardSettings, TimelineConfiguration}; + + fn timeline(segments: serde_json::Value) -> TimelineConfiguration { + serde_json::from_value(serde_json::json!({"segments": segments, "zoomSegments": []})) + .unwrap() + } + + fn events(keys: &[(&str, f64)]) -> KeyboardEvents { + KeyboardEvents { + presses: keys + .iter() + .map(|(key, time)| KeyPressEvent { + key: (*key).into(), + key_code: (*key).into(), + time_ms: time * 1000.0, + down: true, + }) + .collect(), + } + } + + #[test] + fn legacy_tracks_migrate_once_and_preserve_authored_payloads() { + let directory = tempfile::tempdir().unwrap(); + let captured = events(&[("a", 8.2)]); + captured + .write_to_file(&directory.path().join("keyboard.bin")) + .unwrap(); + let mut meta: crate::RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name":"legacy", "segments":[{"display":{"path":"display.mp4","fps":30,"start_time":0.2},"keyboard":"keyboard.bin"}],"cursors":{} + })).unwrap(); + meta.project_path = directory.path().to_path_buf(); + let mut project = crate::ProjectConfiguration { + keyboard: Some(crate::KeyboardData::default()), + timeline: Some(timeline(serde_json::json!([ + {"start":0.0,"end":5.0,"timescale":1.0}, + {"start":6.0,"end":10.0,"timescale":1.0} + ]))), + ..Default::default() + }; + project.timeline.as_mut().unwrap().keyboard_segments = + group_key_events(&captured, 500.0, 800.0, true, true); + project.timeline.as_mut().unwrap().keyboard_segments[0].color_override = + Some("#123456".into()); + let mut manual = project.clone(); + manual.timeline.as_mut().unwrap().keyboard_segments[0].start += 0.1; + synchronize_legacy_keyboard(&meta, &mut manual); + let migrated_manual = &manual.timeline.as_ref().unwrap().keyboard_segments[0]; + assert!((migrated_manual.start - 7.1).abs() < 1e-9); + assert_eq!(migrated_manual.display_text, "a"); + assert_eq!(migrated_manual.color_override.as_deref(), Some("#123456")); + let original_manual = serde_json::to_string(&manual).unwrap(); + synchronize_legacy_keyboard(&meta, &mut manual); + assert_eq!(serde_json::to_string(&manual).unwrap(), original_manual); + synchronize_legacy_keyboard(&meta, &mut project); + let segment = &project.timeline.as_ref().unwrap().keyboard_segments[0]; + assert!((segment.start - 7.0).abs() < 1e-9); + assert_eq!(segment.color_override.as_deref(), Some("#123456")); + let migrated = serde_json::to_string(&project).unwrap(); + synchronize_legacy_keyboard(&meta, &mut project); + assert_eq!(serde_json::to_string(&project).unwrap(), migrated); + assert_eq!( + KeyboardEvents::load_from_file(&directory.path().join("keyboard.bin")) + .unwrap() + .presses, + captured.presses + ); + } + + #[test] + fn legacy_generated_tracks_already_rippled_by_a_cut_are_not_shifted_twice() { + let directory = tempfile::tempdir().unwrap(); + let captured = events(&[("a", 8.2)]); + captured + .write_to_file(&directory.path().join("keyboard.bin")) + .unwrap(); + let mut meta: crate::RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name":"old cut", "segments":[ + {"display":{"path":"display.mp4","fps":30,"start_time":0.2},"keyboard":"keyboard.bin"} + ],"cursors":{} + })).unwrap(); + meta.project_path = directory.path().to_path_buf(); + let mut project = crate::ProjectConfiguration { + keyboard: Some(crate::KeyboardData::default()), + timeline: Some(timeline(serde_json::json!([ + {"start":0.0,"end":5.0,"timescale":1.0}, + {"start":6.0,"end":10.0,"timescale":1.0} + ]))), + ..Default::default() + }; + let mut track = group_key_events(&captured, 500.0, 800.0, true, true).remove(0); + track.start -= 1.0; + track.end -= 1.0; + track.color_override = Some("#123456".into()); + project + .timeline + .as_mut() + .unwrap() + .keyboard_segments + .push(track); + let mut prefix_cut = project.clone(); + prefix_cut.timeline.as_mut().unwrap().segments = timeline(serde_json::json!([ + {"start":1.0,"end":10.0,"timescale":1.0} + ])) + .segments; + synchronize_legacy_keyboard(&meta, &mut prefix_cut); + assert!( + (prefix_cut.timeline.as_ref().unwrap().keyboard_segments[0].start - 7.0).abs() < 1e-9 + ); + synchronize_legacy_keyboard(&meta, &mut project); + let segment = &project.timeline.as_ref().unwrap().keyboard_segments[0]; + assert!((segment.start - 7.0).abs() < 1e-9); + assert!((segment.end - 7.8).abs() < 1e-6); + assert_eq!(segment.display_text, "a"); + assert_eq!(segment.color_override.as_deref(), Some("#123456")); + let saved = serde_json::to_string(&project).unwrap(); + synchronize_legacy_keyboard(&meta, &mut project); + assert_eq!(serde_json::to_string(&project).unwrap(), saved); + } + + #[test] + fn legacy_styles_follow_their_take_during_a_transition() { + let directory = tempfile::tempdir().unwrap(); + let first = events(&[("a", 4.5)]); + let second = events(&[("b", 0.5)]); + first + .write_to_file(&directory.path().join("first.bin")) + .unwrap(); + second + .write_to_file(&directory.path().join("second.bin")) + .unwrap(); + let mut meta: crate::RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name":"transition", "segments":[ + {"display":{"path":"first.mp4","fps":30},"keyboard":"first.bin"}, + {"display":{"path":"second.mp4","fps":30},"keyboard":"second.bin"} + ],"cursors":{} + })) + .unwrap(); + meta.project_path = directory.path().to_path_buf(); + let timeline = serde_json::from_value(serde_json::json!({ + "segments":[ + {"recordingSegment":0,"start":0.0,"end":5.0,"timescale":1.0}, + {"recordingSegment":1,"start":0.0,"end":5.0,"timescale":1.0} + ], "zoomSegments":[], + "transitions":[{"segmentIndex":1,"type":"cross-fade","duration":1.0}] + })) + .unwrap(); + let mut project = crate::ProjectConfiguration { + keyboard: Some(crate::KeyboardData::default()), + timeline: Some(timeline), + ..Default::default() + }; + let mut legacy = + group_key_events(&events(&[("b", 0.5), ("a", 4.5)]), 500.0, 800.0, true, true); + legacy[0].color_override = Some("#222222".into()); + legacy[1].color_override = Some("#111111".into()); + project.timeline.as_mut().unwrap().keyboard_segments = legacy; + synchronize_legacy_keyboard(&meta, &mut project); + let segments = &project.timeline.unwrap().keyboard_segments; + assert_eq!(segments.len(), 2); + for segment in segments { + assert!((segment.start - 4.5).abs() < 1e-9); + assert_eq!( + segment.color_override.as_deref(), + Some(if segment.display_text == "a" { + "#111111" + } else { + "#222222" + }) + ); + } + } + + #[test] + fn authored_legacy_keyboard_projects_speed_holds_and_repeated_takes_without_logs() { + let meta: crate::RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name":"manual", "segments":[ + {"display":{"path":"display.mp4","fps":30,"start_time":0.2}} + ],"cursors":{} + })) + .unwrap(); + let mut project = crate::ProjectConfiguration { + timeline: Some(timeline(serde_json::json!([ + {"start":2.0,"end":6.0,"timescale":2.0}, + {"start":2.0,"end":6.0,"timescale":1.0} + ]))), + ..Default::default() + }; + let mut track = group_key_events( + &events(&[("a", 2.8), ("b", 3.6)]), + 1000.0, + 1000.0, + true, + true, + ) + .remove(0); + track.id = "kb-split-authored".into(); + track.color_override = Some("#123456".into()); + project + .timeline + .as_mut() + .unwrap() + .keyboard_segments + .push(track); + project.timeline.as_mut().unwrap().text_segments = + serde_json::from_value(serde_json::json!([ + {"start":0.5,"end":1.5,"layout":"fullscreen","enabled":true} + ])) + .unwrap(); + synchronize_legacy_keyboard(&meta, &mut project); + let segments = &project.timeline.as_ref().unwrap().keyboard_segments; + assert_eq!(segments.len(), 2); + assert!((segments[0].start - 0.3).abs() < 1e-9); + assert!((segments[0].keys[1].time_offset - 1400.0).abs() < 1e-6); + assert!((segments[1].start - 3.6).abs() < 1e-9); + assert!((segments[1].keys[1].time_offset - 800.0).abs() < 1e-6); + assert!( + segments + .iter() + .all(|segment| segment.id.starts_with("kb-edit-") + && segment.display_text == "ab" + && segment.color_override.as_deref() == Some("#123456")) + ); + let saved = serde_json::to_string(&project).unwrap(); + let mut reopened = serde_json::from_str(&saved).unwrap(); + synchronize_legacy_keyboard(&meta, &mut reopened); + assert_eq!(serde_json::to_string(&reopened).unwrap(), saved); + } + + #[test] + fn authored_legacy_keyboard_discards_keys_from_deleted_footage() { + let meta: crate::RecordingMeta = serde_json::from_value(serde_json::json!({ + "pretty_name":"manual cut", "segments":[ + {"display":{"path":"display.mp4","fps":30}} + ],"cursors":{} + })) + .unwrap(); + let mut project = crate::ProjectConfiguration { + timeline: Some(timeline(serde_json::json!([ + {"start":1.0,"end":4.0,"timescale":1.0}, + {"start":6.0,"end":9.0,"timescale":1.0} + ]))), + ..Default::default() + }; + let mut track = group_key_events( + &events(&[("a", 1.0), ("b", 5.0), ("c", 8.0)]), + 5000.0, + 1000.0, + true, + true, + ) + .remove(0); + track.id = "authored".into(); + track.display_text = "abé".into(); + track.color_override = Some("#123456".into()); + project + .timeline + .as_mut() + .unwrap() + .keyboard_segments + .push(track); + synchronize_legacy_keyboard(&meta, &mut project); + let segments = &project.timeline.as_ref().unwrap().keyboard_segments; + assert_eq!(segments.len(), 2); + assert_eq!(segments[0].display_text, "a"); + assert_eq!(segments[1].display_text, "é"); + assert_eq!(segments[1].start, 3.0); + assert_eq!(segments[1].keys[0].time_offset, 2000.0); + assert!(segments.iter().all(|segment| segment.keys.len() == 1 + && segment.color_override.as_deref() == Some("#123456"))); + } + + #[test] + fn regeneration_reports_corrupt_capture_logs() { + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("keyboard.bin"), + b"invalid capture log", + ) + .unwrap(); + let mut meta: crate::RecordingMeta = serde_json::from_value(serde_json::json!({"pretty_name":"corrupt", "segments":[{"display":{"path":"display.mp4","fps":30,"start_time":0.0},"keyboard":"keyboard.bin"}],"cursors":{}})).unwrap(); + meta.project_path = directory.path().to_path_buf(); + let timeline = timeline(serde_json::json!([{ "start":0.0,"end":10.0,"timescale":1.0 }])); + assert!( + generate_project_keyboard_segments(&meta, &timeline, &KeyboardSettings::default()) + .is_err() + ); + } + + #[test] + fn regeneration_uses_retained_frames_and_capture_start_offset() { + let timeline = timeline(serde_json::json!([ + {"start":0.0,"end":5.0,"timescale":1.0}, + {"start":6.0,"end":10.0,"timescale":1.0} + ])); + let takes = [(events(&[("x", 5.7), ("a", 8.2)]), 0.2)]; + let result = project_keyboard_events(&takes, &timeline, &KeyboardSettings::default()); + assert_eq!(result.len(), 1); + assert_eq!(result[0].display_text, "a"); + assert!((result[0].start - 7.0).abs() < 1e-9); + let (source_time, _) = timeline.get_segment_time(result[0].start).unwrap(); + assert!((source_time + takes[0].1 - 8.2).abs() < 1e-9); + } + + #[test] + fn regeneration_keeps_take_identity_and_repeated_clip_ids() { + let timeline = timeline(serde_json::json!([ + {"recordingSegment":1,"start":0.0,"end":3.0,"timescale":1.0}, + {"recordingSegment":0,"start":0.0,"end":3.0,"timescale":1.0}, + {"recordingSegment":1,"start":0.0,"end":3.0,"timescale":1.0} + ])); + let result = project_keyboard_events( + &[(events(&[("a", 1.0)]), 0.0), (events(&[("b", 1.0)]), 0.0)], + &timeline, + &KeyboardSettings::default(), + ); + assert_eq!( + result + .iter() + .map(|s| (s.display_text.as_str(), s.start)) + .collect::>(), + vec![("b", 1.0), ("a", 4.0), ("b", 7.0)] + ); + assert_ne!(result[0].id, result[2].id); + } + + #[test] + fn regeneration_uses_transition_adjusted_clip_offsets() { + let mut timeline = timeline(serde_json::json!([ + {"recordingSegment":0,"start":0.0,"end":5.0,"timescale":1.0}, + {"recordingSegment":1,"start":0.0,"end":5.0,"timescale":1.0} + ])); + timeline.transitions = serde_json::from_value( + serde_json::json!([{"segmentIndex":1,"type":"cross-fade","duration":1.0}]), + ) + .unwrap(); + let result = project_keyboard_events( + &[ + (KeyboardEvents::default(), 0.0), + (events(&[("b", 1.0)]), 0.0), + ], + &timeline, + &KeyboardSettings::default(), + ); + assert_eq!(result[0].start, 5.0); + let (source, clip) = timeline.get_segment_time(result[0].start).unwrap(); + assert_eq!(source, 1.0); + assert_eq!(clip.recording_clip, 1); + } + + #[test] + fn regeneration_scales_individual_key_offsets() { + let timeline = timeline(serde_json::json!([{ "start":7.0,"end":10.0,"timescale":2.0 }])); + let result = project_keyboard_events( + &[(events(&[("a", 8.0), ("b", 8.4)]), 0.0)], + &timeline, + &KeyboardSettings::default(), + ); + assert_eq!(result.len(), 1); + assert_eq!(result[0].start, 0.5); + assert!((result[0].keys[1].time_offset - 200.0).abs() < 1e-9); + } + + #[test] + fn regeneration_does_not_type_during_fullscreen_holds() { + let mut timeline = + timeline(serde_json::json!([{ "start":0.0,"end":10.0,"timescale":1.0 }])); + timeline.text_segments.push( + serde_json::from_value( + serde_json::json!({"start":2.0,"end":4.0,"layout":"fullscreen"}), + ) + .unwrap(), + ); + let result = project_keyboard_events( + &[(events(&[("a", 2.0), ("b", 3.0)]), 0.0)], + &timeline, + &KeyboardSettings::default(), + ); + assert_eq!(result[0].start, 4.0); + assert_eq!(result[1].start, 5.0); + } + + #[test] + fn regeneration_preserves_modifiers_held_across_trim() { + let timeline = timeline(serde_json::json!([{ "start":2.0,"end":5.0,"timescale":1.0 }])); + let result = project_keyboard_events( + &[(events(&[("LMeta", 1.0), ("w", 2.2)]), 0.0)], + &timeline, + &KeyboardSettings::default(), + ); + assert_eq!(result.len(), 1); + assert_eq!(result[0].display_text, "⌘W"); + assert!((result[0].start - 0.2).abs() < 1e-9); + } + + fn typed_segment() -> KeyboardTrackSegment { + group_key_events( + &events(&[("a", 10.0), ("b", 11.0), ("c", 12.0)]), + 1500.0, + 1000.0, + true, + true, + ) + .remove(0) + } + + #[test] + fn ripple_removes_cut_keys_and_rebases_survivors() { + let mut segment = typed_segment(); + assert!(segment.ripple_delete(10.5, 11.5, 1.0)); + assert_eq!(segment.display_text, "ac"); + assert_eq!(segment.start, 10.0); + assert_eq!(segment.end, 12.0); + assert_eq!(segment.keys[1].time_offset, 1000.0); + } + + #[test] + fn ripple_right_tail_uses_actual_duration_shift() { + let mut segment = typed_segment(); + assert!(segment.ripple_delete(9.5, 11.5, 1.0)); + assert_eq!(segment.start, 10.5); + assert_eq!(segment.end, 12.0); + assert_eq!(segment.display_text, "c"); + assert_eq!(segment.keys[0].time_offset, 500.0); + } + + #[test] + fn speed_remaps_key_times_with_segment_bounds() { + let mut segment = typed_segment(); + segment.remap_times(|time| 10.0 + (time - 10.0) / 2.0); + assert_eq!(segment.end, 11.5); + assert_eq!(segment.keys[1].time_offset, 500.0); + assert_eq!(segment.keys[2].time_offset, 1000.0); + } + + #[test] + fn split_partitions_and_rebases_generated_keys() { + let segment = typed_segment(); + let (left, right) = segment.split_at(11.0).unwrap(); + assert_eq!(left.display_text, "a"); + assert_eq!(right.display_text, "bc"); + assert_eq!(right.keys[0].time_offset, 0.0); + assert_eq!(right.keys[1].time_offset, 1000.0); + } + + #[test] + fn atomic_shortcuts_are_preserved_or_removed_without_corruption() { + let mut segment = typed_segment(); + segment.display_text = "⌘W".into(); + assert!(segment.split_at(11.0).is_none()); + assert!(!segment.ripple_delete(10.5, 11.5, 1.0)); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 0e09fde35e..dea389ee13 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -1,10 +1,12 @@ mod animated_gradient; +mod caption_timing; mod configuration; pub mod cursor; pub mod keyboard; mod meta; pub use animated_gradient::*; +pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions}; pub use configuration::*; pub use cursor::*; pub use keyboard::*; diff --git a/crates/project/src/meta.rs b/crates/project/src/meta.rs index 5e023f4fad..cc7818587e 100644 --- a/crates/project/src/meta.rs +++ b/crates/project/src/meta.rs @@ -254,7 +254,9 @@ impl RecordingMeta { let captions_path = self.project_path.join("captions.json"); debug!("Checking for captions at: {:?}", captions_path); - if let Ok(captions_str) = std::fs::read_to_string(&captions_path) { + if config.captions.is_none() + && let Ok(captions_str) = std::fs::read_to_string(&captions_path) + { debug!("Found captions.json, attempting to parse"); if let Ok(captions_data) = serde_json::from_str::(&captions_str) { info!( @@ -265,7 +267,7 @@ impl RecordingMeta { } else { warn!("Failed to parse captions.json"); } - } else { + } else if config.captions.is_none() { debug!("No captions.json found"); } @@ -809,6 +811,44 @@ mod metadata_save_tests { })); } + #[test] + fn saved_caption_master_takes_precedence_over_legacy_sidecar() { + let directory = tempfile::tempdir().unwrap(); + let meta = recording(directory.path()); + let captions = |text: &str| CaptionsData { + source_timed: true, + segments: vec![crate::CaptionSegment { + id: "caption".into(), + start: 1.0, + end: 2.0, + text: text.into(), + words: Vec::new(), + }], + ..Default::default() + }; + let config = ProjectConfiguration { + captions: Some(captions("edited")), + ..Default::default() + }; + config.write(directory.path()).unwrap(); + std::fs::write( + directory.path().join("captions.json"), + serde_json::to_vec(&captions("stale")).unwrap(), + ) + .unwrap(); + assert_eq!( + meta.project_config().captions.unwrap().segments[0].text, + "edited" + ); + ProjectConfiguration::default() + .write(directory.path()) + .unwrap(); + assert_eq!( + meta.project_config().captions.unwrap().segments[0].text, + "stale" + ); + } + #[test] fn new_metadata_preserves_legacy_serialization_and_loads() { let project = tempfile::tempdir().unwrap(); diff --git a/crates/rendering/src/layers/keyboard.rs b/crates/rendering/src/layers/keyboard.rs index f5ac3b6ecd..5134526046 100644 --- a/crates/rendering/src/layers/keyboard.rs +++ b/crates/rendering/src/layers/keyboard.rs @@ -278,7 +278,7 @@ impl KeyboardLayer { pub fn prepare( &mut self, uniforms: &ProjectUniforms, - segment_frames: &DecodedSegmentFrames, + _segment_frames: &DecodedSegmentFrames, output_size: XY, constants: &RenderVideoConstants, caption_layout: Option, @@ -304,12 +304,7 @@ impl KeyboardLayer { return; } - // Keyboard segments are authored on the recording clock. Use the same - // clock the cursor layer uses: recording_time travels with the decoded - // frame through the timeline mapping, so the overlay follows cuts and - // trims, and includes the recording→first-video-frame start offset - // that raw output time (frame_number / frame_rate) lacks. - let current_time = segment_frames.recording_time as f64; + let current_time = uniforms.frame_number as f64 / uniforms.frame_rate as f64; let settings = &keyboard_data.settings; let active_segment = find_active_keyboard_segment( @@ -609,22 +604,22 @@ fn find_active_keyboard_segment<'a>( segments: &'a [cap_project::KeyboardTrackSegment], default_fade_duration: f32, ) -> Option> { - for segment in segments { - if time >= segment.start && time < segment.end { - return Some(ActiveKeyboardSegment { segment }); - } - } - - for segment in segments { - let fade = segment - .fade_duration_override - .unwrap_or(default_fade_duration) as f64; - if time >= segment.end && time < segment.end + fade { - return Some(ActiveKeyboardSegment { segment }); - } - } - - None + segments + .iter() + .filter(|segment| time >= segment.start && time < segment.end) + .max_by(|a, b| a.start.total_cmp(&b.start)) + .or_else(|| { + segments + .iter() + .filter(|segment| { + let fade = segment + .fade_duration_override + .unwrap_or(default_fade_duration) as f64; + time >= segment.end && time < segment.end + fade + }) + .max_by(|a, b| a.end.total_cmp(&b.end)) + }) + .map(|segment| ActiveKeyboardSegment { segment }) } fn build_visible_text(segment: &cap_project::KeyboardTrackSegment, current_time: f64) -> String { @@ -710,10 +705,31 @@ fn calculate_keyboard_bounce(current_time: f64, start: f64, end: f64, fade_durat #[cfg(test)] mod tests { use super::{ - KeyboardPosition, build_visible_text, resolve_background_top, resolve_keyboard_position, + KeyboardPosition, build_visible_text, find_active_keyboard_segment, resolve_background_top, + resolve_keyboard_position, }; use crate::layers::{CaptionOverlayLayout, CaptionPosition}; + #[test] + fn new_typing_is_not_hidden_by_the_previous_words_linger() { + let segment = |id: &str, start: f64, end: f64| -> cap_project::KeyboardTrackSegment { + serde_json::from_value( + serde_json::json!({"id":id,"start":start,"end":end,"displayText":id}), + ) + .unwrap() + }; + let segments = [segment("previous", 7.0, 7.8), segment("current", 7.2, 8.0)]; + let active = find_active_keyboard_segment(7.5, &segments, 0.15).unwrap(); + assert_eq!(build_visible_text(active.segment, 7.5), "current"); + assert_eq!( + find_active_keyboard_segment(8.1, &segments, 0.15) + .unwrap() + .segment + .id, + "current" + ); + } + #[test] fn shortcut_segments_show_the_full_combo() { let segment = cap_project::KeyboardTrackSegment {