diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index db1004fef56..0d30f40c49d 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -2110,6 +2110,7 @@ fn audio_project_key(project: &ProjectConfiguration) -> Option> { screen_motion_blur: _, screen_movement_spring: _, color_correction: _, + clip_alignment_version: _, text_size_version: _, text_anim_version: _, markers: _, diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 3e626fa0894..8c3ff157ac2 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -524,12 +524,27 @@ impl EditorInstance { } } + // Before any offsets are filled in below: those are already on the + // current segment clock. + let migrated = cap_project::migrate_late_audio_alignment(&mut project, meta, || { + match &preloaded_recordings { + Some(recordings) => recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect(), + None => display_durations(&recording_meta, meta), + } + }); + if project.clips.is_empty() { project.clips = initial_clip_configuration(&recording_meta.project_path, meta); if let Err(e) = project.write(&recording_meta.project_path) { warn!("Failed to save auto-generated clip offsets: {}", e); } + } else if migrated && let Err(e) = project.write(&recording_meta.project_path) { + warn!("Failed to save late audio alignment: {}", e); } // Segment setup (decoder init + kicking off audio decodes) is @@ -1984,6 +1999,22 @@ pub fn initial_clip_configuration( } } +/// Each clip's display duration, the source clock captions are timed on. +fn display_durations(recording_meta: &RecordingMeta, meta: &StudioRecordingMeta) -> Vec { + match meta { + StudioRecordingMeta::SingleSegment { segment } => { + vec![display_video_duration(&recording_meta.path(&segment.display.path)).unwrap_or(0.0)] + } + StudioRecordingMeta::MultipleSegments { inner } => inner + .segments + .iter() + .map(|segment| { + display_video_duration(&recording_meta.path(&segment.display.path)).unwrap_or(0.0) + }) + .collect(), + } +} + fn load_calibration_store(project_path: &std::path::Path) -> cap_audio::CalibrationStore { let calibration_dir = project_path .parent() diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 7b5cd02bd80..f55c769b1bf 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -139,6 +139,11 @@ impl ExporterBuilder { &mut project_config, &recording_meta.project_path, studio_meta, + &recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect::>(), ); cap_project::clamp_timeline_to_sources( @@ -839,12 +844,16 @@ mod forward_read_tests { /// editor fills them in from the tracks' start times (and the devices' /// calibration) when it first opens the project. An export of a project the /// editor never opened would otherwise play every audio track from its own -/// start, late against the screen by how much earlier it started. +/// start, late against the screen by how much earlier it started. A project +/// saved before late audio stopped setting a segment's start is first moved +/// onto the current segment clock, the way the editor does on open. fn fill_clip_offsets( config: &mut ProjectConfiguration, project_path: &std::path::Path, studio_meta: &StudioRecordingMeta, + recording_durations: &[f64], ) { + cap_project::migrate_late_audio_alignment(config, studio_meta, || recording_durations.to_vec()); if config.clips.is_empty() { config.clips = cap_editor::initial_clip_configuration(project_path, studio_meta); } @@ -895,7 +904,7 @@ mod clip_offset_tests { let meta = camera_recording(); let mut config = ProjectConfiguration::default(); assert!(config.clips.is_empty()); - fill_clip_offsets(&mut config, dir.path(), &meta); + fill_clip_offsets(&mut config, dir.path(), &meta, &[10.0]); let offsets = config.clips[0].offsets; assert!((offsets.mic - 0.1695).abs() < 1e-4, "mic {}", offsets.mic); assert!( @@ -921,9 +930,104 @@ mod clip_offset_tests { }], ..Default::default() }; - fill_clip_offsets(&mut config, dir.path(), &camera_recording()); + fill_clip_offsets(&mut config, dir.path(), &camera_recording(), &[10.0]); assert_eq!(config.clips[0].offsets.mic, 0.5); } + + fn late_mic_recording(mic_start: f64) -> StudioRecordingMeta { + let mut meta = camera_recording(); + let StudioRecordingMeta::MultipleSegments { inner } = &mut meta else { + unreachable!() + }; + inner.segments[0].mic.as_mut().unwrap().start_time = Some(mic_start); + meta + } + + fn auto_clip(mic: f32, system_audio: f32, auto: bool) -> ProjectConfiguration { + ProjectConfiguration { + clips: vec![ClipConfiguration { + index: 0, + offsets: ClipOffsets { + camera: 0.0, + mic, + system_audio, + }, + offsets_auto_calculated: auto, + }], + clip_alignment_version: 0, + ..Default::default() + } + } + + // Offsets saved while a late mic set the segment's start (mic 0, system + // audio late - 0) move to the screen's start. + #[test] + fn saved_auto_offsets_of_a_late_mic_are_recomputed() { + let dir = tempfile::tempdir().unwrap(); + let mut config = auto_clip(0.0, 5.2026, true); + fill_clip_offsets( + &mut config, + dir.path(), + &late_mic_recording(5.2026), + &[10.0], + ); + let offsets = config.clips[0].offsets; + assert!((offsets.mic - -5.0).abs() < 1e-4, "mic {}", offsets.mic); + assert!((offsets.system_audio - 0.2026).abs() < 1e-4); + assert_eq!(offsets.camera, 0.0); + assert!(config.clips[0].offsets_auto_calculated); + } + + // Offsets set by hand keep the sync the user chose against the video. + #[test] + fn user_offsets_of_a_late_mic_keep_their_sync() { + let dir = tempfile::tempdir().unwrap(); + let mut config = auto_clip(0.25, 5.2026, false); + fill_clip_offsets( + &mut config, + dir.path(), + &late_mic_recording(5.2026), + &[10.0], + ); + let offsets = config.clips[0].offsets; + assert!((offsets.mic - -4.75).abs() < 1e-4, "mic {}", offsets.mic); + assert!((offsets.system_audio - 0.2026).abs() < 1e-4); + assert!(!config.clips[0].offsets_auto_calculated); + } + + // A project saved by this version is already on the current clock. + #[test] + fn current_projects_are_not_moved_again() { + let dir = tempfile::tempdir().unwrap(); + let mut config = auto_clip(-5.0, 0.2026, true); + config.clip_alignment_version = cap_project::CLIP_ALIGNMENT_VERSION; + fill_clip_offsets( + &mut config, + dir.path(), + &late_mic_recording(5.2026), + &[10.0], + ); + assert_eq!(config.clips[0].offsets.mic.to_bits(), (-5.0f32).to_bits()); + assert_eq!( + config.clips[0].offsets.system_audio.to_bits(), + 0.2026f32.to_bits() + ); + } + + // Only segments with late audio are touched: saved automatic offsets of + // a normal recording stay bit-identical even if they differ from a fresh + // computation (for example a calibration made after they were saved). + #[test] + fn saved_auto_offsets_of_a_normal_recording_are_kept() { + let dir = tempfile::tempdir().unwrap(); + let mut config = auto_clip(0.123, 0.456, true); + fill_clip_offsets(&mut config, dir.path(), &camera_recording(), &[10.0]); + assert_eq!(config.clips[0].offsets.mic.to_bits(), 0.123f32.to_bits()); + assert_eq!( + config.clips[0].offsets.system_audio.to_bits(), + 0.456f32.to_bits() + ); + } } #[cfg(test)] diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index be8638b50e0..0453a42c5e5 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -3074,6 +3074,15 @@ pub struct ProjectConfiguration { /// grade. #[serde(default)] pub color_correction: ColorCorrectionConfiguration, + /// How clip audio offsets, markers and caption words of a segment with a + /// late audio track are timed. 0 (legacy): against the late audio start, + /// which also set where the segment's video began. 1: against the screen + /// and camera start (`MultipleSegment::latest_start_time`); legacy configs + /// are migrated by `migrate_late_audio_alignment` when the recording is + /// opened. The field-level default keeps old files at 0 while + /// `Default::default()` produces the current version. + #[serde(default)] + pub clip_alignment_version: u32, /// How text segment font sizes are interpreted. 0 (legacy): the renderer /// multiplied `font_size` by `size.y / 0.2`, coupling glyph size to the /// box. 1: `font_size` alone determines glyph size (1080p-relative); @@ -3098,6 +3107,7 @@ pub struct ProjectConfiguration { pub const TEXT_SIZE_VERSION: u32 = 1; pub const TEXT_ANIM_VERSION: u32 = 1; +pub const CLIP_ALIGNMENT_VERSION: u32 = 1; /// The keys [`ProjectConfiguration::load`] migrates on presence, read in a pass /// that skips every value instead of building a JSON tree of a file that can @@ -3144,6 +3154,7 @@ impl Default for ProjectConfiguration { screen_motion_blur: Self::default_screen_motion_blur(), screen_movement_spring: Default::default(), color_correction: Default::default(), + clip_alignment_version: CLIP_ALIGNMENT_VERSION, text_size_version: TEXT_SIZE_VERSION, text_anim_version: TEXT_ANIM_VERSION, markers: Vec::new(), @@ -3235,6 +3246,7 @@ impl ProjectConfiguration { (None, None) => None, }; preset.timeline = self.timeline.clone(); + preset.clip_alignment_version = self.clip_alignment_version; preset.text_size_version = self.text_size_version; preset.text_anim_version = self.text_anim_version; preset.overlay_order = self.overlay_order.clone(); diff --git a/crates/project/src/late_audio.rs b/crates/project/src/late_audio.rs new file mode 100644 index 00000000000..50b44aeabda --- /dev/null +++ b/crates/project/src/late_audio.rs @@ -0,0 +1,375 @@ +use crate::{CLIP_ALIGNMENT_VERSION, ProjectConfiguration, StudioRecordingMeta}; + +/// Moves a project saved before late audio stopped setting a segment's start +/// (`LATE_AUDIO_START_SECS`) onto the current segment clock. Only segments +/// whose start moved are touched. On each, everything the project timed +/// against the old, late start shifts by the same amount, so it keeps pointing +/// at the same moment of the recording: +/// +/// - mic and system audio clip offsets, automatic or set by hand (any +/// calibration they carry is kept); +/// - markers on that clip; +/// - caption words in that clip's stretch of the source clock. +/// +/// Screen and camera then play from their own start. `recording_durations` +/// (each clip's display duration, the source clock captions use) is only +/// read when such a segment has markers or captions. Returns whether anything +/// other than the version changed, i.e. whether the project should be saved. +pub fn migrate_late_audio_alignment( + project: &mut ProjectConfiguration, + meta: &StudioRecordingMeta, + recording_durations: impl FnOnce() -> Vec, +) -> bool { + if project.clip_alignment_version >= CLIP_ALIGNMENT_VERSION { + return false; + } + project.clip_alignment_version = CLIP_ALIGNMENT_VERSION; + + let StudioRecordingMeta::MultipleSegments { inner } = meta else { + return false; + }; + let shifts: Vec<(usize, f64)> = inner + .segments + .iter() + .enumerate() + .filter_map(|(index, segment)| { + let shift = segment.legacy_latest_start_time()? - segment.latest_start_time()?; + (shift.is_finite() && shift > 0.0).then_some((index, shift)) + }) + .collect(); + if shifts.is_empty() { + return false; + } + + for &(index, shift) in &shifts { + if let Some(clip) = project + .clips + .iter_mut() + .find(|clip| clip.index as usize == index) + { + clip.offsets.mic -= shift as f32; + clip.offsets.system_audio -= shift as f32; + } + } + + let has_markers = shifts.iter().any(|&(index, _)| { + project + .markers + .iter() + .any(|marker| marker.recording_clip as usize == index) + }); + let has_captions = project + .captions + .as_ref() + .is_some_and(|captions| !captions.segments.is_empty()); + if has_markers || has_captions { + let durations = recording_durations(); + shift_markers(project, &shifts, &durations); + if has_captions { + shift_captions(project, &shifts, &durations); + } + } + + true +} + +fn shift_markers(project: &mut ProjectConfiguration, shifts: &[(usize, f64)], durations: &[f64]) { + for &(index, shift) in shifts { + let end = durations + .get(index) + .copied() + .filter(|d| d.is_finite() && *d > 0.0); + for marker in project + .markers + .iter_mut() + .filter(|marker| marker.recording_clip as usize == index) + { + let time = marker.time + shift; + marker.time = end.map_or(time, |end| time.min(end)); + } + } +} + +fn shift_captions(project: &mut ProjectConfiguration, shifts: &[(usize, f64)], durations: &[f64]) { + if durations.is_empty() || project.timeline.is_none() { + return; + } + // Legacy captions are stored in edited time; put them on the source clock + // first so they can be moved with their clip. + crate::synchronize_captions(project, durations); + let Some(captions) = project.captions.as_mut().filter(|c| c.source_timed) else { + return; + }; + let mut clip_start = Vec::with_capacity(durations.len()); + let mut cumulative = 0.0; + for duration in durations { + clip_start.push(cumulative); + cumulative += duration; + } + // Each caption belongs to the clip its original start falls in, and moves + // at most once, so one clamped to a clip's end never re-enters the next. + let mut moved = false; + for segment in &mut captions.segments { + let source = f64::from(segment.start); + let Some(index) = (0..durations.len()) + .find(|&i| (clip_start[i]..clip_start[i] + durations[i]).contains(&source)) + else { + continue; + }; + let Some(&(_, shift)) = shifts.iter().find(|(clip, _)| *clip == index) else { + continue; + }; + let end = (clip_start[index] + durations[index]) as f32; + let shift = shift as f32; + let clamp = |time: f32| (time + shift).min(end); + segment.start = clamp(segment.start); + segment.end = clamp(segment.end); + for word in &mut segment.words { + word.start = clamp(word.start); + word.end = clamp(word.end); + } + moved = true; + } + if moved { + captions + .segments + .sort_by(|a, b| a.start.total_cmp(&b.start)); + crate::synchronize_captions(project, durations); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + CaptionSegment, CaptionWord, CaptionsData, ClipConfiguration, ClipOffsets, TimelineMarker, + }; + + // Two clips of 10 s. Clip 1: screen and camera start at 0.05, the mic at + // 4.05 (it came back 4 s in), system audio at 0. + fn meta(mic_start: f64) -> StudioRecordingMeta { + serde_json::from_value(serde_json::json!({ + "segments": [ + { + "display": { "path": "a/display.mp4", "fps": 30, "start_time": 0.05 }, + "mic": { "path": "a/mic.ogg", "start_time": 0.03 } + }, + { + "display": { "path": "b/display.mp4", "fps": 30, "start_time": 0.05 }, + "camera": { "path": "b/camera.mp4", "fps": 30, "start_time": 0.05 }, + "mic": { "path": "b/mic.ogg", "start_time": mic_start }, + "system_audio": { "path": "b/system.ogg", "start_time": 0.0 } + } + ] + })) + .unwrap() + } + + fn clip(index: u32, mic: f32, system_audio: f32, auto: bool) -> ClipConfiguration { + ClipConfiguration { + index, + offsets: ClipOffsets { + camera: 0.0, + mic, + system_audio, + }, + offsets_auto_calculated: auto, + } + } + + fn legacy_project() -> ProjectConfiguration { + let mut project: ProjectConfiguration = serde_json::from_value(serde_json::json!({ + "timeline": { + "segments": [ + { "recordingSegment": 0, "start": 0.0, "end": 10.0, "timescale": 1.0 }, + { "recordingSegment": 1, "start": 0.0, "end": 10.0, "timescale": 1.0 } + ], + "zoomSegments": [] + } + })) + .unwrap(); + assert_eq!(project.clip_alignment_version, 0); + // Saved while the mic set clip 1's start: mic 0 + calibration 0.02, + // system audio 4.05 + 0.02. + project.clips = vec![clip(0, 0.04, 0.0, true), clip(1, 0.02, 4.07, true)]; + project + } + + fn close(a: f32, b: f32) -> bool { + (a - b).abs() < 1e-4 + } + + #[test] + fn default_projects_are_current_and_never_migrate() { + let mut project = ProjectConfiguration::default(); + assert_eq!(project.clip_alignment_version, CLIP_ALIGNMENT_VERSION); + project.clips = vec![clip(1, -4.0, 0.05, true)]; + let before = serde_json::to_string(&project).unwrap(); + assert!(!migrate_late_audio_alignment( + &mut project, + &meta(4.05), + || unreachable!() + )); + assert_eq!(serde_json::to_string(&project).unwrap(), before); + } + + #[test] + fn auto_offsets_move_to_the_screen_start_and_keep_calibration() { + let mut project = legacy_project(); + assert!(migrate_late_audio_alignment( + &mut project, + &meta(4.05), + Vec::new + )); + assert_eq!(project.clip_alignment_version, CLIP_ALIGNMENT_VERSION); + let clip1 = project.clips[1].offsets; + assert!(close(clip1.mic, -3.98), "mic {}", clip1.mic); + assert!( + close(clip1.system_audio, 0.07), + "system {}", + clip1.system_audio + ); + assert_eq!(clip1.camera, 0.0); + assert!(project.clips[1].offsets_auto_calculated); + // The untouched clip keeps its exact bits. + assert_eq!(project.clips[0].offsets.mic.to_bits(), 0.04f32.to_bits()); + } + + #[test] + fn user_offsets_keep_their_sync_against_the_video() { + let mut project = legacy_project(); + project.clips[1] = clip(1, 0.25, 4.3, false); + migrate_late_audio_alignment(&mut project, &meta(4.05), Vec::new); + let clip1 = &project.clips[1]; + assert!(!clip1.offsets_auto_calculated); + assert!(close(clip1.offsets.mic, -3.75)); + assert!(close(clip1.offsets.system_audio, 0.3)); + } + + #[test] + fn runs_once() { + let mut project = legacy_project(); + assert!(migrate_late_audio_alignment( + &mut project, + &meta(4.05), + Vec::new + )); + let once = serde_json::to_string(&project).unwrap(); + assert!(!migrate_late_audio_alignment( + &mut project, + &meta(4.05), + || unreachable!() + )); + assert_eq!(serde_json::to_string(&project).unwrap(), once); + let reloaded: ProjectConfiguration = serde_json::from_str(&once).unwrap(); + assert_eq!(reloaded.clip_alignment_version, CLIP_ALIGNMENT_VERSION); + } + + #[test] + fn normal_recordings_only_get_the_version() { + let mut project = legacy_project(); + let clips = serde_json::to_string(&project.clips).unwrap(); + assert!(!migrate_late_audio_alignment( + &mut project, + &meta(0.06), + || unreachable!() + )); + assert_eq!(serde_json::to_string(&project.clips).unwrap(), clips); + assert_eq!(project.clip_alignment_version, CLIP_ALIGNMENT_VERSION); + } + + #[test] + fn markers_on_the_late_clip_follow_their_moment() { + let mut project = legacy_project(); + project.markers = vec![ + TimelineMarker::new(0, 2.0), + TimelineMarker::new(1, 1.5), + TimelineMarker::new(1, 9.0), + ]; + migrate_late_audio_alignment(&mut project, &meta(4.05), || vec![10.0, 10.0]); + let times: Vec<_> = project.markers.iter().map(|m| m.time).collect(); + assert_eq!(times[0], 2.0); + assert!((times[1] - 5.5).abs() < 1e-9); + assert_eq!(times[2], 10.0); + } + + #[test] + fn caption_words_on_the_late_clip_follow_the_audio() { + let mut project = legacy_project(); + let caption = |id: &str, start: f32| CaptionSegment { + id: id.into(), + start, + end: start + 1.0, + text: "hi".into(), + words: vec![CaptionWord { + text: "hi".into(), + start, + end: start + 1.0, + }], + }; + project.captions = Some(CaptionsData { + segments: vec![caption("first", 2.0), caption("second", 11.0)], + source_timed: true, + ..Default::default() + }); + migrate_late_audio_alignment(&mut project, &meta(4.05), || vec![10.0, 10.0]); + let captions = project.captions.as_ref().unwrap(); + assert_eq!(captions.segments[0].start, 2.0); + assert!(close(captions.segments[1].start, 15.0)); + assert!(close(captions.segments[1].words[0].start, 15.0)); + assert!(close(captions.segments[1].words[0].end, 16.0)); + let track = &project.timeline.as_ref().unwrap().caption_segments; + assert!(track.iter().any(|c| (c.start - 15.0).abs() < 1e-4)); + } + + // Two consecutive late clips: a caption clamped to the first clip's end + // must not be moved again by the second. + #[test] + fn captions_move_once_across_consecutive_late_clips() { + let mut project = legacy_project(); + let StudioRecordingMeta::MultipleSegments { mut inner } = meta(4.05) else { + unreachable!() + }; + inner.segments[0].mic.as_mut().unwrap().start_time = Some(4.05); + let meta = StudioRecordingMeta::MultipleSegments { inner }; + let caption = |start: f32| CaptionSegment { + id: format!("c{start}"), + start, + end: start + 1.0, + text: "hi".into(), + words: vec![CaptionWord { + text: "hi".into(), + start, + end: start + 1.0, + }], + }; + project.captions = Some(CaptionsData { + segments: vec![caption(7.0), caption(12.0)], + source_timed: true, + ..Default::default() + }); + migrate_late_audio_alignment(&mut project, &meta, || vec![10.0, 10.0]); + let starts: Vec<_> = project + .captions + .as_ref() + .unwrap() + .segments + .iter() + .map(|s| s.start) + .collect(); + assert_eq!(starts, vec![10.0, 16.0]); + } + + #[test] + fn a_mic_that_never_started_needs_no_durations_without_markers_or_captions() { + let mut project = legacy_project(); + project.clips[1] = clip(1, 0.0, 10.0, true); + assert!(migrate_late_audio_alignment( + &mut project, + &meta(10.0), + || unreachable!() + )); + assert!(close(project.clips[1].offsets.mic, -9.95)); + assert!(close(project.clips[1].offsets.system_audio, 0.05)); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index b5d153e6167..2e12791f2d5 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -7,6 +7,7 @@ mod configuration; pub mod cursor; pub mod disfluency; pub mod keyboard; +mod late_audio; mod meta; mod shapes; @@ -23,6 +24,7 @@ pub use caption_words::{ pub use configuration::*; pub use cursor::*; pub use keyboard::*; +pub use late_audio::migrate_late_audio_alignment; pub use meta::*; pub use shapes::*; diff --git a/crates/project/src/meta.rs b/crates/project/src/meta.rs index fb8399d34e9..8e645c5309e 100644 --- a/crates/project/src/meta.rs +++ b/crates/project/src/meta.rs @@ -733,6 +733,25 @@ impl MultipleSegments { } } +/// An audio track that starts more than this many seconds after the segment's +/// screen and camera is late: it no longer sets the segment's start, and its +/// clip offset turns negative so it plays from where it was recorded with +/// silence before it. A microphone that delivers nothing after a resume, or +/// only comes back part way through a segment, records its start that late +/// (at the stop time when it never delivered). Aligning to it skipped the +/// screen and camera forward and held their last frame. +/// +/// Tracks of a normal recording start within a few hundred milliseconds of +/// each other: the largest audio lag across 152 audio tracks of real Studio +/// recordings was 0.155 s (a legacy first-packet system audio start), so +/// those keep their exact alignment. It matches the recorder's own outage +/// threshold: a microphone silent for a second is treated as stalled. +pub const LATE_AUDIO_START_SECS: f64 = 1.0; + +fn is_late_audio_start(audio_start: f64, video_start: f64) -> bool { + audio_start - video_start > LATE_AUDIO_START_SECS +} + #[derive(Debug, Clone, Serialize, Deserialize, Type)] pub struct MultipleSegment { pub display: VideoMeta, @@ -819,22 +838,57 @@ impl MultipleSegment { } } + /// The segment time every track is aligned to: segment time zero is this + /// many seconds after the recording epoch. It is the latest track start, + /// except that a late audio track (see [`LATE_AUDIO_START_SECS`]) is left + /// out, so it cannot skip the screen and camera forward to its start. pub fn latest_start_time(&self) -> Option { - let mut value = self.display.start_time?; + let video_start = self.video_start_time()?; + let mut value = video_start; - if let Some(camera) = &self.camera { - value = value.max(camera.start_time?); + for audio in self.aligned_audio() { + let start = audio.start_time?; + if !is_late_audio_start(start, video_start) { + value = value.max(start); + } } - if let Some(mic) = &self.mic { - value = value.max(mic.start_time?); + Some(value) + } + + /// Whether an audio track started so long after the video that it no + /// longer sets the segment's start (see [`LATE_AUDIO_START_SECS`]). + pub fn has_late_audio(&self) -> bool { + let Some(video_start) = self.video_start_time() else { + return false; + }; + self.aligned_audio() + .filter_map(|audio| audio.start_time) + .any(|start| is_late_audio_start(start, video_start)) + } + + /// The segment start before late audio was left out: the latest start of + /// every track. Projects saved then are timed against it. + pub(crate) fn legacy_latest_start_time(&self) -> Option { + let mut value = self.video_start_time()?; + for audio in self.aligned_audio() { + value = value.max(audio.start_time?); } + Some(value) + } - if let Some(system_audio) = self.usable_system_audio() { - value = value.max(system_audio.start_time?); + fn video_start_time(&self) -> Option { + let display_start = self.display.start_time?; + match &self.camera { + Some(camera) => Some(display_start.max(camera.start_time?)), + None => Some(display_start), } + } - Some(value) + fn aligned_audio(&self) -> impl Iterator { + [self.mic.as_ref(), self.usable_system_audio()] + .into_iter() + .flatten() } pub fn calculate_audio_offsets(&self) -> crate::ClipOffsets { @@ -1457,6 +1511,78 @@ mod test { segment.display.start_time = None; assert!(segment.usable_system_audio().is_some()); } + + // A microphone that delivered nothing in a segment records the stop + // time as its start. The screen must still start at its own start + // and the microphone play from where it was recorded. + #[test] + fn mic_that_never_started_does_not_move_the_video() { + let mut segment = segment(-0.053, Some(19.947), None); + segment.camera = Some(video(Some(-0.053))); + assert!(segment.has_late_audio()); + assert_eq!(segment.latest_start_time(), Some(-0.053)); + let offsets = segment.calculate_audio_offsets(); + assert_eq!(offsets.camera, 0.0); + assert!((offsets.mic - -20.0).abs() < 1e-4); + } + + // A microphone that came back part way through a segment plays at + // the time it came back, with calibration still applied. + #[test] + fn mic_recovered_mid_segment_keeps_its_recorded_time() { + let segment = segment(0.045, Some(5.045), Some(0.0)); + assert!(segment.has_late_audio()); + assert_eq!(segment.latest_start_time(), Some(0.045)); + let offsets = segment.calculate_audio_offsets_with_calibration(Some(0.05)); + assert!((offsets.mic - -4.95).abs() < 1e-4); + assert!((offsets.system_audio - 0.095).abs() < 1e-4); + } + + // Late system audio is excluded the same way, while an on-time mic + // still sets the start. + #[test] + fn late_system_audio_is_excluded_but_on_time_mic_still_aligns() { + let segment = segment(1.0, Some(1.04), Some(8.0)); + assert!(segment.has_late_audio()); + assert_eq!(segment.latest_start_time(), Some(1.04)); + let offsets = segment.calculate_audio_offsets(); + assert_eq!(offsets.mic, 0.0); + assert!((offsets.system_audio - -6.96).abs() < 1e-4); + } + + // Normal recordings start every track within a fraction of a second, + // up to and including the threshold: their alignment is unchanged. + #[test] + fn audio_within_threshold_keeps_historical_alignment() { + for lag in [0.0, 0.03, 0.155, 0.5, crate::LATE_AUDIO_START_SECS] { + let segment = segment(2.0, Some(2.0 + lag), None); + assert!(!segment.has_late_audio(), "lag {lag}"); + assert_eq!(segment.latest_start_time(), Some(2.0 + lag)); + assert_eq!(segment.calculate_audio_offsets().mic, 0.0); + } + let late = segment(2.0, Some(2.0 + crate::LATE_AUDIO_START_SECS + 0.01), None); + assert!(late.has_late_audio()); + assert_eq!(late.latest_start_time(), Some(2.0)); + } + + // A late camera is video: it still sets the start, and audio is late + // only relative to the later of screen and camera. + #[test] + fn late_audio_is_measured_from_the_later_video_track() { + let mut segment = segment(0.0, Some(3.5), None); + segment.camera = Some(video(Some(3.0))); + assert!(!segment.has_late_audio()); + assert_eq!(segment.latest_start_time(), Some(3.5)); + + segment.camera = Some(video(None)); + assert!(!segment.has_late_audio()); + assert_eq!(segment.latest_start_time(), None); + + segment.camera = None; + segment.mic.as_mut().unwrap().start_time = None; + assert!(!segment.has_late_audio()); + assert_eq!(segment.latest_start_time(), None); + } } } diff --git a/crates/render-farm/src/project.rs b/crates/render-farm/src/project.rs index a4400973f05..28888cab67f 100644 --- a/crates/render-farm/src/project.rs +++ b/crates/render-farm/src/project.rs @@ -33,6 +33,13 @@ impl LoadedProject { .map_err(|error| anyhow!("recordings: {error}"))?, ); cap_export::synthesize_default_timeline(&mut config, &recordings); + cap_project::migrate_late_audio_alignment(&mut config, &studio_meta, || { + recordings + .segments + .iter() + .map(|segment| segment.display.duration) + .collect() + }); cap_project::synchronize_legacy_keyboard(&recording_meta, &mut config); cap_project::synchronize_captions( &mut config, diff --git a/crates/rendering/src/segment_timing.rs b/crates/rendering/src/segment_timing.rs index ff6f454ced8..e81ec99ba8d 100644 --- a/crates/rendering/src/segment_timing.rs +++ b/crates/rendering/src/segment_timing.rs @@ -146,4 +146,22 @@ mod tests { assert_eq!(timing.screen_offset, 0.5); assert_eq!(timing.camera_offset, 0.25); } + + // A microphone that delivered nothing records the stop time as its start. + // The screen and camera must play from their own start, not jump to the + // end of the segment and hold their last frame. + #[test] + fn late_mic_does_not_skip_screen_or_camera() { + let metadata: StudioRecordingMeta = serde_json::from_value(serde_json::json!({ + "segments": [{ + "display": { "path": "display", "fps": 30, "start_time": -0.05 }, + "camera": { "path": "camera", "fps": 30, "start_time": -0.05 }, + "mic": { "path": "mic.ogg", "start_time": 19.95 } + }] + })) + .unwrap(); + let timing = segment_video_timing(&metadata, 0); + assert_eq!(timing.latest_start_time, Some(-0.05)); + assert_eq!((timing.screen_offset, timing.camera_offset), (0.0, 0.0)); + } } diff --git a/crates/verify/src/model.rs b/crates/verify/src/model.rs index 5ac8eae74f7..936ec45e0b1 100644 --- a/crates/verify/src/model.rs +++ b/crates/verify/src/model.rs @@ -12,7 +12,9 @@ //! its last `duration` seconds) and the incoming one's; fades through black //! are not checked. //! - Every track of a recording segment is aligned to the latest track start -//! (`latest_start_time - track.start_time` is added to the request). The +//! (`latest_start_time - track.start_time` is added to the request), except +//! an audio track starting more than `LATE_AUDIO_START_SECS` after the later +//! of the screen and camera starts, which does not count. The //! camera additionally takes the clip's camera offset when the user set it; //! an auto-calculated camera offset describes the same start alignment and //! must not be applied a second time. @@ -157,11 +159,17 @@ impl ReferenceModel { .zip(present) .all(|(start, present)| !present || start.is_some()); let latest = all_known.then(|| { - starts + let video = starts[..2] .iter() .flatten() .copied() - .fold(f64::NEG_INFINITY, f64::max) + .fold(f64::NEG_INFINITY, f64::max); + starts[2..] + .iter() + .flatten() + .copied() + .filter(|audio| audio - video <= cap_project::LATE_AUDIO_START_SECS) + .fold(video, f64::max) }); let align = |start: Option| match (latest, start) { (Some(latest), Some(start)) => latest - start,