Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion crates/enc-avfoundation/src/mp4.rs
Original file line number Diff line number Diff line change
Expand Up @@ -983,9 +983,14 @@ impl MP4Encoder {
&mut self,
timestamp: Option<Duration>,
) -> Result<arc::R<av::AssetWriter>, FinishError> {
if !self.session_started || self.is_finishing {
if self.is_finishing {
return Err(FinishError::NotWriting);
}
// No frame ever reached the writer, so it never started and there is
// nothing to finalize: an empty output, not a broken one.
if !self.session_started {
return Err(FinishError::NoFrames);
}

let mut finish_timestamp = timestamp;

Expand Down Expand Up @@ -4413,6 +4418,55 @@ mod tests {
}
}
}

/// A resumed Studio segment whose screen capture delivered no frame
/// before Pause. The writer never started, so finish reports an empty
/// output (`NoFrames`) rather than `NotWriting`, the error that failed
/// the whole recording. Paused or not, with or without an audio input.
#[test]
fn finish_without_any_frame_reports_no_frames() {
for (paused, audio) in [(false, false), (true, false), (false, true), (true, true)] {
let output = test_output_path("finish_without_any_frame");
let audio_config = audio.then(wireless_audio_config);
let mut encoder =
MP4Encoder::init(output.clone(), valid_video_config(), audio_config, None).unwrap();
if paused {
encoder.pause();
}
let result = encoder.finish(Some(Duration::from_secs(1)));
drop(encoder);
let _ = std::fs::remove_file(&output);
assert!(
matches!(result, Err(FinishError::NoFrames)),
"paused {paused}, audio {audio}: {result:?}"
);
}
}

#[test]
fn finish_after_a_single_frame_writes_a_playable_file() {
let output = test_output_path("finish_after_a_single_frame");
let config = VideoInfo::from_raw_ffmpeg(ffmpeg::format::Pixel::NV12, 1280, 720, 30);
let mut encoder = MP4Encoder::init(output.clone(), config, None, None).unwrap();
let pool = create_pixel_buffer_pool(1280, 720);
let frame = create_test_video_frame(&pool, 0, 33_333);
assert!(queue_video_frame_with_retry(&mut encoder, frame, Duration::ZERO).unwrap());
encoder.finish(Some(Duration::from_micros(33_333))).unwrap();
drop(encoder);

let mut input = ffmpeg::format::input(&output).unwrap();
let index = input
.streams()
.best(ffmpeg::media::Type::Video)
.unwrap()
.index();
let packets = input
.packets()
.filter(|(stream, _)| stream.index() == index)
.count();
let _ = std::fs::remove_file(&output);
assert_eq!(packets, 1);
}
}

trait SampleBufExt {
Expand Down
106 changes: 104 additions & 2 deletions crates/enc-ffmpeg/src/remux.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ struct StreamRemuxer {
stream_mapping: Vec<Option<usize>>,
last_dts: Vec<i64>,
dts_offset: Vec<i64>,
video_frame_rate: Vec<Option<ffmpeg::Rational>>,
}

impl StreamRemuxer {
Expand All @@ -272,6 +273,7 @@ impl StreamRemuxer {
octx: &mut avformat::context::Output,
) -> Result<Self, RemuxError> {
let mut stream_mapping = Vec::new();
let mut video_frame_rate = Vec::new();
let mut output_stream_index = 0usize;

for input_stream in ictx.streams() {
Expand All @@ -281,6 +283,11 @@ impl StreamRemuxer {
if medium == ffmpeg::media::Type::Video || medium == ffmpeg::media::Type::Audio {
stream_mapping.push(Some(output_stream_index));
output_stream_index += 1;
video_frame_rate.push(
(medium == ffmpeg::media::Type::Video)
.then(|| input_stream.avg_frame_rate())
.filter(|rate| rate.numerator() > 0 && rate.denominator() > 0),
);

let mut output_stream = octx.add_stream(None)?;
output_stream.set_parameters(codec_params);
Expand All @@ -298,6 +305,7 @@ impl StreamRemuxer {
stream_mapping,
last_dts: vec![i64::MIN; output_stream_index],
dts_offset: vec![0; output_stream_index],
video_frame_rate,
})
}

Expand All @@ -313,6 +321,20 @@ impl StreamRemuxer {

packet.rescale_ts(input_time_base, output_time_base);

// Fragmented recordings carry no sample durations. The muxer
// derives every duration from the next sample, except the last
// one; a lone frame would end up with a zero-length track whose
// only sample readers discard. Give a first video packet without
// a duration one frame.
if self.last_dts[output_index] == i64::MIN
&& packet.duration() <= 0
&& let Some(rate) = self.video_frame_rate[output_index]
{
packet.set_duration(
Comment thread
richiemcilroy marked this conversation as resolved.
Resolved
ffmpeg::Rescale::rescale(&1i64, rate.invert(), output_time_base).max(1),
);
}

let current_dts = packet.dts().unwrap_or(0);

if self.last_dts[output_index] != i64::MIN && current_dts <= self.last_dts[output_index]
Expand Down Expand Up @@ -1486,6 +1508,12 @@ mod tests {
};

fn encode_test_segments(directory: &Path) -> (PathBuf, Vec<PathBuf>) {
let (init, segments) = encode_test_frames(directory, 320);
assert!(segments.len() >= 2);
(init, segments)
}

fn encode_test_frames(directory: &Path, frames: u64) -> (PathBuf, Vec<PathBuf>) {
use crate::segmented_stream::{SegmentedVideoEncoder, SegmentedVideoEncoderConfig};

ffmpeg::init().unwrap();
Expand All @@ -1501,7 +1529,7 @@ mod tests {
SegmentedVideoEncoderConfig::default(),
)
.unwrap();
for index in 0..320 {
for index in 0..frames {
let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, 320, 240);
frame.data_mut(0).fill(32 + (index % 192) as u8);
frame.data_mut(1).fill(128);
Expand All @@ -1515,7 +1543,6 @@ mod tests {
.iter()
.map(|segment| segment.path.clone())
.collect();
assert!(segments.len() >= 2);
(encoder.init_segment_path(), segments)
}

Expand Down Expand Up @@ -1635,6 +1662,81 @@ mod tests {
let _ = plain;
}

/// A Studio segment paused right after it resumed can hold one frame.
/// Its fragment has no sample durations, so the progressive file used to
/// get a zero-length track whose only frame readers discard, and
/// finalizing failed the whole recording.
#[test]
fn remux_of_a_single_durationless_frame_stays_playable() {
let directory = tempfile::tempdir().unwrap();
let (init, segments) = encode_test_frames(&directory.path().join("source"), 1);
let single = directory.path().join("single.mp4");
concatenate_m4s_segments_with_init(&init, &segments, &single).unwrap();
// The macOS Studio fragments carry no sample durations.
let output = directory.path().join("output.mp4");
{
let mut ictx = ffmpeg::format::input(&single).unwrap();
let mut octx = ffmpeg::format::output(&output).unwrap();
let mut remuxer = super::StreamRemuxer::new(&ictx, &mut octx).unwrap();
for (stream, mut packet) in ictx.packets() {
packet.set_duration(0);
remuxer
.write_packet(packet, stream.time_base(), &mut octx)
.unwrap();
}
octx.write_trailer().unwrap();
}
assert_eq!(decoded_video_frame_count(&output), 1);
assert!(super::probe_video_can_decode(&output).unwrap());
super::probe_video_seek_points(&output, 8).unwrap();
let (time_base, packets) = video_packet_contents(&output);
assert_eq!(packets.len(), 1);
assert_eq!(
packets[0].duration,
i64::from(time_base.denominator()) / (30 * i64::from(time_base.numerator()))
);
}

/// The fallback only lengthens a lone frame: with more frames, a long
/// gap after the first (a static screen right after Resume) keeps every
/// timestamp where it was.
#[test]
fn remux_of_durationless_frames_keeps_their_timestamps() {
let directory = tempfile::tempdir().unwrap();
let (init, segments) = encode_test_frames(&directory.path().join("source"), 3);
let source = directory.path().join("source.mp4");
concatenate_m4s_segments_with_init(&init, &segments, &source).unwrap();
let (source_time_base, source_packets) = video_packet_contents(&source);
let frame = i64::from(source_time_base.denominator())
/ (30 * i64::from(source_time_base.numerator()));
let stamps = [0, 24 * frame, 25 * frame];
assert!(source_packets.iter().all(|packet| packet.pts == packet.dts));

let output = directory.path().join("output.mp4");
{
let mut ictx = ffmpeg::format::input(&source).unwrap();
let mut octx = ffmpeg::format::output(&output).unwrap();
let mut remuxer = super::StreamRemuxer::new(&ictx, &mut octx).unwrap();
for (index, (stream, mut packet)) in ictx.packets().enumerate() {
packet.set_pts(Some(stamps[index]));
packet.set_dts(Some(stamps[index]));
packet.set_duration(0);
remuxer
.write_packet(packet, stream.time_base(), &mut octx)
.unwrap();
}
octx.write_trailer().unwrap();
}
let (time_base, packets) = video_packet_contents(&output);
assert_eq!(time_base, source_time_base);
assert_eq!(
packets.iter().map(|packet| packet.pts).collect::<Vec<_>>(),
stamps.map(Some)
);
assert_eq!(packets[0].duration, 24 * frame);
assert_eq!(decoded_video_frame_count(&output), 3);
}

#[test]
fn validated_m4s_concat_preserves_packets_timestamps_and_all_frames() {
let directory = tempfile::tempdir().unwrap();
Expand Down
71 changes: 71 additions & 0 deletions crates/recording/src/markers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,35 @@ impl RecordingMarkers {
}
}

/// The segment was left out of the recording (its screen captured no
/// frame) and the next segment takes its index. Markers pressed in it
/// move to the end of the segment before, or go when it was the first.
pub(crate) fn segment_dropped(&self, segment: u32) {
let revision = {
let mut state = self.lock();
state.segments.retain(|clock| clock.segment != segment);
if !state.markers.iter().any(|marker| marker.segment == segment) {
return;
}
let before = segment.checked_sub(1);
state.markers = std::mem::take(&mut state.markers)
.into_iter()
.filter_map(|marker| {
if marker.segment != segment {
return Some(marker);
}
before.map(|segment| RecordedMarker {
segment,
offset: None,
})
})
.collect();
state.revision += 1;
state.revision
};
self.persist(revision);
}

pub(crate) fn stopped(&self) {
self.lock().stopped_at.get_or_insert_with(Instant::now);
}
Expand Down Expand Up @@ -408,6 +437,48 @@ mod tests {
assert_eq!(times[3], (1, 5.0));
}

#[test]
fn a_dropped_segment_hands_its_markers_to_the_segment_before() {
let markers = RecordingMarkers::new(None);
let start = Timestamps::now();
markers.segment_started(0, start, None);
markers.add(start.instant() + Duration::from_millis(500));
markers.segment_paused(0);
let resumed = Timestamps::now();
markers.segment_started(1, resumed, None);
markers.add(resumed.instant() + Duration::from_millis(1));
markers.segment_paused(1);

markers.segment_dropped(1);
let kept = markers.snapshot();
assert_eq!(kept.len(), 2);
assert_eq!(kept[0].segment, 0);
assert!(kept[0].offset.is_some());
assert_eq!(
kept[1],
RecordedMarker {
segment: 0,
offset: None
}
);

let reused = Timestamps::now();
markers.segment_started(1, reused, None);
let placed = markers
.add(reused.instant() + Duration::from_millis(1))
.unwrap();
assert_eq!(placed, 3);
assert_eq!(markers.snapshot()[2].segment, 1);
assert!(markers.snapshot()[2].offset.is_some());

let first = RecordingMarkers::new(None);
let start = Timestamps::now();
first.segment_started(0, start, None);
first.add(start.instant() + Duration::from_millis(1));
first.segment_dropped(0);
assert!(first.snapshot().is_empty());
}

#[test]
fn recovery_renumbers_segments_it_kept() {
let recorded = [
Expand Down
7 changes: 7 additions & 0 deletions crates/recording/src/output_pipeline/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5152,6 +5152,13 @@ impl PipelineStopSignal {
#[error("Screen capture stopped from macOS sharing controls")]
pub struct PipelineStoppedByUser;

/// The muxer never received a video frame, so the track has no media to
/// finalize: an empty output rather than a broken one. Studio leaves such a
/// segment out and keeps the rest of the recording.
#[derive(Debug, thiserror::Error)]
#[error("No video frames were captured")]
pub struct NoVideoFramesCaptured;

#[derive(Clone, Debug)]
pub struct PipelineDoneError(Arc<anyhow::Error>);

Expand Down
Loading
Loading