diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs index 6b9237ea..acc2045d 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs @@ -24,6 +24,9 @@ pub mod format_keys { pub const KEY_LEVEL: &str = "level"; pub const KEY_PRIORITY: &str = "priority"; pub const KEY_OPERATING_RATE: &str = "operating-rate"; + pub const KEY_COLOR_STANDARD: &str = "color-standard"; + pub const KEY_COLOR_TRANSFER: &str = "color-transfer"; + pub const KEY_COLOR_RANGE: &str = "color-range"; // Audio keys pub const KEY_SAMPLE_RATE: &str = "sample-rate"; @@ -33,6 +36,13 @@ pub mod format_keys { pub const COLOR_FORMAT_SURFACE: jint = 0x7F000789; pub const COLOR_FORMAT_YUV420_FLEXIBLE: jint = 0x7F420888; + +/// `MediaFormat.COLOR_STANDARD_BT709`, `COLOR_TRANSFER_SDR_VIDEO` and `COLOR_RANGE_LIMITED`. +/// All three were introduced in API 24, below the minimum this crate targets, so setting them +/// needs no version guard. +pub const COLOR_STANDARD_BT709: jint = 1; +pub const COLOR_TRANSFER_SDR_VIDEO: jint = 3; +pub const COLOR_RANGE_LIMITED: jint = 2; pub const AAC_OBJECT_TYPE_AAC_LC: jint = 2; pub const MUXER_OUTPUT_FORMAT_MPEG_4: jint = 0; diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs index a969929a..b233bc88 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs @@ -5,7 +5,10 @@ use unienc_common::{CompletionHandle, Muxer, MuxerInput}; use crate::bindings; use crate::common::*; -use crate::config::MUXER_OUTPUT_FORMAT_MPEG_4; +use crate::config::{ + COLOR_RANGE_LIMITED, COLOR_STANDARD_BT709, COLOR_TRANSFER_SDR_VIDEO, + MUXER_OUTPUT_FORMAT_MPEG_4, format_keys, +}; use crate::error::{AndroidError, Result}; use crate::java::*; @@ -128,6 +131,23 @@ async fn push( "height".to_string(), crate::common::MediaFormatValue::Integer(height as i32), ); + + // The track format is what MediaMuxer writes the `colr` box from. These keys are + // set on the encoder and most codecs echo them back in their output format, but + // that is not guaranteed across vendors, so state them here as well. The values + // must match the ones `create_video_format` configures the encoder with. + map.insert( + format_keys::KEY_COLOR_STANDARD.to_string(), + crate::common::MediaFormatValue::Integer(COLOR_STANDARD_BT709), + ); + map.insert( + format_keys::KEY_COLOR_TRANSFER.to_string(), + crate::common::MediaFormatValue::Integer(COLOR_TRANSFER_SDR_VIDEO), + ); + map.insert( + format_keys::KEY_COLOR_RANGE.to_string(), + crate::common::MediaFormatValue::Integer(COLOR_RANGE_LIMITED), + ); } let mut shared_state_lock = shared_state.write().await; diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs index f1395a75..b598ee2b 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs @@ -447,5 +447,19 @@ fn create_video_format_raw( set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?; set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?; + // Tag the stream as BT.709 limited range. Without these the encoder leaves the color + // information unspecified, so the muxed file gives players nothing to go on. The values match + // the conversion `VideoFrameBgra32::to_yuv420_planes` performs on the buffer input path, and + // on the surface input path they additionally tell MediaCodec which coefficients to use when + // it converts the frames the Vulkan preprocessor hands it. + set_format_integer(env, &format_obj, KEY_COLOR_STANDARD, COLOR_STANDARD_BT709)?; + set_format_integer( + env, + &format_obj, + KEY_COLOR_TRANSFER, + COLOR_TRANSFER_SDR_VIDEO, + )?; + set_format_integer(env, &format_obj, KEY_COLOR_RANGE, COLOR_RANGE_LIMITED)?; + SafeGlobalRef::new(env, format_obj) } diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs index e0a94354..7a0d7c90 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs @@ -16,10 +16,14 @@ use objc2_av_foundation::{ use objc2_core_audio_types::{ AudioStreamBasicDescription, AudioStreamPacketDescription, MPEG4ObjectID, kAudioFormatMPEG4AAC, }; +use objc2_core_foundation::{CFDictionary, CFString, CFType}; use objc2_core_media::{ CMAudioFormatDescriptionCreate, CMAudioSampleBufferCreateReadyWithPacketDescriptions, CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMTime, CMVideoFormatDescriptionCreate, - kCMBlockBufferAssureMemoryNowFlag, kCMTimeZero, kCMVideoCodecType_H264, + kCMBlockBufferAssureMemoryNowFlag, kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionExtension_ColorPrimaries, kCMFormatDescriptionExtension_TransferFunction, + kCMFormatDescriptionExtension_YCbCrMatrix, kCMFormatDescriptionTransferFunction_ITU_R_709_2, + kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, kCMTimeZero, kCMVideoCodecType_H264, }; use objc2_foundation::{NSString, NSURL}; use tokio::sync::{mpsc, oneshot}; @@ -165,6 +169,29 @@ impl AVFMuxer { objc2_av_foundation::AVAssetWriter::assetWriterWithURL_fileType_error(&url, file_type)? }; + // AVAssetWriter writes the `colr` box from the format description it is given, not from + // the one the encoder produced: the encoded frames travel back through the caller as + // opaque buffers, so this muxer never sees the encoder's format description. Without + // these extensions the output carries no color information at all, which makes players + // guess at the color space. The values match the tags the encoder writes into the SPS. + let color_extensions = { + let keys: [&CFString; 3] = unsafe { + [ + kCMFormatDescriptionExtension_ColorPrimaries, + kCMFormatDescriptionExtension_TransferFunction, + kCMFormatDescriptionExtension_YCbCrMatrix, + ] + }; + let values: [&CFType; 3] = unsafe { + [ + kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionTransferFunction_ITU_R_709_2, + kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, + ] + }; + CFDictionary::from_slices(&keys, &values) + }; + let source_format_hint = unsafe { let mut format_desc: *const CMFormatDescription = std::ptr::null(); CMVideoFormatDescriptionCreate( @@ -172,7 +199,7 @@ impl AVFMuxer { kCMVideoCodecType_H264, video_options.width() as i32, video_options.height() as i32, - None, + Some(color_extensions.as_opaque()), NonNull::new(&mut format_desc).unwrap(), ) .to_result()?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs index d214e406..d83fdb59 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs @@ -8,13 +8,17 @@ use objc2_core_foundation::{ CFBoolean, CFDictionary, CFNumber, CFString, CFType, kCFBooleanFalse, kCFBooleanTrue, }; use objc2_core_media::{ - CMSampleBuffer, CMTime, kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, + CMSampleBuffer, CMTime, kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionTransferFunction_ITU_R_709_2, kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, + kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, }; use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelFormatType_32BGRA}; use objc2_video_toolbox::{ VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty, kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate, + kVTCompressionPropertyKey_ColorPrimaries, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, kVTCompressionPropertyKey_RealTime, + kVTCompressionPropertyKey_TransferFunction, kVTCompressionPropertyKey_YCbCrMatrix, kVTInvalidSessionErr, }; use tokio::sync::mpsc; @@ -398,6 +402,36 @@ impl CompressionSession { } .to_result()?; + // Tag the stream as BT.709 limited range. Without these the encoder leaves the color + // information unspecified, so the muxed file carries no `colr` box and players have to + // guess. The values must match the conversion `VideoFrameBgra32::to_yuv420_planes` + // performs on the readback path, and the matrix additionally tells VideoToolbox which + // coefficients to use when it converts the BGRA pixel buffers of the blit path itself. + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_ColorPrimaries, + Some(kCMFormatDescriptionColorPrimaries_ITU_R_709_2 as &CFType), + ) + } + .to_result()?; + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_TransferFunction, + Some(kCMFormatDescriptionTransferFunction_ITU_R_709_2 as &CFType), + ) + } + .to_result()?; + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_YCbCrMatrix, + Some(kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2 as &CFType), + ) + } + .to_result()?; + Ok(CompressionSession { inner: session }) } } diff --git a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs index ef056e7f..e51c7bb8 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs @@ -139,20 +139,66 @@ impl VideoFrameBgra32 { &self, padded_size: Option<(u32, u32)>, ) -> Result<(Vec, Vec, Vec)> { - let data = self.buffer.data(); - let w = padded_size.map_or(self.width, |(w, _)| w); - let h = padded_size.map_or(self.height, |(_, h)| h); - let w_half = (w + 1) >> 1; - let h_half = (h + 1) >> 1; - let padded_y_size = (w * h) as usize; - let padded_uv_size = (w_half * h_half) as usize; + let layout = Yuv420Layout::new(self, padded_size); // Create padded YUV data arrays - let mut y_data = vec![16u8; padded_y_size]; // Black level for Y - let mut u_data = vec![128u8; padded_uv_size]; // Neutral for U - let mut v_data = vec![128u8; padded_uv_size]; // Neutral for V + let mut y_data = vec![16u8; layout.y_size()]; // Black level for Y + let mut u_data = vec![128u8; layout.uv_size()]; // Neutral for U + let mut v_data = vec![128u8; layout.uv_size()]; // Neutral for V + + self.write_yuv420(&layout, &mut y_data, &mut u_data, &mut v_data); + + Ok((y_data, u_data, v_data)) + } + + /// Converts to I420 in a single buffer: the Y plane, then U, then V, each tightly packed. + /// + /// The planes are laid out for `padded_size` as in [`Self::to_yuv420_planes`], so the luma + /// stride is the padded width and the chroma stride is half of it, rounded up. + pub fn to_i420(&self, padded_size: Option<(u32, u32)>) -> Result> { + let layout = Yuv420Layout::new(self, padded_size); + + let mut data = vec![128u8; layout.y_size() + 2 * layout.uv_size()]; + let (y_data, chroma) = data.split_at_mut(layout.y_size()); + let (u_data, v_data) = chroma.split_at_mut(layout.uv_size()); + y_data.fill(16); + + self.write_yuv420(&layout, y_data, u_data, v_data); - // Convert ARGB to YUV for the original image area only + Ok(data) + } + + /// Writes the converted image into planes laid out by `layout`. Samples outside the image + /// are left untouched, so the caller decides what the padding holds. + fn write_yuv420( + &self, + layout: &Yuv420Layout, + y_data: &mut [u8], + u_data: &mut [u8], + v_data: &mut [u8], + ) { + let data = self.buffer.data(); + let w = layout.width; + let w_half = layout.chroma_width(); + + // Convert ARGB to YUV for the original image area only. + // + // BT.709 limited range ("studio swing"), 8-bit fixed point with a denominator of 256: + // + // Y = 16 + (219/255) * ( 0.2126 R + 0.7152 G + 0.0722 B ) + // Cb = 128 + (224/255) * ( B - Y ) / 1.8556 + // Cr = 128 + (224/255) * ( R - Y ) / 1.5748 + // + // Scaling those factors by 256 gives (46.74, 157.24, 15.87) for Y, (-25.76, -86.67, + // 112.43) for Cb and (112.43, -102.13, -10.30) for Cr. The Cb row is rounded to + // (-26, -86, 112) instead of to the nearest integers so that every row sums to the value + // the reference formula requires: 220 for Y, which maps white to 235 and black to 16, and + // 0 for Cb and Cr, which maps neutral colors to exactly 128. That also keeps both chroma + // rows inside the nominal 16..240 range at the primary extremes, as the BT.601 + // coefficients this replaces did. The results therefore always fit in u8 without clamping. + // + // The color tags written by each platform encoder must stay in sync with these + // coefficients. for y in 0..self.height { for x in 0..self.width { let bgra_idx = ((y * self.width + x) * 4) as usize; @@ -160,24 +206,53 @@ impl VideoFrameBgra32 { let g = data[bgra_idx + 1] as i32; let b = data[bgra_idx] as i32; - let y_val = (((66 * r + 129 * g + 25 * b + 128) >> 8) + 16) as u8; + let y_val = (((47 * r + 157 * g + 16 * b + 128) >> 8) + 16) as u8; let y_idx = (y * w + x) as usize; y_data[y_idx] = y_val; // Sample U and V for every 2x2 block (4:2:0 subsampling) if x % 2 == 0 && y % 2 == 0 { - let u_val = (((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128) as u8; - let v_val = (((112 * r - 94 * g - 18 * b + 128) >> 8) + 128) as u8; + let u_val = (((-26 * r - 86 * g + 112 * b + 128) >> 8) + 128) as u8; + let v_val = (((112 * r - 102 * g - 10 * b + 128) >> 8) + 128) as u8; - let uv_idx = ((y / 2) * (w / 2) + (x / 2)) as usize; + let uv_idx = ((y / 2) * w_half + (x / 2)) as usize; u_data[uv_idx] = u_val; v_data[uv_idx] = v_val; } } } + } +} - Ok((y_data, u_data, v_data)) +/// Plane dimensions of a 4:2:0 image, padded or not. +struct Yuv420Layout { + width: u32, + height: u32, +} + +impl Yuv420Layout { + fn new(frame: &VideoFrameBgra32, padded_size: Option<(u32, u32)>) -> Self { + let (width, height) = padded_size.unwrap_or((frame.width, frame.height)); + Self { width, height } + } + + /// Chroma planes cover odd dimensions by rounding up, so the last column and row of an + /// odd-sized image still have a chroma sample. + fn chroma_width(&self) -> u32 { + self.width.div_ceil(2) + } + + fn chroma_height(&self) -> u32 { + self.height.div_ceil(2) + } + + fn y_size(&self) -> usize { + (self.width * self.height) as usize + } + + fn uv_size(&self) -> usize { + (self.chroma_width() * self.chroma_height()) as usize } } @@ -280,4 +355,130 @@ mod tests { vec![0x34, 0x12, 0xfe, 0xff, 0x00, 0x80] ); } + + /// A 2x2 frame of a single color, which converts to exactly one sample on every plane. + fn solid_bgra_frame(r: u8, g: u8, b: u8) -> VideoFrameBgra32 { + VideoFrameBgra32 { + buffer: SharedBuffer::new_unmanaged([b, g, r, 255].repeat(4)), + width: 2, + height: 2, + } + } + + fn convert_solid(r: u8, g: u8, b: u8) -> (u8, u8, u8) { + let (y, u, v) = solid_bgra_frame(r, g, b).to_yuv420_planes(None).unwrap(); + (y[0], u[0], v[0]) + } + + /// BT.709 limited range, evaluated in floating point straight from the definition. + fn bt709_limited_reference(r: u8, g: u8, b: u8) -> (f64, f64, f64) { + let (r, g, b) = (r as f64 / 255.0, g as f64 / 255.0, b as f64 / 255.0); + let luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; + ( + 16.0 + 219.0 * luma, + 128.0 + 224.0 * (b - luma) / 1.8556, + 128.0 + 224.0 * (r - luma) / 1.5748, + ) + } + + #[test] + fn to_yuv420_planes_maps_white_and_black_to_the_limited_range_ends() { + assert_eq!(convert_solid(255, 255, 255), (235, 128, 128)); + assert_eq!(convert_solid(0, 0, 0), (16, 128, 128)); + } + + #[test] + fn to_yuv420_planes_keeps_grays_neutral() { + for level in 0..=255u8 { + let (_, u, v) = convert_solid(level, level, level); + assert_eq!((u, v), (128, 128), "gray level {level}"); + } + } + + #[test] + fn to_yuv420_planes_uses_bt709_rather_than_bt601() { + // The primaries are where the two matrices differ most. Pure red is Y=63 under BT.709 and + // Y=81 under BT.601, so this fails clearly if the BT.601 coefficients come back. + assert_eq!(convert_solid(255, 0, 0), (63, 102, 240)); + assert_eq!(convert_solid(0, 255, 0), (172, 42, 26)); + assert_eq!(convert_solid(0, 0, 255), (32, 240, 118)); + } + + #[test] + fn to_yuv420_planes_matches_the_bt709_definition_within_rounding() { + // The fixed-point coefficients are rounded to 1/256, and the Cb row is deliberately rounded + // away from the nearest integers to keep neutral colors at exactly 128 (see + // `to_yuv420_planes`). That costs up to about 1.15 steps on Cb for saturated greens. The + // BT.601 coefficients this replaced are off by up to 28 steps on Y, so the tolerance + // still tells the two matrices apart by a wide margin. + for r in (0..=255u8).step_by(5) { + for g in (0..=255u8).step_by(5) { + for b in (0..=255u8).step_by(5) { + let actual = convert_solid(r, g, b); + let expected = bt709_limited_reference(r, g, b); + for (component, actual, expected) in [ + ("Y", actual.0, expected.0), + ("Cb", actual.1, expected.1), + ("Cr", actual.2, expected.2), + ] { + assert!( + (actual as f64 - expected).abs() <= 1.5, + "{component} of rgb({r}, {g}, {b}) is {actual}, expected {expected:.2}" + ); + } + } + } + } + } + + #[test] + fn to_i420_packs_the_same_planes_as_to_yuv420_planes() { + let frame = solid_bgra_frame(255, 0, 0); + for padded_size in [None, Some((4, 4))] { + let (y, u, v) = frame.to_yuv420_planes(padded_size).unwrap(); + assert_eq!( + frame.to_i420(padded_size).unwrap(), + [y, u, v].concat(), + "padded to {padded_size:?}" + ); + } + } + + #[test] + fn to_yuv420_planes_gives_the_last_column_of_an_odd_width_its_own_chroma() { + // Three pixels wide, each row red, red, blue, and two chroma rows tall. Each chroma row + // is two samples wide, so the blue column's sample sits at the end of its own row. With + // a stride of one, the second chroma row would start on top of it and overwrite it. + let mut pixels = Vec::new(); + for _ in 0..4 { + pixels.extend_from_slice(&[0, 0, 255, 255, 0, 0, 255, 255, 255, 0, 0, 255]); + } + let frame = VideoFrameBgra32 { + buffer: SharedBuffer::new_unmanaged(pixels), + width: 3, + height: 4, + }; + + let (y, u, v) = frame.to_yuv420_planes(None).unwrap(); + + assert_eq!(y, [63, 63, 32].repeat(4)); + assert_eq!(u, [102, 240].repeat(2)); + assert_eq!(v, [240, 118].repeat(2)); + } + + #[test] + fn to_yuv420_planes_fills_padding_with_black() { + let frame = solid_bgra_frame(255, 255, 255); + let (y, u, v) = frame.to_yuv420_planes(Some((4, 4))).unwrap(); + + assert_eq!(y.len(), 16); + assert_eq!((u.len(), v.len()), (4, 4)); + for row in 0..4 { + for col in 0..4 { + let expected = if row < 2 && col < 2 { 235 } else { 16 }; + assert_eq!(y[row * 4 + col], expected, "luma at ({col}, {row})"); + } + } + assert!(u.iter().chain(&v).all(|&c| c == 128)); + } } diff --git a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs index 33e17782..ce17f426 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs @@ -169,6 +169,21 @@ impl FFmpegVideoEncoder { &format!("{}", options.bitrate()), "-force_key_frames", &format!("expr:gte(t,n_forced*{idr_interval_seconds})"), + // Convert to and tag BT.709 limited range. Tagging the frames makes the + // auto-inserted scale filter use the BT.709 matrix for the BGRA to YUV + // conversion (FFmpeg otherwise derives the matrix from the frame size, which + // yields BT.601 at the sizes this library records) and makes the encoder write + // the color information into the SPS VUI. The muxer stream-copies this + // elementary stream, so the VUI is what carries the tags into the MP4. + // + // The `-color_primaries` / `-color_trc` output options would be the more + // obvious spelling, but FFmpeg does not reliably forward them to the encoder: + // as of 8.0.1 only the matrix and the range reach the VUI that way, leaving + // the primaries and the transfer function unspecified. Setting them on the + // frames is honored by every encoder because it does not depend on the + // encoder wrapper reading them off the codec context. + "-vf", + "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv", ], ffmpeg::Destination::Stdout, )?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs new file mode 100644 index 00000000..c7e7511d --- /dev/null +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs @@ -0,0 +1,300 @@ +//! Just enough H.264 bitstream parsing to read the color description out of a +//! sequence parameter set. +//! +//! An MP4 can state a track's color space twice: in the container's `colr` box +//! and in the VUI of the SPS that the decoder itself reads. Muxers that write no +//! `colr` box leave the VUI as the only statement, and a player that trusts one +//! over the other renders a mismatch wrongly, so both are worth checking. + +use crate::mp4::ColorDescription; + +/// The `profile_idc` values whose SPS carries the chroma format, bit depths +/// and scaling matrices before the fields every profile shares. +const PROFILES_WITH_CHROMA_INFO: [u8; 12] = + [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134]; + +/// The value H.264 assigns to primaries, transfer and matrix when the VUI +/// signals a video type but no colour description. +const UNSPECIFIED: u16 = 2; + +/// Reads the color description from the VUI of the first SPS in an `avcC` +/// body. +/// +/// `Ok(None)` means the SPS carries no video signal type at all, i.e. it says +/// nothing about color. An SPS that does carry one but omits the colour +/// description reports every component as unspecified. +pub(crate) fn sps_color_from_avcc(avcc: &[u8]) -> Result, &'static str> { + // configurationVersion, profile, compatibility, level, lengthSizeMinusOne, + // then the SPS count in the low five bits. + let sps_count = avcc.get(5).ok_or("avcC is truncated")? & 0x1f; + if sps_count == 0 { + return Err("avcC carries no SPS"); + } + let length = u16::from_be_bytes( + avcc.get(6..8) + .ok_or("avcC is truncated")? + .try_into() + .unwrap(), + ) as usize; + let nal = avcc.get(8..8 + length).ok_or("avcC SPS is truncated")?; + // The first byte is the NAL unit header. + let rbsp = remove_emulation_prevention(nal.get(1..).ok_or("SPS is empty")?); + parse_sps_color(&rbsp).ok_or("SPS ends before its VUI does") +} + +/// Undoes the escaping that keeps `00 00 0x` start codes out of a NAL unit's +/// payload. +fn remove_emulation_prevention(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len()); + let mut zeros = 0; + for &byte in data { + if zeros >= 2 && byte == 3 { + zeros = 0; + continue; + } + zeros = if byte == 0 { zeros + 1 } else { 0 }; + out.push(byte); + } + out +} + +/// Walks the SPS (ITU-T H.264 7.3.2.1.1) up to the video signal type in its +/// VUI (E.1.1). `None` means the data ran out first. +fn parse_sps_color(rbsp: &[u8]) -> Option> { + let mut bits = BitReader::new(rbsp); + + let profile_idc = bits.read(8)? as u8; + bits.read(8)?; // constraint flags and reserved bits + bits.read(8)?; // level_idc + bits.read_ue()?; // seq_parameter_set_id + + if PROFILES_WITH_CHROMA_INFO.contains(&profile_idc) { + let chroma_format_idc = bits.read_ue()?; + if chroma_format_idc == 3 { + bits.read(1)?; // separate_colour_plane_flag + } + bits.read_ue()?; // bit_depth_luma_minus8 + bits.read_ue()?; // bit_depth_chroma_minus8 + bits.read(1)?; // qpprime_y_zero_transform_bypass_flag + if bits.read(1)? == 1 { + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for index in 0..lists { + if bits.read(1)? == 1 { + skip_scaling_list(&mut bits, if index < 6 { 16 } else { 64 })?; + } + } + } + } + + bits.read_ue()?; // log2_max_frame_num_minus4 + match bits.read_ue()? { + 0 => { + bits.read_ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } + 1 => { + bits.read(1)?; // delta_pic_order_always_zero_flag + bits.read_se()?; // offset_for_non_ref_pic + bits.read_se()?; // offset_for_top_to_bottom_field + for _ in 0..bits.read_ue()? { + bits.read_se()?; // offset_for_ref_frame + } + } + _ => {} + } + bits.read_ue()?; // max_num_ref_frames + bits.read(1)?; // gaps_in_frame_num_value_allowed_flag + bits.read_ue()?; // pic_width_in_mbs_minus1 + bits.read_ue()?; // pic_height_in_map_units_minus1 + if bits.read(1)? == 0 { + bits.read(1)?; // mb_adaptive_frame_field_flag, present unless frame_mbs_only + } + bits.read(1)?; // direct_8x8_inference_flag + if bits.read(1)? == 1 { + for _ in 0..4 { + bits.read_ue()?; // frame_crop_{left,right,top,bottom}_offset + } + } + + if bits.read(1)? == 0 { + return Some(None); // no VUI + } + if bits.read(1)? == 1 { + // aspect_ratio_idc, with an explicit ratio after Extended_SAR. + if bits.read(8)? == 255 { + bits.read(16)?; // sar_width + bits.read(16)?; // sar_height + } + } + if bits.read(1)? == 1 { + bits.read(1)?; // overscan_appropriate_flag + } + if bits.read(1)? == 0 { + return Some(None); // no video signal type + } + bits.read(3)?; // video_format + let full_range = bits.read(1)? == 1; + let (primaries, transfer, matrix) = if bits.read(1)? == 1 { + ( + bits.read(8)? as u16, + bits.read(8)? as u16, + bits.read(8)? as u16, + ) + } else { + (UNSPECIFIED, UNSPECIFIED, UNSPECIFIED) + }; + + Some(Some(ColorDescription { + primaries, + transfer, + matrix, + full_range: Some(full_range), + })) +} + +/// Skips a `scaling_list()` (7.3.2.1.1.1), whose length depends on its own +/// contents. +fn skip_scaling_list(bits: &mut BitReader<'_>, size: usize) -> Option<()> { + let mut last = 8i64; + let mut next = 8i64; + for _ in 0..size { + if next != 0 { + next = (last + bits.read_se()? + 256) % 256; + } + if next != 0 { + last = next; + } + } + Some(()) +} + +struct BitReader<'a> { + data: &'a [u8], + position: usize, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, position: 0 } + } + + /// Reads `count` bits, at most 32, most significant first. + fn read(&mut self, count: u32) -> Option { + let mut value = 0u32; + for _ in 0..count { + let byte = *self.data.get(self.position / 8)?; + let bit = (byte >> (7 - self.position % 8)) & 1; + value = (value << 1) | bit as u32; + self.position += 1; + } + Some(value) + } + + /// Reads an unsigned Exp-Golomb code. + fn read_ue(&mut self) -> Option { + let mut leading_zeros = 0; + while self.read(1)? == 0 { + leading_zeros += 1; + if leading_zeros > 31 { + return None; + } + } + Some((1u32 << leading_zeros) - 1 + self.read(leading_zeros)?) + } + + /// Reads a signed Exp-Golomb code. + fn read_se(&mut self) -> Option { + let code = self.read_ue()? as i64; + Some(if code % 2 == 1 { + (code + 1) / 2 + } else { + -(code / 2) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Wraps an SPS NAL unit in the smallest `avcC` that carries it. + fn avcc_with_sps(nal: &[u8]) -> Vec { + let mut avcc = vec![1, nal[1], nal[2], nal[3], 0xff, 0xe1]; + avcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + avcc.extend_from_slice(nal); + avcc + } + + // Every SPS below was written by libx264 (FFmpeg 8.0.1), with the VUI set by + // the `h264_metadata` bitstream filter where noted. + + #[test] + fn reads_bt709_limited_range_from_a_high_profile_sps() { + // 1280x720 High 3.1, `colour_primaries=1:transfer_characteristics=1: + // matrix_coefficients=1:video_full_range_flag=0`. + let nal = [ + 0x67, 0x64, 0x00, 0x1f, 0xac, 0xd9, 0x40, 0x50, 0x05, 0xbb, 0x01, 0x6a, 0x02, 0x02, + 0x02, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x19, 0x07, 0x8c, 0x18, 0xcb, + ]; + assert_eq!( + sps_color_from_avcc(&avcc_with_sps(&nal)), + Ok(Some(ColorDescription::BT709_LIMITED)) + ); + } + + #[test] + fn reads_other_color_spaces_and_full_range() { + // 640x480 High 3.0, `colour_primaries=6:transfer_characteristics=6: + // matrix_coefficients=6:video_full_range_flag=1`. + let nal = [ + 0x67, 0x64, 0x00, 0x1e, 0xac, 0xd9, 0x40, 0xa0, 0x3d, 0xb0, 0x16, 0xe0, 0xc0, 0xc0, + 0xc8, 0x00, 0x00, 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x01, 0x90, 0x78, 0xb1, 0x6c, + 0xb0, + ]; + assert_eq!( + sps_color_from_avcc(&avcc_with_sps(&nal)), + Ok(Some(ColorDescription { + primaries: 6, + transfer: 6, + matrix: 6, + full_range: Some(true), + })) + ); + } + + #[test] + fn reports_unspecified_components_for_a_signal_type_without_colour_description() { + // 320x240 Baseline 1.3, `video_format=5:video_full_range_flag=0`. + let nal = [ + 0x67, 0x42, 0xc0, 0x0d, 0xd9, 0x01, 0x41, 0xfb, 0x01, 0x68, 0x80, 0x00, 0x00, 0x03, + 0x00, 0x80, 0x00, 0x00, 0x19, 0x07, 0x8a, 0x15, 0x24, + ]; + assert_eq!( + sps_color_from_avcc(&avcc_with_sps(&nal)), + Ok(Some(ColorDescription { + primaries: UNSPECIFIED, + transfer: UNSPECIFIED, + matrix: UNSPECIFIED, + full_range: Some(false), + })) + ); + } + + #[test] + fn reports_no_color_for_a_vui_without_signal_type() { + // 320x240 Baseline 1.3 as libx264 writes it by default: a VUI with + // timing information only. + let nal = [ + 0x67, 0x42, 0xc0, 0x0d, 0xd9, 0x01, 0x41, 0xfb, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, + 0x10, 0x00, 0x00, 0x03, 0x03, 0x20, 0xf1, 0x42, 0xa4, 0x80, + ]; + assert_eq!(sps_color_from_avcc(&avcc_with_sps(&nal)), Ok(None)); + } + + #[test] + fn removes_emulation_prevention_bytes() { + assert_eq!( + remove_emulation_prevention(&[0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x03, 0x00]), + vec![0x00, 0x00, 0x01, 0x00, 0x00, 0x00] + ); + } +} diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs index ba7d9935..f58981f0 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs @@ -16,6 +16,7 @@ pub mod driver; pub mod e2e; +mod h264; pub mod logging; pub mod mp4; pub mod options; diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs index 356b3085..52caf452 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs @@ -11,9 +11,19 @@ use std::fmt; #[derive(Debug)] pub enum Mp4Error { - Truncated { at: usize, need: usize }, + Truncated { + at: usize, + need: usize, + }, MissingBox(&'static str), - UnsupportedVersion { box_type: &'static str, version: u8 }, + UnsupportedVersion { + box_type: &'static str, + version: u8, + }, + Malformed { + what: &'static str, + reason: &'static str, + }, } impl fmt::Display for Mp4Error { @@ -26,6 +36,7 @@ impl fmt::Display for Mp4Error { Mp4Error::UnsupportedVersion { box_type, version } => { write!(f, "unsupported '{box_type}' version {version}") } + Mp4Error::Malformed { what, reason } => write!(f, "malformed {what}: {reason}"), } } } @@ -70,6 +81,12 @@ pub struct Track { /// the web it is negotiated with the browser at run time — so this reports /// what the file actually contains. pub avc_profile: Option, + /// Color description from the sample entry's `colr` box. `None` when the + /// box is absent or describes color with an ICC profile instead. + pub colr: Option, + /// Color description from the VUI of the SPS in `avcC`. `None` when the SPS + /// signals no video type, which leaves the color space to the player. + pub sps_color: Option, pub timescale: u32, /// Track duration in seconds, from `mdhd`. /// @@ -178,6 +195,8 @@ fn parse_track(trak: &[u8], movie_timescale: u32) -> Result { format: entry.format, has_decoder_config: entry.has_decoder_config, avc_profile: entry.avc_profile, + colr: entry.colr, + sps_color: entry.sps_color, timescale, duration: duration as f64 / timescale.max(1) as f64, width: entry.width, @@ -253,11 +272,76 @@ impl fmt::Display for AvcProfile { } } +/// A color space as ITU-T H.273 code points, which `colr` boxes and H.264 VUI +/// share. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ColorDescription { + pub primaries: u16, + pub transfer: u16, + pub matrix: u16, + /// `None` for a QuickTime `nclc` box, which has no range flag. + pub full_range: Option, +} + +impl ColorDescription { + /// BT.709 primaries, transfer and matrix in limited range, which is what + /// every backend is meant to produce. + pub const BT709_LIMITED: Self = Self { + primaries: 1, + transfer: 1, + matrix: 1, + full_range: Some(false), + }; + + /// True when this describes BT.709 in limited range. A QuickTime `nclc` + /// box states no range and is taken as limited, which is how QuickTime + /// defines it. + pub fn is_bt709_limited(&self) -> bool { + self.primaries == 1 + && self.transfer == 1 + && self.matrix == 1 + && self.full_range != Some(true) + } +} + +impl fmt::Display for ColorDescription { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "primaries {}, transfer {}, matrix {}, ", + self.primaries, self.transfer, self.matrix + )?; + match self.full_range { + Some(true) => write!(f, "full range"), + Some(false) => write!(f, "limited range"), + None => write!(f, "range unstated"), + } + } +} + +/// Reads an `nclx` (ISO/IEC 14496-12) or `nclc` (QuickTime) `colr` body. +fn parse_colr(colr: &[u8]) -> Result> { + let full_range = match &read_array::<4>(colr, 0)? { + b"nclx" => Some(read_array::<1>(colr, 10)?[0] & 0x80 != 0), + b"nclc" => None, + // An ICC profile does not reduce to code points. + _ => return Ok(None), + }; + Ok(Some(ColorDescription { + primaries: read_u16(colr, 4)?, + transfer: read_u16(colr, 6)?, + matrix: read_u16(colr, 8)?, + full_range, + })) +} + #[derive(Default)] struct SampleEntry { format: String, has_decoder_config: bool, avc_profile: Option, + colr: Option, + sps_color: Option, width: u32, height: u32, sample_rate: u32, @@ -317,7 +401,15 @@ impl SampleEntry { if let (Some(&profile), Some(&level)) = (body.get(1), body.get(3)) { parsed.avc_profile = Some(AvcProfile { profile, level }); } + parsed.sps_color = + crate::h264::sps_color_from_avcc(body).map_err(|reason| { + Mp4Error::Malformed { + what: "SPS", + reason, + } + })?; } + b"colr" => parsed.colr = parse_colr(body)?, b"esds" => parsed.has_decoder_config = true, _ => {} } diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs index dc204727..aaf768b1 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs @@ -11,7 +11,7 @@ use std::fmt; use crate::e2e::{E2eConfig, E2eReport}; -use crate::mp4::{Mp4Summary, Track, TrackKind}; +use crate::mp4::{ColorDescription, Mp4Summary, Track, TrackKind}; /// An AAC frame always covers 1024 samples per channel. const AAC_SAMPLES_PER_FRAME: u64 = 1024; @@ -35,6 +35,11 @@ const SYNC_TOLERANCE_SECS: f64 = 0.05; /// and Media Foundation rounds the interval to a whole number of frames. const IDR_TOLERANCE_FRAMES: u32 = 1; +/// Whether the muxer on this target writes a `colr` box. muxide, which the web +/// backend muxes with, writes none, so the SPS VUI is the only place the web +/// output can state its color space. +const MUXER_WRITES_COLR: bool = !cfg!(target_os = "emscripten"); + /// Everything that did not hold. #[derive(Debug)] pub struct VerifyError { @@ -212,6 +217,8 @@ fn verify_video_track( ) }); + verify_video_color(findings, track); + // A decoder joining at the start needs the first sample to be a keyframe. findings.check(track.is_sync_sample(1), || { "the first video sample is not a sync sample".to_string() @@ -286,6 +293,41 @@ fn verify_idr_interval(findings: &mut Findings, track: &Track, config: &E2eConfi } } +/// Every backend converts to BT.709 limited range, so that is what the file has +/// to declare. Without a declaration players guess, and they guess differently. +/// +/// The container's `colr` box and the SPS VUI are checked separately because +/// they come from different places: the box from the muxer's track format, the +/// VUI from the encoder. Players disagree on which one wins, so wherever both +/// are present they have to agree. +/// +/// This checks the declaration only. Whether the pixels were converted with the +/// matrix it names takes decoding, which this harness does not do. +fn verify_video_color(findings: &mut Findings, track: &Track) { + let expected = ColorDescription::BT709_LIMITED; + + match track.colr { + Some(colr) => findings.check(colr.is_bt709_limited(), || { + format!("the 'colr' box declares {colr}, expected {expected}") + }), + None => findings.check(!MUXER_WRITES_COLR, || { + "video track carries no 'colr' box, so players have to guess its color space" + .to_string() + }), + } + + match track.sps_color { + Some(sps) => findings.check(sps.is_bt709_limited(), || { + format!("the SPS VUI declares {sps}, expected {expected}") + }), + None => findings.check(MUXER_WRITES_COLR, || { + "the SPS VUI declares no color space, and this muxer writes no 'colr' box, \ + so nothing does" + .to_string() + }), + } +} + fn verify_audio_track( findings: &mut Findings, track: &Track, @@ -346,7 +388,7 @@ pub fn describe(summary: &Mp4Summary) -> String { for track in &summary.tracks { out.push_str(&match track.kind { TrackKind::Video => format!( - " video: {} {}x{} ({}), {} frames, {} sync, start {:.3} s, {:.3} s\n", + " video: {} {}x{} ({}), {} frames, {} sync, start {:.3} s, {:.3} s\n colr: {}; SPS: {}\n", track.format, track.width, track.height, @@ -361,6 +403,8 @@ pub fn describe(summary: &Mp4Summary) -> String { }, track.start_time, track.duration, + describe_color(track.colr), + describe_color(track.sps_color), ), TrackKind::Audio => format!( " audio: {} {} Hz {} ch, {} frames, start {:.3} s, {:.3} s, decoder config: {}\n", @@ -377,3 +421,10 @@ pub fn describe(summary: &Mp4Summary) -> String { } out } + +fn describe_color(color: Option) -> String { + match color { + Some(color) => color.to_string(), + None => "none".to_string(), + } +} diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts index c757af5c..6071a237 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts @@ -144,6 +144,8 @@ window["unienc_webcodecs"] = { { width: number, height: number, + codedWidth: number, + codedHeight: number, timestamp: number, isKey: boolean }, @@ -212,20 +214,40 @@ window["unienc_webcodecs"] = { return encoder; }, encodeFrame: (encoder, data, options) => { + // The frame arrives as I420, already converted to BT.709 limited range by + // `VideoFrameBgra32::to_i420`, and padded to even dimensions: Y, then U, then V, each + // tightly packed. + const chromaWidth = options.codedWidth / 2; + const chromaHeight = options.codedHeight / 2; + const lumaSize = options.codedWidth * options.codedHeight; + const chromaSize = chromaWidth * chromaHeight; const init: VideoFrameBufferInit = { timestamp: options.timestamp * 1000 * 1000, - codedWidth: options.width, - codedHeight: options.height, + codedWidth: options.codedWidth, + codedHeight: options.codedHeight, visibleRect: {x: 0, y: 0, width: options.width, height: options.height}, displayWidth: options.width, displayHeight: options.height, - format: "BGRA", + format: "I420", layout: [ - { - offset: 0, - stride: options.width * 4 // BGRA = 4 bytes per pixel - } - ] + {offset: 0, stride: options.codedWidth}, + {offset: lumaSize, stride: chromaWidth}, + {offset: lumaSize + chromaSize, stride: chromaWidth}, + ], + // What the planes hold, which the browser copies into the SPS VUI. + // + // The conversion happens on our side because handing the browser an RGB frame + // leaves the YUV color space to it, and browsers disagree. Chrome 154 converts + // into whatever matrix and range the RGB frame's color space names, while the + // Chrome on the CI runner, given the same declaration, converted to BT.601 and + // tagged the stream accordingly. With YUV input there is no conversion left for + // the browser to choose. + colorSpace: { + primaries: "bt709", + transfer: "bt709", + matrix: "bt709", + fullRange: false + } }; const frame = new VideoFrame(data, init); encoder.encode(frame, { diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs index 73da1bfb..c3960273 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs @@ -9,6 +9,17 @@ static LIBRARY: LazyLock = LazyLock::new(Library::new); struct Library; +/// A frame in the layout the JavaScript side reads: I420 with the Y, U and V planes packed one +/// after another, sized for `coded_width` x `coded_height`, of which the top-left `width` x +/// `height` is the picture. +pub struct I420Frame<'a> { + pub data: &'a [u8], + pub width: u32, + pub height: u32, + pub coded_width: u32, + pub coded_height: u32, +} + pub struct VideoEncoderHandle { id: i32, callback: *mut Box, @@ -32,13 +43,11 @@ impl VideoEncoderHandle { pub fn push_video_frame( &self, - data: &[u8], - width: u32, - height: u32, + frame: &I420Frame<'_>, timestamp: f64, is_key: bool, ) -> Result<(), JavaScriptError> { - LIBRARY.push_video_frame(self.id, data, width, height, timestamp, is_key) + LIBRARY.push_video_frame(self.id, frame, timestamp, is_key) } pub async fn flush(&self) -> Result<(), JavaScriptError> { @@ -249,9 +258,7 @@ impl Library { fn push_video_frame( &self, encoder_index: i32, - data: &[u8], - width: u32, - height: u32, + frame: &I420Frame<'_>, timestamp: f64, is_key: bool, ) -> Result<(), JavaScriptError> { @@ -262,13 +269,19 @@ impl Library { const dataLength = {data_length}; const width = {width}; const height = {height}; + const codedWidth = {coded_width}; + const codedHeight = {coded_height}; const timestamp = {timestamp}; const isKey = {is_key}; const dataArray = Module.HEAPU8.subarray(dataPtr, dataPtr + dataLength); - window.unienc_webcodecs.video.push(encoderIndex, dataArray, {{width, height, timestamp, isKey}}); + window.unienc_webcodecs.video.push(encoderIndex, dataArray, {{width, height, codedWidth, codedHeight, timestamp, isKey}}); ", - data_ptr = data.as_ptr() as usize, - data_length = data.len(), + data_ptr = frame.data.as_ptr() as usize, + data_length = frame.data.len(), + width = frame.width, + height = frame.height, + coded_width = frame.coded_width, + coded_height = frame.coded_height, timestamp = timestamp ); self.run_script(&script) diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs index ca3b6e6d..ffb5b0c6 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs @@ -1,4 +1,4 @@ -use crate::js::VideoEncoderHandle; +use crate::js::{I420Frame, VideoEncoderHandle}; use bincode::{Decode, Encode}; use futures::StreamExt; use futures::channel::mpsc; @@ -106,7 +106,12 @@ impl EncoderInput for WebCodecsVideoEncoderInput { let encoder_handle = self.encoder_handle.as_ref().unwrap(); - let pixels = &frame.buffer.data()[..frame.buffer.len()]; + // Converted here rather than by the browser, whose choice of YUV color space for an + // RGB frame varies between versions. I420 needs even dimensions, so odd ones are padded + // and the padding is cropped off again by the frame's visible rectangle. + let coded_width = frame.width.next_multiple_of(2); + let coded_height = frame.height.next_multiple_of(2); + let i420 = frame.to_i420(Some((coded_width, coded_height)))?; let since_prev_key = match self.prev_key_timestamp { Some(prev) => data.timestamp - prev, None => f64::INFINITY, @@ -116,7 +121,17 @@ impl EncoderInput for WebCodecsVideoEncoderInput { self.prev_key_timestamp = Some(data.timestamp); } encoder_handle - .push_video_frame(pixels, frame.width, frame.height, data.timestamp, is_key) + .push_video_frame( + &I420Frame { + data: &i420, + width: frame.width, + height: frame.height, + coded_width, + coded_height, + }, + data.timestamp, + is_key, + ) .context("Failed to push video frame to WebCodecs EncoderHandle")?; Ok(()) } diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs new file mode 100644 index 00000000..56d18c79 --- /dev/null +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs @@ -0,0 +1,376 @@ +//! Adds a `colr` box to the video sample entry of a finished MP4 file. +//! +//! The MPEG-4 file sink writes no `colr` box, whatever color attributes the media type it is +//! given carries: the e2e output on Windows had none although `MF_MT_VIDEO_PRIMARIES`, +//! `MF_MT_TRANSFER_FUNCTION`, `MF_MT_YUV_MATRIX` and `MF_MT_VIDEO_NOMINAL_RANGE` were all set on +//! the sink's video type, and the sink's documentation does not mention the box at all. The box is +//! therefore added after the sink has finalized the file. +//! +//! The sink writes `moov` after `mdat` as the last top-level box, so the box can be added by +//! rewriting `moov` in place, which moves no media data and invalidates no chunk offset. A file laid +//! out any other way is left untouched. + +use std::fs::OpenOptions; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::Path; + +/// A `colr` box of type `nclx` declaring BT.709 primaries, transfer and matrix in limited range +/// (ISO/IEC 14496-12, 12.1.5). +const BT709_LIMITED_COLR: [u8; 19] = [ + 0, 0, 0, 19, b'c', b'o', b'l', b'r', b'n', b'c', b'l', b'x', // + 0, 1, // colour_primaries + 0, 1, // transfer_characteristics + 0, 1, // matrix_coefficients + 0x00, // full_range_flag = 0, reserved = 0 +]; + +/// Offset of the child boxes within a visual sample entry's body: the sample entry header (8) +/// followed by the fields of `VisualSampleEntry` (70). +const VISUAL_SAMPLE_ENTRY_FIELDS: usize = 78; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum ColrOutcome { + Added, + AlreadyPresent, + /// The file was left untouched for the stated reason. + Skipped(&'static str), +} + +/// Adds a BT.709 limited range `colr` box to the first video track of the MP4 file at `path`. +pub(crate) fn add_bt709_colr(path: &Path) -> io::Result { + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + let file_len = file.metadata()?.len(); + + let mut at = 0u64; + let mut moov = None; + while at < file_len { + file.seek(SeekFrom::Start(at))?; + let mut header = [0u8; 16]; + let available = (file_len - at).min(16) as usize; + file.read_exact(&mut header[..available])?; + if available < 8 { + return Ok(ColrOutcome::Skipped("the file ends inside a box header")); + } + let size = match u32::from_be_bytes(header[0..4].try_into().unwrap()) { + 0 => file_len - at, + 1 if available == 16 => u64::from_be_bytes(header[8..16].try_into().unwrap()), + 1 => return Ok(ColrOutcome::Skipped("the file ends inside a box header")), + size => size as u64, + }; + if size < 8 || at + size > file_len { + return Ok(ColrOutcome::Skipped("a top-level box overruns the file")); + } + if &header[4..8] == b"moov" { + moov = Some((at, size)); + } + at += size; + } + + let Some((moov_at, moov_size)) = moov else { + return Ok(ColrOutcome::Skipped("the file has no moov box")); + }; + if moov_at + moov_size != file_len { + return Ok(ColrOutcome::Skipped("moov is not the last box in the file")); + } + + let mut moov = vec![0u8; moov_size as usize]; + file.seek(SeekFrom::Start(moov_at))?; + file.read_exact(&mut moov)?; + + let patched = match insert_bt709_colr(&moov) { + Ok(Some(patched)) => patched, + Ok(None) => return Ok(ColrOutcome::AlreadyPresent), + Err(reason) => return Ok(ColrOutcome::Skipped(reason)), + }; + + // The new moov is longer than the old one and starts at the same offset, so it overwrites the + // old one entirely and extends the file. + file.seek(SeekFrom::Start(moov_at))?; + file.write_all(&patched)?; + file.flush()?; + Ok(ColrOutcome::Added) +} + +/// One box found in a buffer: where it starts, how long its header is and its total size. +#[derive(Clone, Copy)] +struct BoxRef { + at: usize, + header: usize, + size: usize, + box_type: [u8; 4], +} + +impl BoxRef { + fn body(&self) -> (usize, usize) { + (self.at + self.header, self.at + self.size) + } +} + +fn read_box(data: &[u8], at: usize, end: usize) -> Option { + let header = data.get(at..at + 8)?; + let box_type: [u8; 4] = header[4..8].try_into().unwrap(); + let (header_len, size) = match u32::from_be_bytes(header[0..4].try_into().unwrap()) { + 0 => (8, end - at), + 1 => ( + 16, + u64::from_be_bytes(data.get(at + 8..at + 16)?.try_into().unwrap()) as usize, + ), + size => (8, size as usize), + }; + if size < header_len || at + size > end { + return None; + } + Some(BoxRef { + at, + header: header_len, + size, + box_type, + }) +} + +fn find_child(data: &[u8], (start, end): (usize, usize), wanted: &[u8; 4]) -> Option { + children(data, (start, end)).find(|found| &found.box_type == wanted) +} + +fn children(data: &[u8], (start, end): (usize, usize)) -> impl Iterator + '_ { + let mut at = start; + std::iter::from_fn(move || { + let found = read_box(data, at, end)?; + at += found.size; + Some(found) + }) +} + +/// Returns `moov` with a BT.709 limited range `colr` box appended to the first video track's +/// sample entry, or `None` when that entry already has a `colr` box. +fn insert_bt709_colr(moov: &[u8]) -> Result>, &'static str> { + let moov_box = read_box(moov, 0, moov.len()).ok_or("moov is malformed")?; + if &moov_box.box_type != b"moov" { + return Err("the buffer is not a moov box"); + } + + // Every box from moov down to the sample entry contains the insertion point, so each of their + // sizes grows by the size of the new box. + let mut ancestors = vec![moov_box]; + let trak = children(moov, moov_box.body()) + .filter(|found| &found.box_type == b"trak") + .find(|trak| is_video_track(moov, *trak)) + .ok_or("moov has no video track")?; + ancestors.push(trak); + let mut parent = trak; + for wanted in [b"mdia", b"minf", b"stbl", b"stsd"] { + parent = find_child(moov, parent.body(), wanted).ok_or("the video track is incomplete")?; + ancestors.push(parent); + } + + // stsd is a full box: version and flags (4) and the entry count (4) precede the entries. + let (stsd_start, stsd_end) = parent.body(); + let entry = read_box(moov, stsd_start + 8, stsd_end).ok_or("stsd has no sample entry")?; + if !matches!(&entry.box_type, b"avc1" | b"avc3") { + return Err("the video sample entry is not H.264"); + } + ancestors.push(entry); + + let (entry_start, entry_end) = entry.body(); + let entry_children = (entry_start + VISUAL_SAMPLE_ENTRY_FIELDS, entry_end); + if entry_children.0 > entry_end { + return Err("the video sample entry is truncated"); + } + if find_child(moov, entry_children, b"colr").is_some() { + return Ok(None); + } + + let insert_at = entry.at + entry.size; + let mut patched = Vec::with_capacity(moov.len() + BT709_LIMITED_COLR.len()); + patched.extend_from_slice(&moov[..insert_at]); + patched.extend_from_slice(&BT709_LIMITED_COLR); + patched.extend_from_slice(&moov[insert_at..]); + + // Every ancestor starts before the insertion point, so its offset is unchanged. + let grow = BT709_LIMITED_COLR.len(); + for ancestor in ancestors { + let field = &mut patched[ancestor.at..]; + match ancestor.header { + 16 => { + let size = u64::from_be_bytes(field[8..16].try_into().unwrap()) + grow as u64; + field[8..16].copy_from_slice(&size.to_be_bytes()); + } + _ => { + let size = u32::from_be_bytes(field[0..4].try_into().unwrap()); + // A size of 0 runs to the end of the enclosing space and stays valid as it is. + if size != 0 { + let size = size + .checked_add(grow as u32) + .ok_or("a box would exceed the 32-bit size limit")?; + field[0..4].copy_from_slice(&size.to_be_bytes()); + } + } + } + } + + Ok(Some(patched)) +} + +fn is_video_track(moov: &[u8], trak: BoxRef) -> bool { + find_child(moov, trak.body(), b"mdia") + .and_then(|mdia| find_child(moov, mdia.body(), b"hdlr")) + // hdlr: version and flags (4), pre_defined (4), handler_type (4) + .and_then(|hdlr| moov.get(hdlr.at + hdlr.header + 8..hdlr.at + hdlr.header + 12)) + .is_some_and(|handler| handler == b"vide") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn boxed(box_type: &[u8; 4], body: &[u8]) -> Vec { + let mut out = ((body.len() + 8) as u32).to_be_bytes().to_vec(); + out.extend_from_slice(box_type); + out.extend_from_slice(body); + out + } + + fn full_box(box_type: &[u8; 4], body: &[u8]) -> Vec { + boxed(box_type, &[&[0u8; 4][..], body].concat()) + } + + fn hdlr(handler: &[u8; 4]) -> Vec { + full_box(b"hdlr", &[&[0u8; 4][..], handler, &[0u8; 13]].concat()) + } + + fn track(handler: &[u8; 4], entry: Vec) -> Vec { + let stsd = full_box(b"stsd", &[&1u32.to_be_bytes()[..], &entry].concat()); + let stbl = boxed(b"stbl", &[stsd, full_box(b"stts", &[0; 4])].concat()); + let minf = boxed(b"minf", &stbl); + let mdia = boxed(b"mdia", &[hdlr(handler), minf].concat()); + boxed(b"trak", &mdia) + } + + fn avc1(extra_children: &[u8]) -> Vec { + let avcc = boxed(b"avcC", &[1, 0x42, 0xc0, 0x1f, 0xff, 0xe0, 0x00]); + boxed( + b"avc1", + &[ + &[0u8; VISUAL_SAMPLE_ENTRY_FIELDS][..], + &avcc, + extra_children, + ] + .concat(), + ) + } + + fn moov(tracks: &[Vec]) -> Vec { + boxed( + b"moov", + &[full_box(b"mvhd", &[0; 96]), tracks.concat()].concat(), + ) + } + + /// Checks that every box in `data[start..end]` exactly fills its parent, recursing into the + /// containers this module edits. + fn assert_well_formed(data: &[u8], (start, end): (usize, usize)) { + let mut at = start; + while at < end { + let found = read_box(data, at, end).expect("box overruns its parent"); + match &found.box_type { + b"moov" | b"trak" | b"mdia" | b"minf" | b"stbl" => { + assert_well_formed(data, found.body()) + } + b"stsd" => assert_well_formed(data, (found.body().0 + 8, found.body().1)), + b"avc1" => assert_well_formed( + data, + (found.body().0 + VISUAL_SAMPLE_ENTRY_FIELDS, found.body().1), + ), + _ => {} + } + at += found.size; + } + assert_eq!(at, end, "children do not fill their parent"); + } + + #[test] + fn appends_colr_to_the_video_sample_entry_and_grows_every_ancestor() { + let mp4a = boxed(b"mp4a", &[0; 28]); + let original = moov(&[track(b"soun", mp4a), track(b"vide", avc1(&[]))]); + + let patched = insert_bt709_colr(&original).unwrap().unwrap(); + + assert_eq!(patched.len(), original.len() + BT709_LIMITED_COLR.len()); + assert_well_formed(&patched, (0, patched.len())); + // The box lands at the end of the avc1 entry, which is the last thing before the video + // track's stts. + let stts_at = patched + .windows(4) + .rposition(|window| window == b"stts") + .unwrap() + - 4; + assert_eq!( + &patched[stts_at - BT709_LIMITED_COLR.len()..stts_at], + &BT709_LIMITED_COLR + ); + // Past moov's own size field, everything before the video track (mvhd and the audio + // track) is byte-for-byte unchanged. + let video_trak_at = patched + .windows(4) + .rposition(|window| window == b"trak") + .unwrap() + - 4; + assert_eq!(&patched[4..video_trak_at], &original[4..video_trak_at]); + } + + #[test] + fn leaves_an_existing_colr_box_alone() { + let original = moov(&[track(b"vide", avc1(&BT709_LIMITED_COLR))]); + assert_eq!(insert_bt709_colr(&original), Ok(None)); + } + + #[test] + fn refuses_a_file_without_a_video_track() { + let original = moov(&[track(b"soun", boxed(b"mp4a", &[0; 28]))]); + assert_eq!(insert_bt709_colr(&original), Err("moov has no video track")); + } + + #[test] + fn rewrites_a_trailing_moov_in_place() { + let path = std::env::temp_dir().join(format!("unienc-colr-{}.mp4", std::process::id())); + let mdat = boxed(b"mdat", &[0xaa; 64]); + let original_moov = moov(&[track(b"vide", avc1(&[]))]); + let file = [ + boxed(b"ftyp", b"isom\0\0\0\0"), + mdat.clone(), + original_moov.clone(), + ] + .concat(); + std::fs::write(&path, &file).unwrap(); + + assert_eq!(add_bt709_colr(&path).unwrap(), ColrOutcome::Added); + let written = std::fs::read(&path).unwrap(); + // ftyp and mdat are unchanged, so no chunk offset moved. + assert_eq!(&written[..16 + mdat.len()], &file[..16 + mdat.len()]); + assert_eq!(written.len(), file.len() + BT709_LIMITED_COLR.len()); + assert_well_formed(&written, (0, written.len())); + + assert_eq!(add_bt709_colr(&path).unwrap(), ColrOutcome::AlreadyPresent); + std::fs::remove_file(&path).unwrap(); + } + + #[test] + fn leaves_a_file_whose_moov_precedes_mdat_untouched() { + let path = + std::env::temp_dir().join(format!("unienc-colr-front-{}.mp4", std::process::id())); + let file = [ + boxed(b"ftyp", b"isom\0\0\0\0"), + moov(&[track(b"vide", avc1(&[]))]), + boxed(b"mdat", &[0xaa; 64]), + ] + .concat(); + std::fs::write(&path, &file).unwrap(); + + assert_eq!( + add_bt709_colr(&path).unwrap(), + ColrOutcome::Skipped("moov is not the last box in the file") + ); + assert_eq!(std::fs::read(&path).unwrap(), file); + std::fs::remove_file(&path).unwrap(); + } +} diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs index 5c23c7ca..3c470031 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs @@ -18,6 +18,8 @@ use crate::mft::MediaEventGeneratorCustom; use crate::video::VideoEncodedData; use windows::core::Interface; +mod colr; + enum LazyStream { None { tx: oneshot::Sender>>, @@ -87,6 +89,7 @@ impl MediaFoundationMuxer { let (audio_stream_tx, audio_stream_rx) = oneshot::channel::>(); let runtime_clone = runtime.clone(); + let output_path = output_path.to_path_buf(); runtime.spawn_ret(async move { let result: Result<()> = async move { @@ -172,10 +175,15 @@ impl MediaFoundationMuxer { let _ = unsafe { sink.Shutdown() }; } + // The sink has written everything it will by now. Closing the byte stream releases + // the file so that it can be reopened below. + let _ = unsafe { file.Close() }; + result } .await; + let result = result.and_then(|()| add_color_description(&output_path)); let _ = finish_tx.send(result); }); @@ -200,6 +208,23 @@ impl MediaFoundationMuxer { /// a non-empty PROPVARIANT (see the PlaceMarker call in `Stream::new`). const END_OF_SEGMENT_MARKER_CONTEXT: u32 = 1; +/// Adds the `colr` box the MPEG-4 file sink does not write. See [`colr`] for why it is needed. +/// +/// A file that cannot be patched safely is left as the sink wrote it, which is still playable; +/// only an I/O error, which may have left the file half-written, fails the mux. +fn add_color_description(path: &Path) -> Result<()> { + match colr::add_bt709_colr(path) { + Ok(colr::ColrOutcome::Added | colr::ColrOutcome::AlreadyPresent) => Ok(()), + Ok(colr::ColrOutcome::Skipped(reason)) => { + log::warn!("Muxed file left without a colr box: {reason}"); + Ok(()) + } + Err(err) => Err(WindowsError::Other(format!( + "Failed to add a colr box to the muxed file: {err}" + ))), + } +} + struct Stream { sample_tx: mpsc::Sender>, } diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs index c36373d8..abf6e3cc 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs @@ -18,6 +18,26 @@ pub struct MediaFoundationVideoEncoder { fps_hint: f64, } +/// Describes a media type as BT.709 limited range. +/// +/// On the input type this states what the NV12 buffers `VideoFrameBgra32::to_yuv420_planes` +/// produces actually contain, and on the output type it asks the encoder to record the same in the +/// H.264 VUI. +/// +/// Not every encoder honors the request. The Microsoft software H.264 encoder writes no video +/// signal type into the VUI regardless, and its documentation lists no color attribute or codec +/// property among those it supports. The muxer therefore states the color space in a `colr` box of +/// its own (see `mux::colr`) rather than relying on the encoder. +fn set_bt709_color_attributes(media_type: &IMFMediaType) -> Result<()> { + unsafe { + media_type.SetUINT32(&MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709.0 as u32)?; + media_type.SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709.0 as u32)?; + media_type.SetUINT32(&MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709.0 as u32)?; + media_type.SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235.0 as u32)?; + } + Ok(()) +} + impl MediaFoundationVideoEncoder { pub fn new(options: &V, runtime: &impl Runtime) -> Result { let input_type = unsafe { @@ -32,6 +52,7 @@ impl MediaFoundationVideoEncoder { )?; input_type.SetUINT64(&MF_MT_FRAME_RATE, ((options.fps_hint() as u64) << 32) + 1)?; + set_bt709_color_attributes(&input_type)?; input_type }; @@ -47,6 +68,7 @@ impl MediaFoundationVideoEncoder { )?; output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; output_type.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base.0 as u32)?; + set_bt709_color_attributes(&output_type)?; output_type };