From 7149c1e78fae03c800f14ba4a4ba75eb1d0793b7 Mon Sep 17 00:00:00 2001 From: ruccho Date: Wed, 19 Aug 2026 14:57:08 +0900 Subject: [PATCH 1/7] Tag encoded video as BT.709 and convert to match The MP4 files produced by every backend carried no color information at all: no `colr` box and no colour description in the SPS VUI. Players have to guess at the color space, and some warn about it. Tag the output as BT.709 limited range on all backends, and make the conversions actually produce BT.709 so the tags describe the pixels: - `VideoFrameBgra32::to_yuv420_planes`, used by the readback paths on Android and Windows, used BT.601 limited range coefficients. Replace them with the BT.709 equivalents. - The FFmpeg backend let FFmpeg pick the matrix, which is BT.601 at the resolutions this library records. Tag the frames so the conversion and the VUI both become BT.709. Per backend: - Apple: set the color properties on the VTCompressionSession, and add the matching format description extensions in the muxer. AVAssetWriter writes the `colr` box from the format description it is handed, and since encoded frames travel back through the caller as opaque buffers, the muxer never sees the encoder's own. - Android: set `color-standard`, `color-transfer` and `color-range` on the encoder format, and restate them on the muxer track format because codecs are not guaranteed to echo them back. - Windows: set the color attributes on both the input and the output media type. The muxer inherits them, as media type attributes are serialized generically. - FFmpeg: tag the frames with `setparams`. The `-color_primaries` and `-color_trc` output options are not reliably forwarded to the encoder. - WebCodecs: declare the source frame's color space, the only lever the API offers. Co-Authored-By: Claude Opus 5 --- .../crates/unienc_android_mc/src/config.rs | 10 +++++ .../crates/unienc_android_mc/src/mux/mod.rs | 22 ++++++++++- .../crates/unienc_android_mc/src/video/mod.rs | 14 +++++++ .../crates/unienc_apple_vt/src/mux/mod.rs | 31 ++++++++++++++- .../crates/unienc_apple_vt/src/video/mod.rs | 38 ++++++++++++++++++- .../unienc/crates/unienc_common/src/lib.rs | 25 ++++++++++-- .../crates/unienc_ffmpeg/src/video/mod.rs | 15 ++++++++ .../crates/unienc_webcodecs/src/js/library.ts | 13 ++++++- .../crates/unienc_windows_mf/src/video/mod.rs | 18 +++++++++ 9 files changed, 176 insertions(+), 10 deletions(-) diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs index 6b9237ea..acc2045d 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs @@ -24,6 +24,9 @@ pub mod format_keys { pub const KEY_LEVEL: &str = "level"; pub const KEY_PRIORITY: &str = "priority"; pub const KEY_OPERATING_RATE: &str = "operating-rate"; + pub const KEY_COLOR_STANDARD: &str = "color-standard"; + pub const KEY_COLOR_TRANSFER: &str = "color-transfer"; + pub const KEY_COLOR_RANGE: &str = "color-range"; // Audio keys pub const KEY_SAMPLE_RATE: &str = "sample-rate"; @@ -33,6 +36,13 @@ pub mod format_keys { pub const COLOR_FORMAT_SURFACE: jint = 0x7F000789; pub const COLOR_FORMAT_YUV420_FLEXIBLE: jint = 0x7F420888; + +/// `MediaFormat.COLOR_STANDARD_BT709`, `COLOR_TRANSFER_SDR_VIDEO` and `COLOR_RANGE_LIMITED`. +/// All three were introduced in API 24, below the minimum this crate targets, so setting them +/// needs no version guard. +pub const COLOR_STANDARD_BT709: jint = 1; +pub const COLOR_TRANSFER_SDR_VIDEO: jint = 3; +pub const COLOR_RANGE_LIMITED: jint = 2; pub const AAC_OBJECT_TYPE_AAC_LC: jint = 2; pub const MUXER_OUTPUT_FORMAT_MPEG_4: jint = 0; diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs index e0d74427..56ffd6af 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs @@ -4,7 +4,10 @@ use tokio::sync::{RwLock, oneshot}; use unienc_common::{CompletionHandle, Muxer, MuxerInput}; use crate::common::*; -use crate::config::MUXER_OUTPUT_FORMAT_MPEG_4; +use crate::config::{ + COLOR_RANGE_LIMITED, COLOR_STANDARD_BT709, COLOR_TRANSFER_SDR_VIDEO, + MUXER_OUTPUT_FORMAT_MPEG_4, format_keys, +}; use crate::error::{AndroidError, Result}; use crate::java::*; @@ -127,6 +130,23 @@ async fn push( "height".to_string(), crate::common::MediaFormatValue::Integer(height as i32), ); + + // The track format is what MediaMuxer writes the `colr` box from. These keys are + // set on the encoder and most codecs echo them back in their output format, but + // that is not guaranteed across vendors, so state them here as well. The values + // must match the ones `create_video_format` configures the encoder with. + map.insert( + format_keys::KEY_COLOR_STANDARD.to_string(), + crate::common::MediaFormatValue::Integer(COLOR_STANDARD_BT709), + ); + map.insert( + format_keys::KEY_COLOR_TRANSFER.to_string(), + crate::common::MediaFormatValue::Integer(COLOR_TRANSFER_SDR_VIDEO), + ); + map.insert( + format_keys::KEY_COLOR_RANGE.to_string(), + crate::common::MediaFormatValue::Integer(COLOR_RANGE_LIMITED), + ); } let mut shared_state_lock = shared_state.write().await; diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs index def6dca6..7ca6daa1 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs @@ -415,5 +415,19 @@ fn create_video_format_raw( set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?; set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?; + // Tag the stream as BT.709 limited range. Without these the encoder leaves the color + // information unspecified, so the muxed file gives players nothing to go on. The values match + // the conversion `VideoFrameBgra32::to_yuv420_planes` performs on the buffer input path, and + // on the surface input path they additionally tell MediaCodec which coefficients to use when + // it converts the frames the Vulkan preprocessor hands it. + set_format_integer(env, &format_obj, KEY_COLOR_STANDARD, COLOR_STANDARD_BT709)?; + set_format_integer( + env, + &format_obj, + KEY_COLOR_TRANSFER, + COLOR_TRANSFER_SDR_VIDEO, + )?; + set_format_integer(env, &format_obj, KEY_COLOR_RANGE, COLOR_RANGE_LIMITED)?; + SafeGlobalRef::new(env, format_obj) } diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs index e7b3ec34..1d040974 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs @@ -16,10 +16,14 @@ use objc2_av_foundation::{ use objc2_core_audio_types::{ AudioStreamBasicDescription, AudioStreamPacketDescription, MPEG4ObjectID, kAudioFormatMPEG4AAC, }; +use objc2_core_foundation::{CFDictionary, CFString, CFType}; use objc2_core_media::{ CMAudioFormatDescriptionCreate, CMAudioSampleBufferCreateReadyWithPacketDescriptions, CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMTime, CMVideoFormatDescriptionCreate, - kCMBlockBufferAssureMemoryNowFlag, kCMTimeZero, kCMVideoCodecType_H264, + kCMBlockBufferAssureMemoryNowFlag, kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionExtension_ColorPrimaries, kCMFormatDescriptionExtension_TransferFunction, + kCMFormatDescriptionExtension_YCbCrMatrix, kCMFormatDescriptionTransferFunction_ITU_R_709_2, + kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, kCMTimeZero, kCMVideoCodecType_H264, }; use objc2_foundation::{NSString, NSURL}; use tokio::sync::{mpsc, oneshot}; @@ -165,6 +169,29 @@ impl AVFMuxer { objc2_av_foundation::AVAssetWriter::assetWriterWithURL_fileType_error(&url, file_type)? }; + // AVAssetWriter writes the `colr` box from the format description it is given, not from + // the one the encoder produced: the encoded frames travel back through the caller as + // opaque buffers, so this muxer never sees the encoder's format description. Without + // these extensions the output carries no color information at all, which makes players + // guess at the color space. The values match the tags the encoder writes into the SPS. + let color_extensions = { + let keys: [&CFString; 3] = unsafe { + [ + kCMFormatDescriptionExtension_ColorPrimaries, + kCMFormatDescriptionExtension_TransferFunction, + kCMFormatDescriptionExtension_YCbCrMatrix, + ] + }; + let values: [&CFType; 3] = unsafe { + [ + kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionTransferFunction_ITU_R_709_2, + kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, + ] + }; + CFDictionary::from_slices(&keys, &values) + }; + let source_format_hint = unsafe { let mut format_desc: *const CMFormatDescription = std::ptr::null(); CMVideoFormatDescriptionCreate( @@ -172,7 +199,7 @@ impl AVFMuxer { kCMVideoCodecType_H264, video_options.width() as i32, video_options.height() as i32, - None, + Some(color_extensions.as_opaque()), NonNull::new(&mut format_desc).unwrap(), ) .to_result()?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs index 116a7e1c..4b72175b 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs @@ -8,13 +8,17 @@ use objc2_core_foundation::{ CFBoolean, CFDictionary, CFNumber, CFString, CFType, kCFBooleanFalse, kCFBooleanTrue, }; use objc2_core_media::{ - CMSampleBuffer, CMTime, kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, + CMSampleBuffer, CMTime, kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionTransferFunction_ITU_R_709_2, kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, + kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, }; use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelFormatType_32BGRA}; use objc2_video_toolbox::{ VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty, kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate, - kVTCompressionPropertyKey_RealTime, kVTInvalidSessionErr, + kVTCompressionPropertyKey_ColorPrimaries, kVTCompressionPropertyKey_RealTime, + kVTCompressionPropertyKey_TransferFunction, kVTCompressionPropertyKey_YCbCrMatrix, + kVTInvalidSessionErr, }; use tokio::sync::mpsc; use unienc_common::{ @@ -383,6 +387,36 @@ impl CompressionSession { } .to_result()?; + // Tag the stream as BT.709 limited range. Without these the encoder leaves the color + // information unspecified, so the muxed file carries no `colr` box and players have to + // guess. The values must match the conversion `VideoFrameBgra32::to_yuv420_planes` + // performs on the readback path, and the matrix additionally tells VideoToolbox which + // coefficients to use when it converts the BGRA pixel buffers of the blit path itself. + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_ColorPrimaries, + Some(kCMFormatDescriptionColorPrimaries_ITU_R_709_2 as &CFType), + ) + } + .to_result()?; + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_TransferFunction, + Some(kCMFormatDescriptionTransferFunction_ITU_R_709_2 as &CFType), + ) + } + .to_result()?; + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_YCbCrMatrix, + Some(kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2 as &CFType), + ) + } + .to_result()?; + Ok(CompressionSession { inner: session }) } } diff --git a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs index 44262da7..f302e65d 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs @@ -147,7 +147,24 @@ impl VideoFrameBgra32 { let mut u_data = vec![128u8; padded_uv_size]; // Neutral for U let mut v_data = vec![128u8; padded_uv_size]; // Neutral for V - // Convert ARGB to YUV for the original image area only + // Convert ARGB to YUV for the original image area only. + // + // BT.709 limited range ("studio swing"), 8-bit fixed point with a denominator of 256: + // + // Y = 16 + (219/255) * ( 0.2126 R + 0.7152 G + 0.0722 B ) + // Cb = 128 + (224/255) * ( B - Y ) / 1.8556 + // Cr = 128 + (224/255) * ( R - Y ) / 1.5748 + // + // Scaling those factors by 256 gives (46.74, 157.24, 15.87) for Y, (-25.76, -86.67, + // 112.43) for Cb and (112.43, -102.13, -10.30) for Cr. The Cb row is rounded to + // (-26, -86, 112) instead of to the nearest integers so that every row sums to the value + // the reference formula requires: 220 for Y, which maps white to 235 and black to 16, and + // 0 for Cb and Cr, which maps neutral colors to exactly 128. That also keeps both chroma + // rows inside the nominal 16..240 range at the primary extremes, as the BT.601 + // coefficients this replaces did. The results therefore always fit in u8 without clamping. + // + // The color tags written by each platform encoder must stay in sync with these + // coefficients. for y in 0..self.height { for x in 0..self.width { let bgra_idx = ((y * self.width + x) * 4) as usize; @@ -155,15 +172,15 @@ impl VideoFrameBgra32 { let g = data[bgra_idx + 1] as i32; let b = data[bgra_idx] as i32; - let y_val = (((66 * r + 129 * g + 25 * b + 128) >> 8) + 16) as u8; + let y_val = (((47 * r + 157 * g + 16 * b + 128) >> 8) + 16) as u8; let y_idx = (y * w + x) as usize; y_data[y_idx] = y_val; // Sample U and V for every 2x2 block (4:2:0 subsampling) if x % 2 == 0 && y % 2 == 0 { - let u_val = (((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128) as u8; - let v_val = (((112 * r - 94 * g - 18 * b + 128) >> 8) + 128) as u8; + let u_val = (((-26 * r - 86 * g + 112 * b + 128) >> 8) + 128) as u8; + let v_val = (((112 * r - 102 * g - 10 * b + 128) >> 8) + 128) as u8; let uv_idx = ((y / 2) * (w / 2) + (x / 2)) as usize; u_data[uv_idx] = u_val; diff --git a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs index 91c2f1a4..fcc8aa39 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs @@ -158,6 +158,21 @@ impl FFmpegVideoEncoder { &format!("{}", options.bitrate()), "-force_key_frames", "expr:gte(t,n_forced*1)", + // Convert to and tag BT.709 limited range. Tagging the frames makes the + // auto-inserted scale filter use the BT.709 matrix for the BGRA to YUV + // conversion (FFmpeg otherwise derives the matrix from the frame size, which + // yields BT.601 at the sizes this library records) and makes the encoder write + // the color information into the SPS VUI. The muxer stream-copies this + // elementary stream, so the VUI is what carries the tags into the MP4. + // + // The `-color_primaries` / `-color_trc` output options would be the more + // obvious spelling, but FFmpeg does not reliably forward them to the encoder: + // as of 8.0.1 only the matrix and the range reach the VUI that way, leaving + // the primaries and the transfer function unspecified. Setting them on the + // frames is honored by every encoder because it does not depend on the + // encoder wrapper reading them off the codec context. + "-vf", + "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv", ], ffmpeg::Destination::Stdout, )?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts index 1b431444..7eb9292e 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts @@ -195,7 +195,18 @@ window["unienc_webcodecs"] = { offset: 0, stride: options.width * 4 // BGRA = 4 bytes per pixel } - ] + ], + // Describe the source buffer as sRGB. WebCodecs has no way to ask the encoder for + // a particular output color space, so this is the only lever available: the user + // agent converts to the encoder's YUV space itself and tags the stream from what + // the source frame declares. Left unset it has to guess, which is what makes the + // output carry no color information today. + colorSpace: { + primaries: "bt709", + transfer: "iec61966-2-1", + matrix: "rgb", + fullRange: true + } }; const frame = new VideoFrame(data, init); encoder.encode(frame, { diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs index b33708f1..e68f1daf 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs @@ -16,6 +16,22 @@ pub struct MediaFoundationVideoEncoder { fps_hint: f64, } +/// Describes a media type as BT.709 limited range. +/// +/// On the input type this states what the NV12 buffers `VideoFrameBgra32::to_yuv420_planes` +/// produces actually contain, and on the output type it asks the encoder to record the same in the +/// H.264 VUI. Without it the color information stays unspecified all the way into the file and +/// players have to guess at the color space. +fn set_bt709_color_attributes(media_type: &IMFMediaType) -> Result<()> { + unsafe { + media_type.SetUINT32(&MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709.0 as u32)?; + media_type.SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709.0 as u32)?; + media_type.SetUINT32(&MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709.0 as u32)?; + media_type.SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235.0 as u32)?; + } + Ok(()) +} + impl MediaFoundationVideoEncoder { pub fn new(options: &V, runtime: &impl Runtime) -> Result { let input_type = unsafe { @@ -30,6 +46,7 @@ impl MediaFoundationVideoEncoder { )?; input_type.SetUINT64(&MF_MT_FRAME_RATE, ((options.fps_hint() as u64) << 32) + 1)?; + set_bt709_color_attributes(&input_type)?; input_type }; @@ -45,6 +62,7 @@ impl MediaFoundationVideoEncoder { )?; output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; output_type.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base.0 as u32)?; + set_bt709_color_attributes(&output_type)?; output_type }; From 06c3fafd75ca187682e997cfe3afdde8ac8c1aae Mon Sep 17 00:00:00 2001 From: ruccho Date: Fri, 2 Oct 2026 11:42:55 +0900 Subject: [PATCH 2/7] Test that encoded video declares BT.709 and is converted to it The end-to-end harness now reads the color description from both the sample entry's colr box and the VUI of the SPS in avcC, and requires each present one to state BT.709 primaries, transfer and matrix in limited range. A colr box is required on every target except the web, whose muxer (muxide) writes none; there the SPS VUI is required instead. The CPU conversion in VideoFrameBgra32::to_yuv420_planes gains unit tests that pin the limited-range end points, neutral chroma for grays, the BT.709 values of the primaries, and agreement with the floating-point BT.709 definition across the RGB cube. --- .../unienc/crates/unienc_common/src/lib.rs | 91 ++++++ .../unienc/crates/unienc_testkit/src/h264.rs | 300 ++++++++++++++++++ .../unienc/crates/unienc_testkit/src/lib.rs | 1 + .../unienc/crates/unienc_testkit/src/mp4.rs | 96 +++++- .../crates/unienc_testkit/src/verify.rs | 55 +++- 5 files changed, 539 insertions(+), 4 deletions(-) create mode 100644 InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs diff --git a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs index f302e65d..931d2660 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs @@ -292,4 +292,95 @@ mod tests { vec![0x34, 0x12, 0xfe, 0xff, 0x00, 0x80] ); } + + /// A 2x2 frame of a single color, which converts to exactly one sample on every plane. + fn solid_bgra_frame(r: u8, g: u8, b: u8) -> VideoFrameBgra32 { + VideoFrameBgra32 { + buffer: SharedBuffer::new_unmanaged([b, g, r, 255].repeat(4)), + width: 2, + height: 2, + } + } + + fn convert_solid(r: u8, g: u8, b: u8) -> (u8, u8, u8) { + let (y, u, v) = solid_bgra_frame(r, g, b).to_yuv420_planes(None).unwrap(); + (y[0], u[0], v[0]) + } + + /// BT.709 limited range, evaluated in floating point straight from the definition. + fn bt709_limited_reference(r: u8, g: u8, b: u8) -> (f64, f64, f64) { + let (r, g, b) = (r as f64 / 255.0, g as f64 / 255.0, b as f64 / 255.0); + let luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; + ( + 16.0 + 219.0 * luma, + 128.0 + 224.0 * (b - luma) / 1.8556, + 128.0 + 224.0 * (r - luma) / 1.5748, + ) + } + + #[test] + fn to_yuv420_planes_maps_white_and_black_to_the_limited_range_ends() { + assert_eq!(convert_solid(255, 255, 255), (235, 128, 128)); + assert_eq!(convert_solid(0, 0, 0), (16, 128, 128)); + } + + #[test] + fn to_yuv420_planes_keeps_grays_neutral() { + for level in 0..=255u8 { + let (_, u, v) = convert_solid(level, level, level); + assert_eq!((u, v), (128, 128), "gray level {level}"); + } + } + + #[test] + fn to_yuv420_planes_uses_bt709_rather_than_bt601() { + // The primaries are where the two matrices differ most. Pure red is Y=63 under BT.709 and + // Y=81 under BT.601, so this fails clearly if the BT.601 coefficients come back. + assert_eq!(convert_solid(255, 0, 0), (63, 102, 240)); + assert_eq!(convert_solid(0, 255, 0), (172, 42, 26)); + assert_eq!(convert_solid(0, 0, 255), (32, 240, 118)); + } + + #[test] + fn to_yuv420_planes_matches_the_bt709_definition_within_rounding() { + // The fixed-point coefficients are rounded to 1/256, and the Cb row is deliberately rounded + // away from the nearest integers to keep neutral colors at exactly 128 (see + // `to_yuv420_planes`). That costs up to about 1.15 steps on Cb for saturated greens. The + // BT.601 coefficients this replaced are off by up to 28 steps on Y, so the tolerance + // still tells the two matrices apart by a wide margin. + for r in (0..=255u8).step_by(5) { + for g in (0..=255u8).step_by(5) { + for b in (0..=255u8).step_by(5) { + let actual = convert_solid(r, g, b); + let expected = bt709_limited_reference(r, g, b); + for (component, actual, expected) in [ + ("Y", actual.0, expected.0), + ("Cb", actual.1, expected.1), + ("Cr", actual.2, expected.2), + ] { + assert!( + (actual as f64 - expected).abs() <= 1.5, + "{component} of rgb({r}, {g}, {b}) is {actual}, expected {expected:.2}" + ); + } + } + } + } + } + + #[test] + fn to_yuv420_planes_fills_padding_with_black() { + let frame = solid_bgra_frame(255, 255, 255); + let (y, u, v) = frame.to_yuv420_planes(Some((4, 4))).unwrap(); + + assert_eq!(y.len(), 16); + assert_eq!((u.len(), v.len()), (4, 4)); + for row in 0..4 { + for col in 0..4 { + let expected = if row < 2 && col < 2 { 235 } else { 16 }; + assert_eq!(y[row * 4 + col], expected, "luma at ({col}, {row})"); + } + } + assert!(u.iter().chain(&v).all(|&c| c == 128)); + } } diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs new file mode 100644 index 00000000..c7e7511d --- /dev/null +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/h264.rs @@ -0,0 +1,300 @@ +//! Just enough H.264 bitstream parsing to read the color description out of a +//! sequence parameter set. +//! +//! An MP4 can state a track's color space twice: in the container's `colr` box +//! and in the VUI of the SPS that the decoder itself reads. Muxers that write no +//! `colr` box leave the VUI as the only statement, and a player that trusts one +//! over the other renders a mismatch wrongly, so both are worth checking. + +use crate::mp4::ColorDescription; + +/// The `profile_idc` values whose SPS carries the chroma format, bit depths +/// and scaling matrices before the fields every profile shares. +const PROFILES_WITH_CHROMA_INFO: [u8; 12] = + [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134]; + +/// The value H.264 assigns to primaries, transfer and matrix when the VUI +/// signals a video type but no colour description. +const UNSPECIFIED: u16 = 2; + +/// Reads the color description from the VUI of the first SPS in an `avcC` +/// body. +/// +/// `Ok(None)` means the SPS carries no video signal type at all, i.e. it says +/// nothing about color. An SPS that does carry one but omits the colour +/// description reports every component as unspecified. +pub(crate) fn sps_color_from_avcc(avcc: &[u8]) -> Result, &'static str> { + // configurationVersion, profile, compatibility, level, lengthSizeMinusOne, + // then the SPS count in the low five bits. + let sps_count = avcc.get(5).ok_or("avcC is truncated")? & 0x1f; + if sps_count == 0 { + return Err("avcC carries no SPS"); + } + let length = u16::from_be_bytes( + avcc.get(6..8) + .ok_or("avcC is truncated")? + .try_into() + .unwrap(), + ) as usize; + let nal = avcc.get(8..8 + length).ok_or("avcC SPS is truncated")?; + // The first byte is the NAL unit header. + let rbsp = remove_emulation_prevention(nal.get(1..).ok_or("SPS is empty")?); + parse_sps_color(&rbsp).ok_or("SPS ends before its VUI does") +} + +/// Undoes the escaping that keeps `00 00 0x` start codes out of a NAL unit's +/// payload. +fn remove_emulation_prevention(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len()); + let mut zeros = 0; + for &byte in data { + if zeros >= 2 && byte == 3 { + zeros = 0; + continue; + } + zeros = if byte == 0 { zeros + 1 } else { 0 }; + out.push(byte); + } + out +} + +/// Walks the SPS (ITU-T H.264 7.3.2.1.1) up to the video signal type in its +/// VUI (E.1.1). `None` means the data ran out first. +fn parse_sps_color(rbsp: &[u8]) -> Option> { + let mut bits = BitReader::new(rbsp); + + let profile_idc = bits.read(8)? as u8; + bits.read(8)?; // constraint flags and reserved bits + bits.read(8)?; // level_idc + bits.read_ue()?; // seq_parameter_set_id + + if PROFILES_WITH_CHROMA_INFO.contains(&profile_idc) { + let chroma_format_idc = bits.read_ue()?; + if chroma_format_idc == 3 { + bits.read(1)?; // separate_colour_plane_flag + } + bits.read_ue()?; // bit_depth_luma_minus8 + bits.read_ue()?; // bit_depth_chroma_minus8 + bits.read(1)?; // qpprime_y_zero_transform_bypass_flag + if bits.read(1)? == 1 { + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for index in 0..lists { + if bits.read(1)? == 1 { + skip_scaling_list(&mut bits, if index < 6 { 16 } else { 64 })?; + } + } + } + } + + bits.read_ue()?; // log2_max_frame_num_minus4 + match bits.read_ue()? { + 0 => { + bits.read_ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } + 1 => { + bits.read(1)?; // delta_pic_order_always_zero_flag + bits.read_se()?; // offset_for_non_ref_pic + bits.read_se()?; // offset_for_top_to_bottom_field + for _ in 0..bits.read_ue()? { + bits.read_se()?; // offset_for_ref_frame + } + } + _ => {} + } + bits.read_ue()?; // max_num_ref_frames + bits.read(1)?; // gaps_in_frame_num_value_allowed_flag + bits.read_ue()?; // pic_width_in_mbs_minus1 + bits.read_ue()?; // pic_height_in_map_units_minus1 + if bits.read(1)? == 0 { + bits.read(1)?; // mb_adaptive_frame_field_flag, present unless frame_mbs_only + } + bits.read(1)?; // direct_8x8_inference_flag + if bits.read(1)? == 1 { + for _ in 0..4 { + bits.read_ue()?; // frame_crop_{left,right,top,bottom}_offset + } + } + + if bits.read(1)? == 0 { + return Some(None); // no VUI + } + if bits.read(1)? == 1 { + // aspect_ratio_idc, with an explicit ratio after Extended_SAR. + if bits.read(8)? == 255 { + bits.read(16)?; // sar_width + bits.read(16)?; // sar_height + } + } + if bits.read(1)? == 1 { + bits.read(1)?; // overscan_appropriate_flag + } + if bits.read(1)? == 0 { + return Some(None); // no video signal type + } + bits.read(3)?; // video_format + let full_range = bits.read(1)? == 1; + let (primaries, transfer, matrix) = if bits.read(1)? == 1 { + ( + bits.read(8)? as u16, + bits.read(8)? as u16, + bits.read(8)? as u16, + ) + } else { + (UNSPECIFIED, UNSPECIFIED, UNSPECIFIED) + }; + + Some(Some(ColorDescription { + primaries, + transfer, + matrix, + full_range: Some(full_range), + })) +} + +/// Skips a `scaling_list()` (7.3.2.1.1.1), whose length depends on its own +/// contents. +fn skip_scaling_list(bits: &mut BitReader<'_>, size: usize) -> Option<()> { + let mut last = 8i64; + let mut next = 8i64; + for _ in 0..size { + if next != 0 { + next = (last + bits.read_se()? + 256) % 256; + } + if next != 0 { + last = next; + } + } + Some(()) +} + +struct BitReader<'a> { + data: &'a [u8], + position: usize, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, position: 0 } + } + + /// Reads `count` bits, at most 32, most significant first. + fn read(&mut self, count: u32) -> Option { + let mut value = 0u32; + for _ in 0..count { + let byte = *self.data.get(self.position / 8)?; + let bit = (byte >> (7 - self.position % 8)) & 1; + value = (value << 1) | bit as u32; + self.position += 1; + } + Some(value) + } + + /// Reads an unsigned Exp-Golomb code. + fn read_ue(&mut self) -> Option { + let mut leading_zeros = 0; + while self.read(1)? == 0 { + leading_zeros += 1; + if leading_zeros > 31 { + return None; + } + } + Some((1u32 << leading_zeros) - 1 + self.read(leading_zeros)?) + } + + /// Reads a signed Exp-Golomb code. + fn read_se(&mut self) -> Option { + let code = self.read_ue()? as i64; + Some(if code % 2 == 1 { + (code + 1) / 2 + } else { + -(code / 2) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Wraps an SPS NAL unit in the smallest `avcC` that carries it. + fn avcc_with_sps(nal: &[u8]) -> Vec { + let mut avcc = vec![1, nal[1], nal[2], nal[3], 0xff, 0xe1]; + avcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + avcc.extend_from_slice(nal); + avcc + } + + // Every SPS below was written by libx264 (FFmpeg 8.0.1), with the VUI set by + // the `h264_metadata` bitstream filter where noted. + + #[test] + fn reads_bt709_limited_range_from_a_high_profile_sps() { + // 1280x720 High 3.1, `colour_primaries=1:transfer_characteristics=1: + // matrix_coefficients=1:video_full_range_flag=0`. + let nal = [ + 0x67, 0x64, 0x00, 0x1f, 0xac, 0xd9, 0x40, 0x50, 0x05, 0xbb, 0x01, 0x6a, 0x02, 0x02, + 0x02, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x19, 0x07, 0x8c, 0x18, 0xcb, + ]; + assert_eq!( + sps_color_from_avcc(&avcc_with_sps(&nal)), + Ok(Some(ColorDescription::BT709_LIMITED)) + ); + } + + #[test] + fn reads_other_color_spaces_and_full_range() { + // 640x480 High 3.0, `colour_primaries=6:transfer_characteristics=6: + // matrix_coefficients=6:video_full_range_flag=1`. + let nal = [ + 0x67, 0x64, 0x00, 0x1e, 0xac, 0xd9, 0x40, 0xa0, 0x3d, 0xb0, 0x16, 0xe0, 0xc0, 0xc0, + 0xc8, 0x00, 0x00, 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x01, 0x90, 0x78, 0xb1, 0x6c, + 0xb0, + ]; + assert_eq!( + sps_color_from_avcc(&avcc_with_sps(&nal)), + Ok(Some(ColorDescription { + primaries: 6, + transfer: 6, + matrix: 6, + full_range: Some(true), + })) + ); + } + + #[test] + fn reports_unspecified_components_for_a_signal_type_without_colour_description() { + // 320x240 Baseline 1.3, `video_format=5:video_full_range_flag=0`. + let nal = [ + 0x67, 0x42, 0xc0, 0x0d, 0xd9, 0x01, 0x41, 0xfb, 0x01, 0x68, 0x80, 0x00, 0x00, 0x03, + 0x00, 0x80, 0x00, 0x00, 0x19, 0x07, 0x8a, 0x15, 0x24, + ]; + assert_eq!( + sps_color_from_avcc(&avcc_with_sps(&nal)), + Ok(Some(ColorDescription { + primaries: UNSPECIFIED, + transfer: UNSPECIFIED, + matrix: UNSPECIFIED, + full_range: Some(false), + })) + ); + } + + #[test] + fn reports_no_color_for_a_vui_without_signal_type() { + // 320x240 Baseline 1.3 as libx264 writes it by default: a VUI with + // timing information only. + let nal = [ + 0x67, 0x42, 0xc0, 0x0d, 0xd9, 0x01, 0x41, 0xfb, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, + 0x10, 0x00, 0x00, 0x03, 0x03, 0x20, 0xf1, 0x42, 0xa4, 0x80, + ]; + assert_eq!(sps_color_from_avcc(&avcc_with_sps(&nal)), Ok(None)); + } + + #[test] + fn removes_emulation_prevention_bytes() { + assert_eq!( + remove_emulation_prevention(&[0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x03, 0x00]), + vec![0x00, 0x00, 0x01, 0x00, 0x00, 0x00] + ); + } +} diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs index d085968c..fa4c6cd8 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/lib.rs @@ -16,6 +16,7 @@ pub mod driver; pub mod e2e; +mod h264; pub mod mp4; pub mod options; pub mod pattern; diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs index 356b3085..52caf452 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/mp4.rs @@ -11,9 +11,19 @@ use std::fmt; #[derive(Debug)] pub enum Mp4Error { - Truncated { at: usize, need: usize }, + Truncated { + at: usize, + need: usize, + }, MissingBox(&'static str), - UnsupportedVersion { box_type: &'static str, version: u8 }, + UnsupportedVersion { + box_type: &'static str, + version: u8, + }, + Malformed { + what: &'static str, + reason: &'static str, + }, } impl fmt::Display for Mp4Error { @@ -26,6 +36,7 @@ impl fmt::Display for Mp4Error { Mp4Error::UnsupportedVersion { box_type, version } => { write!(f, "unsupported '{box_type}' version {version}") } + Mp4Error::Malformed { what, reason } => write!(f, "malformed {what}: {reason}"), } } } @@ -70,6 +81,12 @@ pub struct Track { /// the web it is negotiated with the browser at run time — so this reports /// what the file actually contains. pub avc_profile: Option, + /// Color description from the sample entry's `colr` box. `None` when the + /// box is absent or describes color with an ICC profile instead. + pub colr: Option, + /// Color description from the VUI of the SPS in `avcC`. `None` when the SPS + /// signals no video type, which leaves the color space to the player. + pub sps_color: Option, pub timescale: u32, /// Track duration in seconds, from `mdhd`. /// @@ -178,6 +195,8 @@ fn parse_track(trak: &[u8], movie_timescale: u32) -> Result { format: entry.format, has_decoder_config: entry.has_decoder_config, avc_profile: entry.avc_profile, + colr: entry.colr, + sps_color: entry.sps_color, timescale, duration: duration as f64 / timescale.max(1) as f64, width: entry.width, @@ -253,11 +272,76 @@ impl fmt::Display for AvcProfile { } } +/// A color space as ITU-T H.273 code points, which `colr` boxes and H.264 VUI +/// share. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ColorDescription { + pub primaries: u16, + pub transfer: u16, + pub matrix: u16, + /// `None` for a QuickTime `nclc` box, which has no range flag. + pub full_range: Option, +} + +impl ColorDescription { + /// BT.709 primaries, transfer and matrix in limited range, which is what + /// every backend is meant to produce. + pub const BT709_LIMITED: Self = Self { + primaries: 1, + transfer: 1, + matrix: 1, + full_range: Some(false), + }; + + /// True when this describes BT.709 in limited range. A QuickTime `nclc` + /// box states no range and is taken as limited, which is how QuickTime + /// defines it. + pub fn is_bt709_limited(&self) -> bool { + self.primaries == 1 + && self.transfer == 1 + && self.matrix == 1 + && self.full_range != Some(true) + } +} + +impl fmt::Display for ColorDescription { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "primaries {}, transfer {}, matrix {}, ", + self.primaries, self.transfer, self.matrix + )?; + match self.full_range { + Some(true) => write!(f, "full range"), + Some(false) => write!(f, "limited range"), + None => write!(f, "range unstated"), + } + } +} + +/// Reads an `nclx` (ISO/IEC 14496-12) or `nclc` (QuickTime) `colr` body. +fn parse_colr(colr: &[u8]) -> Result> { + let full_range = match &read_array::<4>(colr, 0)? { + b"nclx" => Some(read_array::<1>(colr, 10)?[0] & 0x80 != 0), + b"nclc" => None, + // An ICC profile does not reduce to code points. + _ => return Ok(None), + }; + Ok(Some(ColorDescription { + primaries: read_u16(colr, 4)?, + transfer: read_u16(colr, 6)?, + matrix: read_u16(colr, 8)?, + full_range, + })) +} + #[derive(Default)] struct SampleEntry { format: String, has_decoder_config: bool, avc_profile: Option, + colr: Option, + sps_color: Option, width: u32, height: u32, sample_rate: u32, @@ -317,7 +401,15 @@ impl SampleEntry { if let (Some(&profile), Some(&level)) = (body.get(1), body.get(3)) { parsed.avc_profile = Some(AvcProfile { profile, level }); } + parsed.sps_color = + crate::h264::sps_color_from_avcc(body).map_err(|reason| { + Mp4Error::Malformed { + what: "SPS", + reason, + } + })?; } + b"colr" => parsed.colr = parse_colr(body)?, b"esds" => parsed.has_decoder_config = true, _ => {} } diff --git a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs index d83fe7f0..4f0b85e6 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_testkit/src/verify.rs @@ -11,7 +11,7 @@ use std::fmt; use crate::e2e::{E2eConfig, E2eReport}; -use crate::mp4::{Mp4Summary, Track, TrackKind}; +use crate::mp4::{ColorDescription, Mp4Summary, Track, TrackKind}; /// An AAC frame always covers 1024 samples per channel. const AAC_SAMPLES_PER_FRAME: u64 = 1024; @@ -29,6 +29,11 @@ const DURATION_TOLERANCE_SECS: f64 = 0.25; /// Encoder priming shifts one track by a few milliseconds at most. const SYNC_TOLERANCE_SECS: f64 = 0.05; +/// Whether the muxer on this target writes a `colr` box. muxide, which the web +/// backend muxes with, writes none, so the SPS VUI is the only place the web +/// output can state its color space. +const MUXER_WRITES_COLR: bool = !cfg!(target_os = "emscripten"); + /// Everything that did not hold. #[derive(Debug)] pub struct VerifyError { @@ -206,6 +211,8 @@ fn verify_video_track( ) }); + verify_video_color(findings, track); + // A decoder joining at the start needs the first sample to be a keyframe. findings.check(track.is_sync_sample(1), || { "the first video sample is not a sync sample".to_string() @@ -239,6 +246,41 @@ fn verify_video_track( }); } +/// Every backend converts to BT.709 limited range, so that is what the file has +/// to declare. Without a declaration players guess, and they guess differently. +/// +/// The container's `colr` box and the SPS VUI are checked separately because +/// they come from different places: the box from the muxer's track format, the +/// VUI from the encoder. Players disagree on which one wins, so wherever both +/// are present they have to agree. +/// +/// This checks the declaration only. Whether the pixels were converted with the +/// matrix it names takes decoding, which this harness does not do. +fn verify_video_color(findings: &mut Findings, track: &Track) { + let expected = ColorDescription::BT709_LIMITED; + + match track.colr { + Some(colr) => findings.check(colr.is_bt709_limited(), || { + format!("the 'colr' box declares {colr}, expected {expected}") + }), + None => findings.check(!MUXER_WRITES_COLR, || { + "video track carries no 'colr' box, so players have to guess its color space" + .to_string() + }), + } + + match track.sps_color { + Some(sps) => findings.check(sps.is_bt709_limited(), || { + format!("the SPS VUI declares {sps}, expected {expected}") + }), + None => findings.check(MUXER_WRITES_COLR, || { + "the SPS VUI declares no color space, and this muxer writes no 'colr' box, \ + so nothing does" + .to_string() + }), + } +} + fn verify_audio_track( findings: &mut Findings, track: &Track, @@ -299,7 +341,7 @@ pub fn describe(summary: &Mp4Summary) -> String { for track in &summary.tracks { out.push_str(&match track.kind { TrackKind::Video => format!( - " video: {} {}x{} ({}), {} frames, start {:.3} s, {:.3} s\n", + " video: {} {}x{} ({}), {} frames, start {:.3} s, {:.3} s\n colr: {}; SPS: {}\n", track.format, track.width, track.height, @@ -310,6 +352,8 @@ pub fn describe(summary: &Mp4Summary) -> String { track.sample_count, track.start_time, track.duration, + describe_color(track.colr), + describe_color(track.sps_color), ), TrackKind::Audio => format!( " audio: {} {} Hz {} ch, {} frames, start {:.3} s, {:.3} s, decoder config: {}\n", @@ -326,3 +370,10 @@ pub fn describe(summary: &Mp4Summary) -> String { } out } + +fn describe_color(color: Option) -> String { + match color { + Some(color) => color.to_string(), + None => "none".to_string(), + } +} From 442318d42d10bcfb69a55edde528ad7ce9bec09d Mon Sep 17 00:00:00 2001 From: ruccho Date: Fri, 2 Oct 2026 11:48:58 +0900 Subject: [PATCH 3/7] Declare BT.709 limited range as the WebCodecs frame color space WebCodecs has no encoder option for the output color space. Chrome converts an RGB VideoFrame into the matrix and range its color space declares and copies all four fields into the SPS VUI. Declaring the BGRA buffer as sRGB, as before, made the stream claim the sRGB transfer and full range, unlike every other backend; Chrome does the same when no color space is given. Declaring BT.709 primaries, transfer and matrix in limited range instead yields that exact VUI. Verified on Chrome 154 on macOS with both the OpenH264 software encoder and the hardware encoder, including that the decoded pixels are limited range (white at Y=235). --- .../crates/unienc_webcodecs/src/js/library.ts | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts index 6b7d829e..093b4b6c 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts @@ -226,16 +226,25 @@ window["unienc_webcodecs"] = { stride: options.width * 4 // BGRA = 4 bytes per pixel } ], - // Describe the source buffer as sRGB. WebCodecs has no way to ask the encoder for - // a particular output color space, so this is the only lever available: the user - // agent converts to the encoder's YUV space itself and tags the stream from what - // the source frame declares. Left unset it has to guess, which is what makes the - // output carry no color information today. + // The YUV color space to encode in: BT.709 limited range, as on every other + // backend. WebCodecs has no encoder option for the output color space, so the + // frame's color space is the only lever. Chrome converts an RGB frame into the + // matrix and range declared here and copies all four fields into the SPS VUI. + // + // This deliberately does not describe the BGRA buffer itself, which would be + // `matrix: "rgb"` and `fullRange: true`. Chrome tags the stream with exactly that + // declaration, and it does so even when no color space is given at all: the + // output then claims full range and the sRGB transfer, unlike every other + // backend. + // + // Verified on Chrome 154 (macOS) with both the OpenH264 software encoder and the + // hardware encoder: the SPS carries primaries 1, transfer 1, matrix 1 in limited + // range, and the decoded pixels are limited range (white at Y=235). colorSpace: { primaries: "bt709", - transfer: "iec61966-2-1", - matrix: "rgb", - fullRange: true + transfer: "bt709", + matrix: "bt709", + fullRange: false } }; const frame = new VideoFrame(data, init); From e51c5547d7b8b5c569347ba0be930c551558096d Mon Sep 17 00:00:00 2001 From: ruccho Date: Fri, 2 Oct 2026 13:06:38 +0900 Subject: [PATCH 4/7] Convert to BT.709 I420 before handing frames to WebCodecs Handing the browser a BGRA frame leaves the YUV color space to it, and browsers disagree. Chrome 154 converts into the matrix and range the frame's color space names, while the Chrome on the CI runner, given the same declaration, converted to BT.601 and tagged the stream as such. The WebCodecs backend now converts with VideoFrameBgra32::to_i420, the same BT.709 limited-range conversion the Android and Windows readback paths use, and passes the browser I420 tagged accordingly. Dimensions are padded to even values for I420 and cropped by the visible rectangle. to_i420 writes the three planes into a single buffer, since a VideoFrame takes one buffer with per-plane offsets. Sharing the plane layout with to_yuv420_planes also fixes the chroma stride for odd widths, which was rounded down while the plane size was rounded up, so each chroma row after the first overwrote the end of the previous one. --- .../unienc/crates/unienc_common/src/lib.rs | 117 ++++++++++++++++-- .../crates/unienc_webcodecs/src/js/library.ts | 42 ++++--- .../crates/unienc_webcodecs/src/js/mod.rs | 33 +++-- .../crates/unienc_webcodecs/src/video/mod.rs | 19 ++- 4 files changed, 164 insertions(+), 47 deletions(-) diff --git a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs index 931d2660..270f6f89 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs @@ -134,18 +134,47 @@ impl VideoFrameBgra32 { &self, padded_size: Option<(u32, u32)>, ) -> Result<(Vec, Vec, Vec)> { - let data = self.buffer.data(); - let w = padded_size.map_or(self.width, |(w, _)| w); - let h = padded_size.map_or(self.height, |(_, h)| h); - let w_half = (w + 1) >> 1; - let h_half = (h + 1) >> 1; - let padded_y_size = (w * h) as usize; - let padded_uv_size = (w_half * h_half) as usize; + let layout = Yuv420Layout::new(self, padded_size); // Create padded YUV data arrays - let mut y_data = vec![16u8; padded_y_size]; // Black level for Y - let mut u_data = vec![128u8; padded_uv_size]; // Neutral for U - let mut v_data = vec![128u8; padded_uv_size]; // Neutral for V + let mut y_data = vec![16u8; layout.y_size()]; // Black level for Y + let mut u_data = vec![128u8; layout.uv_size()]; // Neutral for U + let mut v_data = vec![128u8; layout.uv_size()]; // Neutral for V + + self.write_yuv420(&layout, &mut y_data, &mut u_data, &mut v_data); + + Ok((y_data, u_data, v_data)) + } + + /// Converts to I420 in a single buffer: the Y plane, then U, then V, each tightly packed. + /// + /// The planes are laid out for `padded_size` as in [`Self::to_yuv420_planes`], so the luma + /// stride is the padded width and the chroma stride is half of it, rounded up. + pub fn to_i420(&self, padded_size: Option<(u32, u32)>) -> Result> { + let layout = Yuv420Layout::new(self, padded_size); + + let mut data = vec![128u8; layout.y_size() + 2 * layout.uv_size()]; + let (y_data, chroma) = data.split_at_mut(layout.y_size()); + let (u_data, v_data) = chroma.split_at_mut(layout.uv_size()); + y_data.fill(16); + + self.write_yuv420(&layout, y_data, u_data, v_data); + + Ok(data) + } + + /// Writes the converted image into planes laid out by `layout`. Samples outside the image + /// are left untouched, so the caller decides what the padding holds. + fn write_yuv420( + &self, + layout: &Yuv420Layout, + y_data: &mut [u8], + u_data: &mut [u8], + v_data: &mut [u8], + ) { + let data = self.buffer.data(); + let w = layout.width; + let w_half = layout.chroma_width(); // Convert ARGB to YUV for the original image area only. // @@ -182,14 +211,43 @@ impl VideoFrameBgra32 { let u_val = (((-26 * r - 86 * g + 112 * b + 128) >> 8) + 128) as u8; let v_val = (((112 * r - 102 * g - 10 * b + 128) >> 8) + 128) as u8; - let uv_idx = ((y / 2) * (w / 2) + (x / 2)) as usize; + let uv_idx = ((y / 2) * w_half + (x / 2)) as usize; u_data[uv_idx] = u_val; v_data[uv_idx] = v_val; } } } + } +} - Ok((y_data, u_data, v_data)) +/// Plane dimensions of a 4:2:0 image, padded or not. +struct Yuv420Layout { + width: u32, + height: u32, +} + +impl Yuv420Layout { + fn new(frame: &VideoFrameBgra32, padded_size: Option<(u32, u32)>) -> Self { + let (width, height) = padded_size.unwrap_or((frame.width, frame.height)); + Self { width, height } + } + + /// Chroma planes cover odd dimensions by rounding up, so the last column and row of an + /// odd-sized image still have a chroma sample. + fn chroma_width(&self) -> u32 { + self.width.div_ceil(2) + } + + fn chroma_height(&self) -> u32 { + self.height.div_ceil(2) + } + + fn y_size(&self) -> usize { + (self.width * self.height) as usize + } + + fn uv_size(&self) -> usize { + (self.chroma_width() * self.chroma_height()) as usize } } @@ -368,6 +426,41 @@ mod tests { } } + #[test] + fn to_i420_packs_the_same_planes_as_to_yuv420_planes() { + let frame = solid_bgra_frame(255, 0, 0); + for padded_size in [None, Some((4, 4))] { + let (y, u, v) = frame.to_yuv420_planes(padded_size).unwrap(); + assert_eq!( + frame.to_i420(padded_size).unwrap(), + [y, u, v].concat(), + "padded to {padded_size:?}" + ); + } + } + + #[test] + fn to_yuv420_planes_gives_the_last_column_of_an_odd_width_its_own_chroma() { + // Three pixels wide, each row red, red, blue, and two chroma rows tall. Each chroma row + // is two samples wide, so the blue column's sample sits at the end of its own row. With + // a stride of one, the second chroma row would start on top of it and overwrite it. + let mut pixels = Vec::new(); + for _ in 0..4 { + pixels.extend_from_slice(&[0, 0, 255, 255, 0, 0, 255, 255, 255, 0, 0, 255]); + } + let frame = VideoFrameBgra32 { + buffer: SharedBuffer::new_unmanaged(pixels), + width: 3, + height: 4, + }; + + let (y, u, v) = frame.to_yuv420_planes(None).unwrap(); + + assert_eq!(y, [63, 63, 32].repeat(4)); + assert_eq!(u, [102, 240].repeat(2)); + assert_eq!(v, [240, 118].repeat(2)); + } + #[test] fn to_yuv420_planes_fills_padding_with_black() { let frame = solid_bgra_frame(255, 255, 255); diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts index 093b4b6c..6071a237 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts @@ -144,6 +144,8 @@ window["unienc_webcodecs"] = { { width: number, height: number, + codedWidth: number, + codedHeight: number, timestamp: number, isKey: boolean }, @@ -212,34 +214,34 @@ window["unienc_webcodecs"] = { return encoder; }, encodeFrame: (encoder, data, options) => { + // The frame arrives as I420, already converted to BT.709 limited range by + // `VideoFrameBgra32::to_i420`, and padded to even dimensions: Y, then U, then V, each + // tightly packed. + const chromaWidth = options.codedWidth / 2; + const chromaHeight = options.codedHeight / 2; + const lumaSize = options.codedWidth * options.codedHeight; + const chromaSize = chromaWidth * chromaHeight; const init: VideoFrameBufferInit = { timestamp: options.timestamp * 1000 * 1000, - codedWidth: options.width, - codedHeight: options.height, + codedWidth: options.codedWidth, + codedHeight: options.codedHeight, visibleRect: {x: 0, y: 0, width: options.width, height: options.height}, displayWidth: options.width, displayHeight: options.height, - format: "BGRA", + format: "I420", layout: [ - { - offset: 0, - stride: options.width * 4 // BGRA = 4 bytes per pixel - } + {offset: 0, stride: options.codedWidth}, + {offset: lumaSize, stride: chromaWidth}, + {offset: lumaSize + chromaSize, stride: chromaWidth}, ], - // The YUV color space to encode in: BT.709 limited range, as on every other - // backend. WebCodecs has no encoder option for the output color space, so the - // frame's color space is the only lever. Chrome converts an RGB frame into the - // matrix and range declared here and copies all four fields into the SPS VUI. - // - // This deliberately does not describe the BGRA buffer itself, which would be - // `matrix: "rgb"` and `fullRange: true`. Chrome tags the stream with exactly that - // declaration, and it does so even when no color space is given at all: the - // output then claims full range and the sRGB transfer, unlike every other - // backend. + // What the planes hold, which the browser copies into the SPS VUI. // - // Verified on Chrome 154 (macOS) with both the OpenH264 software encoder and the - // hardware encoder: the SPS carries primaries 1, transfer 1, matrix 1 in limited - // range, and the decoded pixels are limited range (white at Y=235). + // The conversion happens on our side because handing the browser an RGB frame + // leaves the YUV color space to it, and browsers disagree. Chrome 154 converts + // into whatever matrix and range the RGB frame's color space names, while the + // Chrome on the CI runner, given the same declaration, converted to BT.601 and + // tagged the stream accordingly. With YUV input there is no conversion left for + // the browser to choose. colorSpace: { primaries: "bt709", transfer: "bt709", diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs index 73da1bfb..c3960273 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/mod.rs @@ -9,6 +9,17 @@ static LIBRARY: LazyLock = LazyLock::new(Library::new); struct Library; +/// A frame in the layout the JavaScript side reads: I420 with the Y, U and V planes packed one +/// after another, sized for `coded_width` x `coded_height`, of which the top-left `width` x +/// `height` is the picture. +pub struct I420Frame<'a> { + pub data: &'a [u8], + pub width: u32, + pub height: u32, + pub coded_width: u32, + pub coded_height: u32, +} + pub struct VideoEncoderHandle { id: i32, callback: *mut Box, @@ -32,13 +43,11 @@ impl VideoEncoderHandle { pub fn push_video_frame( &self, - data: &[u8], - width: u32, - height: u32, + frame: &I420Frame<'_>, timestamp: f64, is_key: bool, ) -> Result<(), JavaScriptError> { - LIBRARY.push_video_frame(self.id, data, width, height, timestamp, is_key) + LIBRARY.push_video_frame(self.id, frame, timestamp, is_key) } pub async fn flush(&self) -> Result<(), JavaScriptError> { @@ -249,9 +258,7 @@ impl Library { fn push_video_frame( &self, encoder_index: i32, - data: &[u8], - width: u32, - height: u32, + frame: &I420Frame<'_>, timestamp: f64, is_key: bool, ) -> Result<(), JavaScriptError> { @@ -262,13 +269,19 @@ impl Library { const dataLength = {data_length}; const width = {width}; const height = {height}; + const codedWidth = {coded_width}; + const codedHeight = {coded_height}; const timestamp = {timestamp}; const isKey = {is_key}; const dataArray = Module.HEAPU8.subarray(dataPtr, dataPtr + dataLength); - window.unienc_webcodecs.video.push(encoderIndex, dataArray, {{width, height, timestamp, isKey}}); + window.unienc_webcodecs.video.push(encoderIndex, dataArray, {{width, height, codedWidth, codedHeight, timestamp, isKey}}); ", - data_ptr = data.as_ptr() as usize, - data_length = data.len(), + data_ptr = frame.data.as_ptr() as usize, + data_length = frame.data.len(), + width = frame.width, + height = frame.height, + coded_width = frame.coded_width, + coded_height = frame.coded_height, timestamp = timestamp ); self.run_script(&script) diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs index ce9dc412..ec1cf671 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs @@ -1,4 +1,4 @@ -use crate::js::VideoEncoderHandle; +use crate::js::{I420Frame, VideoEncoderHandle}; use bincode::{Decode, Encode}; use futures::StreamExt; use futures::channel::mpsc; @@ -103,7 +103,12 @@ impl EncoderInput for WebCodecsVideoEncoderInput { let encoder_handle = self.encoder_handle.as_ref().unwrap(); - let pixels = &frame.buffer.data()[..frame.buffer.len()]; + // Converted here rather than by the browser, whose choice of YUV color space for an + // RGB frame varies between versions. I420 needs even dimensions, so odd ones are padded + // and the padding is cropped off again by the frame's visible rectangle. + let coded_width = frame.width.next_multiple_of(2); + let coded_height = frame.height.next_multiple_of(2); + let i420 = frame.to_i420(Some((coded_width, coded_height)))?; let since_prev_key = match self.prev_key_timestamp { Some(prev) => data.timestamp - prev, None => f64::INFINITY, @@ -113,9 +118,13 @@ impl EncoderInput for WebCodecsVideoEncoderInput { } encoder_handle .push_video_frame( - pixels, - frame.width, - frame.height, + &I420Frame { + data: &i420, + width: frame.width, + height: frame.height, + coded_width, + coded_height, + }, data.timestamp, since_prev_key >= 1.0, ) From d6a149bb90b428e477f9c31fd2a121e3ecc7824e Mon Sep 17 00:00:00 2001 From: ruccho Date: Fri, 2 Oct 2026 13:07:54 +0900 Subject: [PATCH 5/7] Add the colr box the Media Foundation MPEG-4 sink does not write The Windows e2e output carried no color description at all. The MPEG-4 file sink writes no colr box although the BT.709 attributes are set on the media type it is given, and the Microsoft software H.264 encoder writes no video signal type into the VUI; its documentation lists no color attribute or codec property among those it supports. After the sink has finalized the file, the muxer now appends a BT.709 limited-range nclx colr box to the video sample entry. The sink writes moov as the last top-level box, so moov is rewritten in place and no media data or chunk offset moves. A file laid out otherwise is left as the sink wrote it, with a warning. The color attributes on the encoder's media types are kept, since a hardware encoder may honor them. --- .../crates/unienc_windows_mf/src/mux/colr.rs | 376 ++++++++++++++++++ .../crates/unienc_windows_mf/src/mux/mod.rs | 25 ++ .../crates/unienc_windows_mf/src/video/mod.rs | 8 +- 3 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs new file mode 100644 index 00000000..56d18c79 --- /dev/null +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/colr.rs @@ -0,0 +1,376 @@ +//! Adds a `colr` box to the video sample entry of a finished MP4 file. +//! +//! The MPEG-4 file sink writes no `colr` box, whatever color attributes the media type it is +//! given carries: the e2e output on Windows had none although `MF_MT_VIDEO_PRIMARIES`, +//! `MF_MT_TRANSFER_FUNCTION`, `MF_MT_YUV_MATRIX` and `MF_MT_VIDEO_NOMINAL_RANGE` were all set on +//! the sink's video type, and the sink's documentation does not mention the box at all. The box is +//! therefore added after the sink has finalized the file. +//! +//! The sink writes `moov` after `mdat` as the last top-level box, so the box can be added by +//! rewriting `moov` in place, which moves no media data and invalidates no chunk offset. A file laid +//! out any other way is left untouched. + +use std::fs::OpenOptions; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::Path; + +/// A `colr` box of type `nclx` declaring BT.709 primaries, transfer and matrix in limited range +/// (ISO/IEC 14496-12, 12.1.5). +const BT709_LIMITED_COLR: [u8; 19] = [ + 0, 0, 0, 19, b'c', b'o', b'l', b'r', b'n', b'c', b'l', b'x', // + 0, 1, // colour_primaries + 0, 1, // transfer_characteristics + 0, 1, // matrix_coefficients + 0x00, // full_range_flag = 0, reserved = 0 +]; + +/// Offset of the child boxes within a visual sample entry's body: the sample entry header (8) +/// followed by the fields of `VisualSampleEntry` (70). +const VISUAL_SAMPLE_ENTRY_FIELDS: usize = 78; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum ColrOutcome { + Added, + AlreadyPresent, + /// The file was left untouched for the stated reason. + Skipped(&'static str), +} + +/// Adds a BT.709 limited range `colr` box to the first video track of the MP4 file at `path`. +pub(crate) fn add_bt709_colr(path: &Path) -> io::Result { + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + let file_len = file.metadata()?.len(); + + let mut at = 0u64; + let mut moov = None; + while at < file_len { + file.seek(SeekFrom::Start(at))?; + let mut header = [0u8; 16]; + let available = (file_len - at).min(16) as usize; + file.read_exact(&mut header[..available])?; + if available < 8 { + return Ok(ColrOutcome::Skipped("the file ends inside a box header")); + } + let size = match u32::from_be_bytes(header[0..4].try_into().unwrap()) { + 0 => file_len - at, + 1 if available == 16 => u64::from_be_bytes(header[8..16].try_into().unwrap()), + 1 => return Ok(ColrOutcome::Skipped("the file ends inside a box header")), + size => size as u64, + }; + if size < 8 || at + size > file_len { + return Ok(ColrOutcome::Skipped("a top-level box overruns the file")); + } + if &header[4..8] == b"moov" { + moov = Some((at, size)); + } + at += size; + } + + let Some((moov_at, moov_size)) = moov else { + return Ok(ColrOutcome::Skipped("the file has no moov box")); + }; + if moov_at + moov_size != file_len { + return Ok(ColrOutcome::Skipped("moov is not the last box in the file")); + } + + let mut moov = vec![0u8; moov_size as usize]; + file.seek(SeekFrom::Start(moov_at))?; + file.read_exact(&mut moov)?; + + let patched = match insert_bt709_colr(&moov) { + Ok(Some(patched)) => patched, + Ok(None) => return Ok(ColrOutcome::AlreadyPresent), + Err(reason) => return Ok(ColrOutcome::Skipped(reason)), + }; + + // The new moov is longer than the old one and starts at the same offset, so it overwrites the + // old one entirely and extends the file. + file.seek(SeekFrom::Start(moov_at))?; + file.write_all(&patched)?; + file.flush()?; + Ok(ColrOutcome::Added) +} + +/// One box found in a buffer: where it starts, how long its header is and its total size. +#[derive(Clone, Copy)] +struct BoxRef { + at: usize, + header: usize, + size: usize, + box_type: [u8; 4], +} + +impl BoxRef { + fn body(&self) -> (usize, usize) { + (self.at + self.header, self.at + self.size) + } +} + +fn read_box(data: &[u8], at: usize, end: usize) -> Option { + let header = data.get(at..at + 8)?; + let box_type: [u8; 4] = header[4..8].try_into().unwrap(); + let (header_len, size) = match u32::from_be_bytes(header[0..4].try_into().unwrap()) { + 0 => (8, end - at), + 1 => ( + 16, + u64::from_be_bytes(data.get(at + 8..at + 16)?.try_into().unwrap()) as usize, + ), + size => (8, size as usize), + }; + if size < header_len || at + size > end { + return None; + } + Some(BoxRef { + at, + header: header_len, + size, + box_type, + }) +} + +fn find_child(data: &[u8], (start, end): (usize, usize), wanted: &[u8; 4]) -> Option { + children(data, (start, end)).find(|found| &found.box_type == wanted) +} + +fn children(data: &[u8], (start, end): (usize, usize)) -> impl Iterator + '_ { + let mut at = start; + std::iter::from_fn(move || { + let found = read_box(data, at, end)?; + at += found.size; + Some(found) + }) +} + +/// Returns `moov` with a BT.709 limited range `colr` box appended to the first video track's +/// sample entry, or `None` when that entry already has a `colr` box. +fn insert_bt709_colr(moov: &[u8]) -> Result>, &'static str> { + let moov_box = read_box(moov, 0, moov.len()).ok_or("moov is malformed")?; + if &moov_box.box_type != b"moov" { + return Err("the buffer is not a moov box"); + } + + // Every box from moov down to the sample entry contains the insertion point, so each of their + // sizes grows by the size of the new box. + let mut ancestors = vec![moov_box]; + let trak = children(moov, moov_box.body()) + .filter(|found| &found.box_type == b"trak") + .find(|trak| is_video_track(moov, *trak)) + .ok_or("moov has no video track")?; + ancestors.push(trak); + let mut parent = trak; + for wanted in [b"mdia", b"minf", b"stbl", b"stsd"] { + parent = find_child(moov, parent.body(), wanted).ok_or("the video track is incomplete")?; + ancestors.push(parent); + } + + // stsd is a full box: version and flags (4) and the entry count (4) precede the entries. + let (stsd_start, stsd_end) = parent.body(); + let entry = read_box(moov, stsd_start + 8, stsd_end).ok_or("stsd has no sample entry")?; + if !matches!(&entry.box_type, b"avc1" | b"avc3") { + return Err("the video sample entry is not H.264"); + } + ancestors.push(entry); + + let (entry_start, entry_end) = entry.body(); + let entry_children = (entry_start + VISUAL_SAMPLE_ENTRY_FIELDS, entry_end); + if entry_children.0 > entry_end { + return Err("the video sample entry is truncated"); + } + if find_child(moov, entry_children, b"colr").is_some() { + return Ok(None); + } + + let insert_at = entry.at + entry.size; + let mut patched = Vec::with_capacity(moov.len() + BT709_LIMITED_COLR.len()); + patched.extend_from_slice(&moov[..insert_at]); + patched.extend_from_slice(&BT709_LIMITED_COLR); + patched.extend_from_slice(&moov[insert_at..]); + + // Every ancestor starts before the insertion point, so its offset is unchanged. + let grow = BT709_LIMITED_COLR.len(); + for ancestor in ancestors { + let field = &mut patched[ancestor.at..]; + match ancestor.header { + 16 => { + let size = u64::from_be_bytes(field[8..16].try_into().unwrap()) + grow as u64; + field[8..16].copy_from_slice(&size.to_be_bytes()); + } + _ => { + let size = u32::from_be_bytes(field[0..4].try_into().unwrap()); + // A size of 0 runs to the end of the enclosing space and stays valid as it is. + if size != 0 { + let size = size + .checked_add(grow as u32) + .ok_or("a box would exceed the 32-bit size limit")?; + field[0..4].copy_from_slice(&size.to_be_bytes()); + } + } + } + } + + Ok(Some(patched)) +} + +fn is_video_track(moov: &[u8], trak: BoxRef) -> bool { + find_child(moov, trak.body(), b"mdia") + .and_then(|mdia| find_child(moov, mdia.body(), b"hdlr")) + // hdlr: version and flags (4), pre_defined (4), handler_type (4) + .and_then(|hdlr| moov.get(hdlr.at + hdlr.header + 8..hdlr.at + hdlr.header + 12)) + .is_some_and(|handler| handler == b"vide") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn boxed(box_type: &[u8; 4], body: &[u8]) -> Vec { + let mut out = ((body.len() + 8) as u32).to_be_bytes().to_vec(); + out.extend_from_slice(box_type); + out.extend_from_slice(body); + out + } + + fn full_box(box_type: &[u8; 4], body: &[u8]) -> Vec { + boxed(box_type, &[&[0u8; 4][..], body].concat()) + } + + fn hdlr(handler: &[u8; 4]) -> Vec { + full_box(b"hdlr", &[&[0u8; 4][..], handler, &[0u8; 13]].concat()) + } + + fn track(handler: &[u8; 4], entry: Vec) -> Vec { + let stsd = full_box(b"stsd", &[&1u32.to_be_bytes()[..], &entry].concat()); + let stbl = boxed(b"stbl", &[stsd, full_box(b"stts", &[0; 4])].concat()); + let minf = boxed(b"minf", &stbl); + let mdia = boxed(b"mdia", &[hdlr(handler), minf].concat()); + boxed(b"trak", &mdia) + } + + fn avc1(extra_children: &[u8]) -> Vec { + let avcc = boxed(b"avcC", &[1, 0x42, 0xc0, 0x1f, 0xff, 0xe0, 0x00]); + boxed( + b"avc1", + &[ + &[0u8; VISUAL_SAMPLE_ENTRY_FIELDS][..], + &avcc, + extra_children, + ] + .concat(), + ) + } + + fn moov(tracks: &[Vec]) -> Vec { + boxed( + b"moov", + &[full_box(b"mvhd", &[0; 96]), tracks.concat()].concat(), + ) + } + + /// Checks that every box in `data[start..end]` exactly fills its parent, recursing into the + /// containers this module edits. + fn assert_well_formed(data: &[u8], (start, end): (usize, usize)) { + let mut at = start; + while at < end { + let found = read_box(data, at, end).expect("box overruns its parent"); + match &found.box_type { + b"moov" | b"trak" | b"mdia" | b"minf" | b"stbl" => { + assert_well_formed(data, found.body()) + } + b"stsd" => assert_well_formed(data, (found.body().0 + 8, found.body().1)), + b"avc1" => assert_well_formed( + data, + (found.body().0 + VISUAL_SAMPLE_ENTRY_FIELDS, found.body().1), + ), + _ => {} + } + at += found.size; + } + assert_eq!(at, end, "children do not fill their parent"); + } + + #[test] + fn appends_colr_to_the_video_sample_entry_and_grows_every_ancestor() { + let mp4a = boxed(b"mp4a", &[0; 28]); + let original = moov(&[track(b"soun", mp4a), track(b"vide", avc1(&[]))]); + + let patched = insert_bt709_colr(&original).unwrap().unwrap(); + + assert_eq!(patched.len(), original.len() + BT709_LIMITED_COLR.len()); + assert_well_formed(&patched, (0, patched.len())); + // The box lands at the end of the avc1 entry, which is the last thing before the video + // track's stts. + let stts_at = patched + .windows(4) + .rposition(|window| window == b"stts") + .unwrap() + - 4; + assert_eq!( + &patched[stts_at - BT709_LIMITED_COLR.len()..stts_at], + &BT709_LIMITED_COLR + ); + // Past moov's own size field, everything before the video track (mvhd and the audio + // track) is byte-for-byte unchanged. + let video_trak_at = patched + .windows(4) + .rposition(|window| window == b"trak") + .unwrap() + - 4; + assert_eq!(&patched[4..video_trak_at], &original[4..video_trak_at]); + } + + #[test] + fn leaves_an_existing_colr_box_alone() { + let original = moov(&[track(b"vide", avc1(&BT709_LIMITED_COLR))]); + assert_eq!(insert_bt709_colr(&original), Ok(None)); + } + + #[test] + fn refuses_a_file_without_a_video_track() { + let original = moov(&[track(b"soun", boxed(b"mp4a", &[0; 28]))]); + assert_eq!(insert_bt709_colr(&original), Err("moov has no video track")); + } + + #[test] + fn rewrites_a_trailing_moov_in_place() { + let path = std::env::temp_dir().join(format!("unienc-colr-{}.mp4", std::process::id())); + let mdat = boxed(b"mdat", &[0xaa; 64]); + let original_moov = moov(&[track(b"vide", avc1(&[]))]); + let file = [ + boxed(b"ftyp", b"isom\0\0\0\0"), + mdat.clone(), + original_moov.clone(), + ] + .concat(); + std::fs::write(&path, &file).unwrap(); + + assert_eq!(add_bt709_colr(&path).unwrap(), ColrOutcome::Added); + let written = std::fs::read(&path).unwrap(); + // ftyp and mdat are unchanged, so no chunk offset moved. + assert_eq!(&written[..16 + mdat.len()], &file[..16 + mdat.len()]); + assert_eq!(written.len(), file.len() + BT709_LIMITED_COLR.len()); + assert_well_formed(&written, (0, written.len())); + + assert_eq!(add_bt709_colr(&path).unwrap(), ColrOutcome::AlreadyPresent); + std::fs::remove_file(&path).unwrap(); + } + + #[test] + fn leaves_a_file_whose_moov_precedes_mdat_untouched() { + let path = + std::env::temp_dir().join(format!("unienc-colr-front-{}.mp4", std::process::id())); + let file = [ + boxed(b"ftyp", b"isom\0\0\0\0"), + moov(&[track(b"vide", avc1(&[]))]), + boxed(b"mdat", &[0xaa; 64]), + ] + .concat(); + std::fs::write(&path, &file).unwrap(); + + assert_eq!( + add_bt709_colr(&path).unwrap(), + ColrOutcome::Skipped("moov is not the last box in the file") + ); + assert_eq!(std::fs::read(&path).unwrap(), file); + std::fs::remove_file(&path).unwrap(); + } +} diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs index 72c40691..c14d8123 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs @@ -17,6 +17,8 @@ use crate::mft::MediaEventGeneratorCustom; use crate::video::VideoEncodedData; use windows::core::Interface; +mod colr; + enum LazyStream { None { tx: oneshot::Sender>>, @@ -86,6 +88,7 @@ impl MediaFoundationMuxer { let (audio_stream_tx, audio_stream_rx) = oneshot::channel::>(); let runtime_clone = runtime.clone(); + let output_path = output_path.to_path_buf(); runtime.spawn_ret(async move { let result: Result<()> = async move { @@ -171,10 +174,15 @@ impl MediaFoundationMuxer { let _ = unsafe { sink.Shutdown() }; } + // The sink has written everything it will by now. Closing the byte stream releases + // the file so that it can be reopened below. + let _ = unsafe { file.Close() }; + result } .await; + let result = result.and_then(|()| add_color_description(&output_path)); let _ = finish_tx.send(result); }); @@ -195,6 +203,23 @@ impl MediaFoundationMuxer { } } +/// Adds the `colr` box the MPEG-4 file sink does not write. See [`colr`] for why it is needed. +/// +/// A file that cannot be patched safely is left as the sink wrote it, which is still playable; +/// only an I/O error, which may have left the file half-written, fails the mux. +fn add_color_description(path: &Path) -> Result<()> { + match colr::add_bt709_colr(path) { + Ok(colr::ColrOutcome::Added | colr::ColrOutcome::AlreadyPresent) => Ok(()), + Ok(colr::ColrOutcome::Skipped(reason)) => { + log::warn!("Muxed file left without a colr box: {reason}"); + Ok(()) + } + Err(err) => Err(WindowsError::Other(format!( + "Failed to add a colr box to the muxed file: {err}" + ))), + } +} + struct Stream { sample_tx: mpsc::Sender>, } diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs index e68f1daf..c18c5590 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs @@ -20,8 +20,12 @@ pub struct MediaFoundationVideoEncoder { /// /// On the input type this states what the NV12 buffers `VideoFrameBgra32::to_yuv420_planes` /// produces actually contain, and on the output type it asks the encoder to record the same in the -/// H.264 VUI. Without it the color information stays unspecified all the way into the file and -/// players have to guess at the color space. +/// H.264 VUI. +/// +/// Not every encoder honors the request. The Microsoft software H.264 encoder writes no video +/// signal type into the VUI regardless, and its documentation lists no color attribute or codec +/// property among those it supports. The muxer therefore states the color space in a `colr` box of +/// its own (see `mux::colr`) rather than relying on the encoder. fn set_bt709_color_attributes(media_type: &IMFMediaType) -> Result<()> { unsafe { media_type.SetUINT32(&MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709.0 as u32)?; From 545b969cbe5b4dab7c968c2e35d06a8797e2a0b8 Mon Sep 17 00:00:00 2001 From: ruccho Date: Mon, 19 Oct 2026 19:07:06 +0900 Subject: [PATCH 6/7] reformat --- .../unienc/crates/unienc_apple_vt/src/video/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs index b8f3ba52..d83fdb59 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs @@ -16,8 +16,8 @@ use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelForm use objc2_video_toolbox::{ VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty, kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate, - kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, - kVTCompressionPropertyKey_ColorPrimaries, kVTCompressionPropertyKey_RealTime, + kVTCompressionPropertyKey_ColorPrimaries, + kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, kVTCompressionPropertyKey_RealTime, kVTCompressionPropertyKey_TransferFunction, kVTCompressionPropertyKey_YCbCrMatrix, kVTInvalidSessionErr, }; From a232ee23c3a754396402b1236da23fd3c0aba4f3 Mon Sep 17 00:00:00 2001 From: ruccho Date: Mon, 19 Oct 2026 19:11:51 +0900 Subject: [PATCH 7/7] fix compilation error --- .../unienc/crates/unienc_windows_mf/src/mux/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs index 6d378023..3c470031 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mux/mod.rs @@ -1,3 +1,4 @@ +use crate::error::{Result, WindowsError}; use std::path::Path; use tokio::sync::mpsc; use tokio::sync::oneshot;