Tag encoded video as BT.709 and convert to match

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs

@@ -24,6 +24,9 @@ pub mod format_keys {
     pub const KEY_LEVEL: &str = "level";
     pub const KEY_PRIORITY: &str = "priority";
     pub const KEY_OPERATING_RATE: &str = "operating-rate";
+    pub const KEY_COLOR_STANDARD: &str = "color-standard";
+    pub const KEY_COLOR_TRANSFER: &str = "color-transfer";
+    pub const KEY_COLOR_RANGE: &str = "color-range";
 
     // Audio keys
     pub const KEY_SAMPLE_RATE: &str = "sample-rate";
@@ -33,6 +36,13 @@ pub mod format_keys {
 
 pub const COLOR_FORMAT_SURFACE: jint = 0x7F000789;
 pub const COLOR_FORMAT_YUV420_FLEXIBLE: jint = 0x7F420888;
+
+/// `MediaFormat.COLOR_STANDARD_BT709`, `COLOR_TRANSFER_SDR_VIDEO` and `COLOR_RANGE_LIMITED`.
+/// All three were introduced in API 24, below the minimum this crate targets, so setting them
+/// needs no version guard.
+pub const COLOR_STANDARD_BT709: jint = 1;
+pub const COLOR_TRANSFER_SDR_VIDEO: jint = 3;
+pub const COLOR_RANGE_LIMITED: jint = 2;
 pub const AAC_OBJECT_TYPE_AAC_LC: jint = 2;
 
 pub const MUXER_OUTPUT_FORMAT_MPEG_4: jint = 0;

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs

@@ -5,7 +5,10 @@ use unienc_common::{CompletionHandle, Muxer, MuxerInput};
 
 use crate::bindings;
 use crate::common::*;
-use crate::config::MUXER_OUTPUT_FORMAT_MPEG_4;
+use crate::config::{
+    COLOR_RANGE_LIMITED, COLOR_STANDARD_BT709, COLOR_TRANSFER_SDR_VIDEO,
+    MUXER_OUTPUT_FORMAT_MPEG_4, format_keys,
+};
 use crate::error::{AndroidError, Result};
 use crate::java::*;
 
@@ -128,6 +131,23 @@ async fn push(
                     "height".to_string(),
                     crate::common::MediaFormatValue::Integer(height as i32),
                 );
+
+                // The track format is what MediaMuxer writes the `colr` box from. These keys are
+                // set on the encoder and most codecs echo them back in their output format, but
+                // that is not guaranteed across vendors, so state them here as well. The values
+                // must match the ones `create_video_format` configures the encoder with.
+                map.insert(
+                    format_keys::KEY_COLOR_STANDARD.to_string(),
+                    crate::common::MediaFormatValue::Integer(COLOR_STANDARD_BT709),
+                );
+                map.insert(
+                    format_keys::KEY_COLOR_TRANSFER.to_string(),
+                    crate::common::MediaFormatValue::Integer(COLOR_TRANSFER_SDR_VIDEO),
+                );
+                map.insert(
+                    format_keys::KEY_COLOR_RANGE.to_string(),
+                    crate::common::MediaFormatValue::Integer(COLOR_RANGE_LIMITED),
+                );
             }
 
             let mut shared_state_lock = shared_state.write().await;

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs

@@ -447,5 +447,19 @@ fn create_video_format_raw(
     set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?;
     set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?;
 
+    // Tag the stream as BT.709 limited range. Without these the encoder leaves the color
+    // information unspecified, so the muxed file gives players nothing to go on. The values match
+    // the conversion `VideoFrameBgra32::to_yuv420_planes` performs on the buffer input path, and
+    // on the surface input path they additionally tell MediaCodec which coefficients to use when
+    // it converts the frames the Vulkan preprocessor hands it.
+    set_format_integer(env, &format_obj, KEY_COLOR_STANDARD, COLOR_STANDARD_BT709)?;
+    set_format_integer(
+        env,
+        &format_obj,
+        KEY_COLOR_TRANSFER,
+        COLOR_TRANSFER_SDR_VIDEO,
+    )?;
+    set_format_integer(env, &format_obj, KEY_COLOR_RANGE, COLOR_RANGE_LIMITED)?;
+
     SafeGlobalRef::new(env, format_obj)
 }

InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs

@@ -16,10 +16,14 @@ use objc2_av_foundation::{
 use objc2_core_audio_types::{
     AudioStreamBasicDescription, AudioStreamPacketDescription, MPEG4ObjectID, kAudioFormatMPEG4AAC,
 };
+use objc2_core_foundation::{CFDictionary, CFString, CFType};
 use objc2_core_media::{
     CMAudioFormatDescriptionCreate, CMAudioSampleBufferCreateReadyWithPacketDescriptions,
     CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMTime, CMVideoFormatDescriptionCreate,
-    kCMBlockBufferAssureMemoryNowFlag, kCMTimeZero, kCMVideoCodecType_H264,
+    kCMBlockBufferAssureMemoryNowFlag, kCMFormatDescriptionColorPrimaries_ITU_R_709_2,
+    kCMFormatDescriptionExtension_ColorPrimaries, kCMFormatDescriptionExtension_TransferFunction,
+    kCMFormatDescriptionExtension_YCbCrMatrix, kCMFormatDescriptionTransferFunction_ITU_R_709_2,
+    kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, kCMTimeZero, kCMVideoCodecType_H264,
 };
 use objc2_foundation::{NSString, NSURL};
 use tokio::sync::{mpsc, oneshot};
@@ -165,14 +169,37 @@ impl AVFMuxer {
             objc2_av_foundation::AVAssetWriter::assetWriterWithURL_fileType_error(&url, file_type)?
         };
 
+        // AVAssetWriter writes the `colr` box from the format description it is given, not from
+        // the one the encoder produced: the encoded frames travel back through the caller as
+        // opaque buffers, so this muxer never sees the encoder's format description. Without
+        // these extensions the output carries no color information at all, which makes players
+        // guess at the color space. The values match the tags the encoder writes into the SPS.
+        let color_extensions = {
+            let keys: [&CFString; 3] = unsafe {
+                [
+                    kCMFormatDescriptionExtension_ColorPrimaries,
+                    kCMFormatDescriptionExtension_TransferFunction,
+                    kCMFormatDescriptionExtension_YCbCrMatrix,
+                ]
+            };
+            let values: [&CFType; 3] = unsafe {
+                [
+                    kCMFormatDescriptionColorPrimaries_ITU_R_709_2,
+                    kCMFormatDescriptionTransferFunction_ITU_R_709_2,
+                    kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2,
+                ]
+            };
+            CFDictionary::from_slices(&keys, &values)
+        };
+
         let source_format_hint = unsafe {
             let mut format_desc: *const CMFormatDescription = std::ptr::null();
             CMVideoFormatDescriptionCreate(
                 allocator::default(),
                 kCMVideoCodecType_H264,
                 video_options.width() as i32,
                 video_options.height() as i32,
-                None,
+                Some(color_extensions.as_opaque()),
                 NonNull::new(&mut format_desc).unwrap(),
             )
             .to_result()?;

InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs

@@ -8,13 +8,17 @@ use objc2_core_foundation::{
     CFBoolean, CFDictionary, CFNumber, CFString, CFType, kCFBooleanFalse, kCFBooleanTrue,
 };
 use objc2_core_media::{
-    CMSampleBuffer, CMTime, kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264,
+    CMSampleBuffer, CMTime, kCMFormatDescriptionColorPrimaries_ITU_R_709_2,
+    kCMFormatDescriptionTransferFunction_ITU_R_709_2, kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2,
+    kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264,
 };
 use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelFormatType_32BGRA};
 use objc2_video_toolbox::{
     VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty,
     kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate,
+    kVTCompressionPropertyKey_ColorPrimaries,
     kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, kVTCompressionPropertyKey_RealTime,
+    kVTCompressionPropertyKey_TransferFunction, kVTCompressionPropertyKey_YCbCrMatrix,
     kVTInvalidSessionErr,
 };
 use tokio::sync::mpsc;
@@ -398,6 +402,36 @@ impl CompressionSession {
         }
         .to_result()?;
 
+        // Tag the stream as BT.709 limited range. Without these the encoder leaves the color
+        // information unspecified, so the muxed file carries no `colr` box and players have to
+        // guess. The values must match the conversion `VideoFrameBgra32::to_yuv420_planes`
+        // performs on the readback path, and the matrix additionally tells VideoToolbox which
+        // coefficients to use when it converts the BGRA pixel buffers of the blit path itself.
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_ColorPrimaries,
+                Some(kCMFormatDescriptionColorPrimaries_ITU_R_709_2 as &CFType),
+            )
+        }
+        .to_result()?;
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_TransferFunction,
+                Some(kCMFormatDescriptionTransferFunction_ITU_R_709_2 as &CFType),
+            )
+        }
+        .to_result()?;
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_YCbCrMatrix,
+                Some(kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2 as &CFType),
+            )
+        }
+        .to_result()?;
+
         Ok(CompressionSession { inner: session })
     }
 }