From 088dc4c2cc5867fc5c1375dea2321aa38bd63599 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 07:45:56 +0100 Subject: [PATCH 1/9] refactor(encoder): make internal modules private and remove dead code --- CHANGELOG.md | 6 + src/decoder/mod.rs | 2 +- src/encoder/bitwriter.rs | 277 --------------- src/encoder/codec.rs | 8 +- src/encoder/dpb/entry.rs | 40 +-- src/encoder/dpb/h264.rs | 310 +---------------- src/encoder/dpb/h265.rs | 528 +--------------------------- src/encoder/dpb/mod.rs | 68 +--- src/encoder/dpb/reference_lists.rs | 536 ----------------------------- src/encoder/dpb/types.rs | 158 +-------- src/encoder/gop.rs | 104 +----- src/encoder/h264/init.rs | 7 - src/encoder/h264/mod.rs | 14 +- src/encoder/h264/record.rs | 90 +---- src/encoder/h264/session_params.rs | 8 +- src/encoder/h265/init.rs | 6 - src/encoder/h265/mod.rs | 23 +- src/encoder/h265/record.rs | 70 +--- src/encoder/mod.rs | 23 +- src/encoder/reorder.rs | 328 ------------------ 20 files changed, 65 insertions(+), 2541 deletions(-) delete mode 100644 src/encoder/bitwriter.rs delete mode 100644 src/encoder/dpb/reference_lists.rs delete mode 100644 src/encoder/reorder.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e5e1f8e..89ef403d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `wall_latency_ns` starts before the encode submit. It started after it, so a submit that blocked in the driver made it shorter than the GPU time. +### Removed + +- The encoder's internal modules are no longer public: + `encoder::{av1, bitwriter, dpb, gop, h264, h265, reorder, resources}` and + `decoder::h264`. Nothing in them was needed to encode or decode. + ## [0.10.0] - 19-10-2026 ### Added diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 342728f7..cbf0f803 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -44,7 +44,7 @@ pub(crate) mod common; /// Frame ownership: pool images, DPB slot pins, display-order reordering. pub(crate) mod frames; /// H.264 decoding. -pub mod h264; +pub(crate) mod h264; pub(crate) mod pipeline; /// Readback and copies, all of which run on the transfer queue. pub(crate) mod transfer; diff --git a/src/encoder/bitwriter.rs b/src/encoder/bitwriter.rs deleted file mode 100644 index 00ded39d..00000000 --- a/src/encoder/bitwriter.rs +++ /dev/null @@ -1,277 +0,0 @@ -//! Shared bit writer for video codec utilities (used by H.264 and H.265). -/// Small, efficient bit writer used to assemble RBSP and other bitstreams. -#[derive(Default)] -pub struct BitWriter { - data: Vec, - current_byte: u8, - bit_position: u8, // number of bits currently written into current_byte (0-7) -} - -impl BitWriter { - pub fn new() -> Self { - Self { - data: Vec::new(), - current_byte: 0, - bit_position: 0, - } - } - - pub fn write_bits(&mut self, mut value: u64, mut num_bits: u8) { - if num_bits == 0 { - return; - } - - while num_bits > 0 { - let space = 8 - self.bit_position; - let take = std::cmp::min(space, num_bits); - - // shift value so the bits we need are in the high positions - let shift = num_bits - take; - let bits = ((value >> shift) & ((1u64 << take) - 1)) as u8; - - // Avoid shifting a u8 by >= 8 which panics. If we're taking a full. - // byte (take == 8) we can assign the bits directly; otherwise - // shift the partial byte left by `take` and OR-in the new bits. - if take == 8 { - self.current_byte = bits; - } else { - self.current_byte = (self.current_byte << take) | bits; - } - self.bit_position += take; - num_bits -= take; - if shift > 0 { - value &= (1u64 << shift) - 1; - } else { - value = 0; - } - - if self.bit_position == 8 { - self.data.push(self.current_byte); - self.current_byte = 0; - self.bit_position = 0; - } - } - } - - pub fn write_ue(&mut self, mut value: u32) { - value += 1; - let num_bits = 32u8 - value.leading_zeros() as u8; - let leading_zeros = num_bits - 1; - - // write leading zeros - for _ in 0..leading_zeros { - self.write_bits(0, 1); - } - - // write value bits - self.write_bits(value as u64, num_bits); - } - - pub fn write_se(&mut self, value: i32) { - let code = if value <= 0 { - (-2 * value) as u32 - } else { - (2 * value - 1) as u32 - }; - self.write_ue(code); - } - - pub fn rbsp_trailing_bits(&mut self) { - // write rbsp_stop_one_bit = 1 - self.write_bits(1, 1); - // pad with zeros to byte-align - if self.bit_position != 0 { - let pad = 8 - self.bit_position; - self.write_bits(0, pad); - } - } - - pub fn finish(mut self) -> Vec { - if self.bit_position != 0 { - // left-align the current partial byte - self.current_byte <<= 8 - self.bit_position; - self.data.push(self.current_byte); - } - self.data - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_write_bits_single_byte() { - let mut writer = BitWriter::new(); - writer.write_bits(0b10101010, 8); - assert_eq!(writer.finish(), vec![0b10101010]); - } - - #[test] - fn test_write_bits_partial_byte() { - let mut writer = BitWriter::new(); - writer.write_bits(0b101, 3); - // Partial byte gets left-aligned on finish: 101 -> 10100000 - assert_eq!(writer.finish(), vec![0b10100000]); - } - - #[test] - fn test_write_bits_across_bytes() { - let mut writer = BitWriter::new(); - writer.write_bits(0b1111, 4); - writer.write_bits(0b00001111, 8); - // First 4 bits: 1111, next 8 bits: 00001111. - // Result: 11110000 11110000 (after left-align) - assert_eq!(writer.finish(), vec![0b11110000, 0b11110000]); - } - - #[test] - fn test_write_bits_zero() { - let mut writer = BitWriter::new(); - writer.write_bits(0, 0); // Should be a no-op - assert_eq!(writer.finish(), vec![]); - } - - #[test] - fn test_write_bits_multiple_bytes() { - let mut writer = BitWriter::new(); - writer.write_bits(0xABCD, 16); - assert_eq!(writer.finish(), vec![0xAB, 0xCD]); - } - - #[test] - fn test_write_ue_zero() { - // ue(0) = 1 (single bit '1') - let mut writer = BitWriter::new(); - writer.write_ue(0); - // 0 -> code = 1 -> binary '1' -> 1 bit - assert_eq!(writer.finish(), vec![0b10000000]); - } - - #[test] - fn test_write_ue_one() { - // ue(1) = 010 (leading 0, then 10) - let mut writer = BitWriter::new(); - writer.write_ue(1); - // 1 -> code = 2 -> binary '10' -> 0 + 10 = 010 (3 bits) - assert_eq!(writer.finish(), vec![0b01000000]); - } - - #[test] - fn test_write_ue_two() { - // ue(2) = 011 (leading 0, then 11) - let mut writer = BitWriter::new(); - writer.write_ue(2); - // 2 -> code = 3 -> binary '11' -> 0 + 11 = 011 (3 bits) - assert_eq!(writer.finish(), vec![0b01100000]); - } - - #[test] - fn test_write_ue_larger_values() { - // ue(3) = 00100 (2 leading zeros, then 100) - let mut writer = BitWriter::new(); - writer.write_ue(3); - // 3 -> code = 4 -> binary '100' -> 00 + 100 = 00100 (5 bits) - assert_eq!(writer.finish(), vec![0b00100000]); - - // ue(6) = 00111 (2 leading zeros, then 111) - let mut writer = BitWriter::new(); - writer.write_ue(6); - // 6 -> code = 7 -> binary '111' -> 00 + 111 = 00111 (5 bits) - assert_eq!(writer.finish(), vec![0b00111000]); - - // ue(7) = 0001000 (3 leading zeros, then 1000) - let mut writer = BitWriter::new(); - writer.write_ue(7); - // 7 -> code = 8 -> binary '1000' -> 000 + 1000 = 0001000 (7 bits) - assert_eq!(writer.finish(), vec![0b00010000]); - } - - #[test] - fn test_write_se_zero() { - // se(0) = ue(0) = 1 - let mut writer = BitWriter::new(); - writer.write_se(0); - assert_eq!(writer.finish(), vec![0b10000000]); - } - - #[test] - fn test_write_se_positive() { - // se(1) = ue(1) = 010 - let mut writer = BitWriter::new(); - writer.write_se(1); - // 1 -> code = 2*1 - 1 = 1 -> ue(1) = 010 - assert_eq!(writer.finish(), vec![0b01000000]); - - // se(2) = ue(3) = 00100 - let mut writer = BitWriter::new(); - writer.write_se(2); - // 2 -> code = 2*2 - 1 = 3 -> ue(3) = 00100 - assert_eq!(writer.finish(), vec![0b00100000]); - } - - #[test] - fn test_write_se_negative() { - // se(-1) = ue(2) = 011 - let mut writer = BitWriter::new(); - writer.write_se(-1); - // -1 -> code = -2*(-1) = 2 -> ue(2) = 011 - assert_eq!(writer.finish(), vec![0b01100000]); - - // se(-2) = ue(4) = 00101 - let mut writer = BitWriter::new(); - writer.write_se(-2); - // -2 -> code = -2*(-2) = 4 -> ue(4) = 00101 - assert_eq!(writer.finish(), vec![0b00101000]); - } - - #[test] - fn test_rbsp_trailing_bits_aligned() { - let mut writer = BitWriter::new(); - writer.write_bits(0xFF, 8); // Already byte-aligned - writer.rbsp_trailing_bits(); - // Writes '1' then pads to next byte. - assert_eq!(writer.finish(), vec![0xFF, 0b10000000]); - } - - #[test] - fn test_rbsp_trailing_bits_unaligned() { - let mut writer = BitWriter::new(); - writer.write_bits(0b111, 3); // 3 bits written - writer.rbsp_trailing_bits(); - // Current: 111xxxxx, writes 1 -> 1111xxxx, then pad with 4 zeros -> 11110000 - assert_eq!(writer.finish(), vec![0b11110000]); - } - - #[test] - fn test_rbsp_trailing_bits_almost_aligned() { - let mut writer = BitWriter::new(); - writer.write_bits(0b1111111, 7); // 7 bits written - writer.rbsp_trailing_bits(); - // Current: 1111111x, writes 1 -> 11111111, byte complete, no padding needed - assert_eq!(writer.finish(), vec![0b11111111]); - } - - #[test] - fn test_complex_bitstream() { - // Simulate writing a simple NAL unit header + some data. - let mut writer = BitWriter::new(); - - // Write forbidden_zero_bit (1 bit) - writer.write_bits(0, 1); - // Write nal_ref_idc (2 bits) - writer.write_bits(3, 2); - // Write nal_unit_type (5 bits) - SPS type 7. - writer.write_bits(7, 5); - - // First byte should be 0b01100111 = 0x67 (SPS NAL header) - let result = writer.finish(); - assert_eq!(result[0], 0x67); - } - - #[test] - fn test_default_impl() { - let writer = BitWriter::default(); - assert_eq!(writer.finish(), vec![]); - } -} diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 8712123a..0e5e7beb 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -353,9 +353,6 @@ impl FramePlan { pub fn is_reference(&self) -> bool { self.gop.is_reference } - pub fn is_b_frame(&self) -> bool { - self.gop.frame_type == GopFrameType::B - } pub fn is_first_frame(&self) -> bool { self.encode_index == 0 } @@ -365,9 +362,8 @@ impl FramePlan { /// The stream-level frame type for packet metadata. pub fn frame_type(&self) -> FrameType { match self.gop.frame_type { - GopFrameType::Idr | GopFrameType::I => FrameType::I, + GopFrameType::Idr => FrameType::I, GopFrameType::P => FrameType::P, - GopFrameType::B => FrameType::B, } } } @@ -951,7 +947,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result // B-frames are not yet supported, so an I-P GOP. Set the SPS-matching // counters (no-ops for AV1, which keys off order hints). - let mut gop = GopStructure::new_ip_only(config.gop_size); + let mut gop = GopStructure::new(config.gop_size); gop.set_max_frame_num(4); gop.set_max_poc_lsb(4); diff --git a/src/encoder/dpb/entry.rs b/src/encoder/dpb/entry.rs index 386acb92..e77548f5 100644 --- a/src/encoder/dpb/entry.rs +++ b/src/encoder/dpb/entry.rs @@ -84,18 +84,8 @@ pub struct DpbEntry { pub temporal_id: i32, /// Timestamp of this picture. pub timestamp: u64, - /// Reference frame timestamp (for error recovery). - pub ref_frame_timestamp: u64, /// Picture type. pub pic_type: PictureType, - /// Dirty intra-refresh regions (for intra-refresh encoding). - pub dirty_intra_refresh_regions: u32, - /// View ID (for MVC support). - pub view_id: u32, - /// Whether top field was decoded first. - pub top_decoded_first: bool, - /// Whether this is a complementary field pair. - pub complementary_field_pair: bool, /// Whether this entry represents a non-existing frame (gap handling). pub not_existing: bool, } @@ -130,12 +120,7 @@ impl DpbEntry { frame_id: 0, temporal_id: 0, timestamp: 0, - ref_frame_timestamp: 0, pic_type: PictureType::I, - dirty_intra_refresh_regions: 0, - view_id: 0, - top_decoded_first: true, - complementary_field_pair: false, not_existing: false, } } @@ -145,11 +130,6 @@ impl DpbEntry { *self = Self::new(); } - /// Check if this entry is in use. - pub fn is_in_use(&self) -> bool { - self.state == DpbState::InUse - } - /// Check if this entry is a reference picture. pub fn is_reference(&self) -> bool { self.marking.is_reference() || self.bottom_field_marking.is_reference() @@ -170,19 +150,6 @@ impl DpbEntry { self.marking = MarkingState::Unused; self.bottom_field_marking = MarkingState::Unused; } - - /// Mark as short-term reference. - pub fn mark_short_term(&mut self) { - self.marking = MarkingState::ShortTerm; - self.bottom_field_marking = MarkingState::ShortTerm; - } - - /// Mark as long-term reference. - pub fn mark_long_term(&mut self, long_term_frame_idx: i32) { - self.marking = MarkingState::LongTerm; - self.bottom_field_marking = MarkingState::LongTerm; - self.long_term_frame_idx = long_term_frame_idx; - } } #[cfg(test)] @@ -194,7 +161,6 @@ mod tests { let entry = DpbEntry::new(); assert_eq!(entry.state, DpbState::Empty); assert_eq!(entry.marking, MarkingState::Unused); - assert!(!entry.is_in_use()); assert!(!entry.is_reference()); } @@ -202,12 +168,14 @@ mod tests { fn test_marking_state() { let mut entry = DpbEntry::new(); - entry.mark_short_term(); + entry.marking = MarkingState::ShortTerm; + entry.bottom_field_marking = MarkingState::ShortTerm; assert!(entry.is_short_term_reference()); assert!(entry.is_reference()); assert!(!entry.is_long_term_reference()); - entry.mark_long_term(0); + entry.marking = MarkingState::LongTerm; + entry.bottom_field_marking = MarkingState::LongTerm; assert!(entry.is_long_term_reference()); assert!(entry.is_reference()); assert!(!entry.is_short_term_reference()); diff --git a/src/encoder/dpb/h264.rs b/src/encoder/dpb/h264.rs index c98787a9..7dea5aa6 100644 --- a/src/encoder/dpb/h264.rs +++ b/src/encoder/dpb/h264.rs @@ -2,15 +2,12 @@ //! //! This module implements the H.264 DPB according to the specification, //! including: -//! - POC calculation (Type 0 and Type 2) +//! - POC calculation (Type 0) //! - Picture number (PicNum) derivation -//! - Sliding window and adaptive memory management -//! - Reference picture list construction for P and B frames -//! - Gap handling for frame_num discontinuities +//! - Sliding window memory management use super::entry::{DpbEntry, DpbState, MarkingState}; -use super::reference_lists::{H264ReferenceListBuilder, ReferenceList}; -use super::types::{DpbConfig, MmcoOperation, PictureStartInfo, PocType}; +use super::types::{DpbConfig, PictureStartInfo}; use super::{DecodedPictureBufferTrait, MAX_DPB_SIZE, PictureType}; /// H.264-specific DPB implementation. @@ -24,8 +21,6 @@ pub struct DpbH264 { max_dpb_size: i8, /// Maximum long-term frame index (-1 means no LTR allowed). max_long_term_frame_idx: i32, - /// POC type. - poc_type: PocType, /// Previous POC MSB (for POC Type 0). prev_poc_msb: i32, /// Previous POC LSB (for POC Type 0). @@ -40,8 +35,6 @@ pub struct DpbH264 { max_frame_num: u32, /// Maximum POC LSB value. max_poc_lsb: i32, - /// Whether to use multiple references. - use_multiple_refs: bool, /// Last IDR timestamp. last_idr_timestamp: u64, /// Number of temporal layers. @@ -62,7 +55,6 @@ impl DpbH264 { current_slot: -1, max_dpb_size: 0, max_long_term_frame_idx: -1, - poc_type: PocType::Type0, prev_poc_msb: 0, prev_poc_lsb: 0, prev_frame_num_offset: 0, @@ -70,61 +62,14 @@ impl DpbH264 { prev_ref_frame_num: 0, max_frame_num: 16, max_poc_lsb: 16, - use_multiple_refs: true, last_idr_timestamp: 0, num_temporal_layers: 1, } } - /// Get reference to the entries array. - pub fn entries(&self) -> &[DpbEntry; MAX_DPB_SIZE] { - &self.entries - } - - /// Get the maximum DPB size. - pub fn dpb_size(&self) -> i8 { - self.max_dpb_size - } - - /// Get the maximum frame_num. - pub fn max_frame_num(&self) -> u32 { - self.max_frame_num - } - - /// Set the POC type. - pub fn set_poc_type(&mut self, poc_type: PocType) { - self.poc_type = poc_type; - } - - /// Get reference picture list for P-frames. - pub fn get_ref_list_p(&self) -> ReferenceList { - H264ReferenceListBuilder::init_ref_list_p_frame( - &self.entries, - self.max_dpb_size, - self.prev_frame_num, - self.max_frame_num, - ) - } - - /// Get reference picture lists for B-frames. - pub fn get_ref_lists_b(&self, current_poc: i32) -> (ReferenceList, ReferenceList) { - H264ReferenceListBuilder::init_ref_lists_b_frame( - &self.entries, - self.max_dpb_size, - current_poc, - ) - } - /// Calculate POC for the current picture. fn calculate_poc(&mut self, info: &PictureStartInfo) { - match self.poc_type { - PocType::Type0 => self.calculate_poc_type0(info), - PocType::Type1 => { - // POC Type 1 not commonly used, fallback to Type 2. - self.calculate_poc_type2(info); - } - PocType::Type2 => self.calculate_poc_type2(info), - } + self.calculate_poc_type0(info); // Derive PicOrderCnt (8-1) let entry = &mut self.entries[self.current_slot as usize]; @@ -165,36 +110,6 @@ impl DpbH264 { } } - /// Calculate POC Type 2 (8.2.1.3). - fn calculate_poc_type2(&mut self, info: &PictureStartInfo) { - // FrameNumOffset (8-12) - let frame_num_offset = if info.pic_type == PictureType::Idr { - 0 - } else if self.prev_frame_num > info.frame_num { - self.prev_frame_num_offset + self.max_frame_num as i32 - } else { - self.prev_frame_num_offset - }; - - // tempPicOrderCnt (8-13) - let temp_poc = if info.pic_type == PictureType::Idr { - 0 - } else if !info.is_reference { - 2 * (frame_num_offset + info.frame_num as i32) - 1 - } else { - 2 * (frame_num_offset + info.frame_num as i32) - }; - - let entry = &mut self.entries[self.current_slot as usize]; - - // (8-14) - entry.top_foc = temp_poc; - entry.bottom_foc = temp_poc; - - self.prev_frame_num_offset = frame_num_offset; - self.prev_frame_num = info.frame_num; - } - /// Calculate PicNum for all DPB entries (8.2.4.1). fn calculate_pic_num(&mut self, frame_num: u32) { for i in 0..self.max_dpb_size as usize { @@ -218,133 +133,6 @@ impl DpbH264 { } } - /// Execute Memory Management Control Operations (8.2.5.4). - pub fn execute_mmco(&mut self, operations: &[MmcoOperation], current_pic_num: i32) { - for op in operations { - match op { - MmcoOperation::End => break, - - MmcoOperation::UnmarkShortTerm { - difference_of_pic_nums_minus1, - } => { - // (8-40) picNumX - let pic_num_x = current_pic_num - (*difference_of_pic_nums_minus1 as i32 + 1); - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - if entry.marking == MarkingState::ShortTerm - && entry.top_pic_num == pic_num_x - { - entry.marking = MarkingState::Unused; - } - if entry.bottom_field_marking == MarkingState::ShortTerm - && entry.bottom_pic_num == pic_num_x - { - entry.bottom_field_marking = MarkingState::Unused; - } - } - } - - MmcoOperation::UnmarkLongTerm { long_term_pic_num } => { - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - if entry.marking == MarkingState::LongTerm - && entry.top_long_term_pic_num == *long_term_pic_num as i32 - { - entry.marking = MarkingState::Unused; - } - if entry.bottom_field_marking == MarkingState::LongTerm - && entry.bottom_long_term_pic_num == *long_term_pic_num as i32 - { - entry.bottom_field_marking = MarkingState::Unused; - } - } - } - - MmcoOperation::MarkLongTerm { - difference_of_pic_nums_minus1, - long_term_frame_idx, - } => { - let pic_num_x = current_pic_num - (*difference_of_pic_nums_minus1 as i32 + 1); - - // First unmark any existing LTR with this index. - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - if entry.marking == MarkingState::LongTerm - && entry.long_term_frame_idx == *long_term_frame_idx as i32 - { - entry.marking = MarkingState::Unused; - } - } - - // Mark the short-term as long-term. - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - if entry.marking == MarkingState::ShortTerm - && entry.top_pic_num == pic_num_x - { - entry.mark_long_term(*long_term_frame_idx as i32); - } - } - } - - MmcoOperation::SetMaxLongTermIndex { - max_long_term_frame_idx_plus1, - } => { - self.max_long_term_frame_idx = *max_long_term_frame_idx_plus1 as i32 - 1; - - // Mark all LTR with index > max as unused. - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - if entry.marking == MarkingState::LongTerm - && entry.long_term_frame_idx > self.max_long_term_frame_idx - { - entry.marking = MarkingState::Unused; - } - } - } - - MmcoOperation::UnmarkAll => { - for i in 0..self.max_dpb_size as usize { - self.entries[i].mark_unused(); - } - self.max_long_term_frame_idx = -1; - - // Reset frame_num and POC for current. - if self.current_slot >= 0 { - let entry = &mut self.entries[self.current_slot as usize]; - entry.frame_num = 0; - let poc = entry.pic_order_cnt; - entry.top_foc -= poc; - entry.bottom_foc -= poc; - entry.pic_order_cnt = 0; - } - } - - MmcoOperation::MarkCurrentAsLongTerm { - long_term_frame_idx, - } => { - // Unmark any existing LTR with this index. - for i in 0..self.max_dpb_size as usize { - if i != self.current_slot as usize { - let entry = &mut self.entries[i]; - if entry.marking == MarkingState::LongTerm - && entry.long_term_frame_idx == *long_term_frame_idx as i32 - { - entry.marking = MarkingState::Unused; - } - } - } - - // Mark current as long-term. - if self.current_slot >= 0 { - let entry = &mut self.entries[self.current_slot as usize]; - entry.mark_long_term(*long_term_frame_idx as i32); - } - } - } - } - } - /// DPB bumping process (C.4.5.3). fn dpb_bumping(&mut self) { // Find picture with smallest POC that needs output. @@ -367,66 +155,6 @@ impl DpbH264 { } } } - - /// Fill frame_num gaps (8.2.5.2). - pub fn fill_frame_num_gaps(&mut self, current_frame_num: u32, info: &PictureStartInfo) { - if info.pic_type == PictureType::Idr { - self.prev_ref_frame_num = 0; - } - - if current_frame_num != self.prev_ref_frame_num { - let mut unused_frame_num = (self.prev_ref_frame_num + 1) % self.max_frame_num; - - while unused_frame_num != current_frame_num { - // Create non-existing frame. - while self.is_full() { - self.dpb_bumping(); - } - - // Find empty slot. - let mut slot: Option = None; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::Empty { - slot = Some(i); - break; - } - } - - if let Some(idx) = slot { - let entry = &mut self.entries[idx]; - entry.state = DpbState::InUse; - entry.frame_num = unused_frame_num; - entry.not_existing = true; - entry.output = false; - entry.mark_short_term(); - } - - unused_frame_num = (unused_frame_num + 1) % self.max_frame_num; - } - } - - if info.is_reference { - self.prev_ref_frame_num = current_frame_num; - } - } - - /// Get an entry by index. - pub fn get_entry(&self, index: usize) -> Option<&DpbEntry> { - if index < MAX_DPB_SIZE && self.entries[index].state == DpbState::InUse { - Some(&self.entries[index]) - } else { - None - } - } - - /// Get mutable entry by index. - pub fn get_entry_mut(&mut self, index: usize) -> Option<&mut DpbEntry> { - if index < MAX_DPB_SIZE && self.entries[index].state == DpbState::InUse { - Some(&mut self.entries[index]) - } else { - None - } - } } impl DecodedPictureBufferTrait for DpbH264 { @@ -437,7 +165,6 @@ impl DecodedPictureBufferTrait for DpbH264 { } self.max_dpb_size = std::cmp::min(config.dpb_size as i8, MAX_DPB_SIZE as i8); - self.use_multiple_refs = config.use_multiple_references; self.max_long_term_frame_idx = -1; self.max_frame_num = config.max_frame_num(); self.max_poc_lsb = config.max_pic_order_cnt_lsb(); @@ -564,10 +291,6 @@ impl DecodedPictureBufferTrait for DpbH264 { } } - fn current_slot(&self) -> i8 { - self.current_slot - } - fn is_full(&self) -> bool { let mut count = 0; for i in 0..self.max_dpb_size as usize { @@ -659,29 +382,4 @@ mod tests { assert!(!dpb.is_empty()); assert_eq!(dpb.num_short_term_refs(), 1); } - - #[test] - fn test_h264_dpb_ref_lists_p() { - let mut dpb = DpbH264::new(); - dpb.sequence_start(DpbConfig { - dpb_size: 4, - ..Default::default() - }); - - // Add IDR. - dpb.picture_start(PictureStartInfo { - frame_id: 0, - pic_order_cnt: 0, - frame_num: 0, - pic_type: PictureType::Idr, - is_reference: true, - ..Default::default() - }); - dpb.picture_end(true); - - // Get ref list for P-frame. - let list = dpb.get_ref_list_p(); - assert_eq!(list.count, 1); - assert_eq!(list.refs[0].dpb_index, 0); - } } diff --git a/src/encoder/dpb/h265.rs b/src/encoder/dpb/h265.rs index 4855e5b5..81709f97 100644 --- a/src/encoder/dpb/h265.rs +++ b/src/encoder/dpb/h265.rs @@ -1,18 +1,10 @@ //! H.265/HEVC-specific DPB implementation. //! -//! This module implements the H.265 DPB according to the specification, -//! including: -//! - Reference Picture Set (RPS) management -//! - Short-term and long-term reference picture handling -//! - Reference picture list construction for P and B frames -//! - Temporal layer support -//! - CRA (Clean Random Access) handling -use std::cmp::Reverse; - +//! This module keeps track of the H.265 reference pictures: which DPB slot +//! holds each picture, its POC, and whether it is still a reference. use super::entry::{DpbEntry, DpbState, MarkingState}; -use super::reference_lists::{H265ReferenceListBuilder, ReferenceList}; -use super::types::{DpbConfig, PictureStartInfo, RefPicSet, ShortTermRefPicSet}; -use super::{DecodedPictureBufferTrait, MAX_DPB_SIZE, MAX_REF_LIST_SIZE, PictureType}; +use super::types::{DpbConfig, PictureStartInfo}; +use super::{DecodedPictureBufferTrait, MAX_DPB_SIZE, PictureType}; /// H.265-specific DPB implementation. #[derive(Debug)] @@ -35,14 +27,8 @@ pub struct DpbH265 { num_poc_lt_foll: i8, /// Last IDR timestamp. last_idr_timestamp: u64, - /// POC of CRA picture (for refresh pending). - pic_order_cnt_cra: i32, - /// Whether a refresh is pending. - refresh_pending: bool, /// Long-term flags bitmask. long_term_flags: u32, - /// Whether to use multiple references. - use_multiple_refs: bool, /// Maximum POC LSB value. max_poc_lsb: i32, /// Number of temporal layers. @@ -68,464 +54,12 @@ impl DpbH265 { num_poc_lt_curr: 0, num_poc_lt_foll: 0, last_idr_timestamp: 0, - pic_order_cnt_cra: 0, - refresh_pending: false, long_term_flags: 0, - use_multiple_refs: true, max_poc_lsb: 16, num_temporal_layers: 1, } } - /// Get reference to the entries array. - pub fn entries(&self) -> &[DpbEntry; MAX_DPB_SIZE] { - &self.entries - } - - /// Get the maximum DPB size. - pub fn dpb_size(&self) -> i8 { - self.max_dpb_size - } - - /// Get the maximum POC LSB value. - pub fn max_poc_lsb(&self) -> i32 { - self.max_poc_lsb - } - - /// Reference picture marking for H.265. - /// - /// Marks pictures as unused based on picture type and long-term flags. - pub fn reference_picture_marking( - &mut self, - current_poc: i32, - pic_type: PictureType, - long_term_refs_present: bool, - ) { - if pic_type == PictureType::Idr { - // IDR: mark all as unused. - for entry in &mut self.entries[..self.max_dpb_size as usize] { - entry.mark_unused(); - } - return; - } - - // Handle CRA refresh pending. - if self.refresh_pending && current_poc > self.pic_order_cnt_cra { - for entry in &mut self.entries[..self.max_dpb_size as usize] { - if entry.pic_order_cnt != self.pic_order_cnt_cra { - entry.mark_unused(); - } - } - self.refresh_pending = false; - } - - // CRA picture triggers refresh. - if pic_type == PictureType::I { - self.refresh_pending = true; - self.pic_order_cnt_cra = current_poc; - } - - if !self.use_multiple_refs { - return; - } - - // Balance short-term and long-term references. - let mut num_short_term = 0; - let mut num_long_term = 0; - let mut num_corrupted = 0; - let mut oldest_st_idx: Option = None; - let mut oldest_st_poc = u32::MAX; - let mut oldest_lt_idx: Option = None; - let mut oldest_lt_poc = u32::MAX; - let mut oldest_corrupted_idx: Option = None; - let mut oldest_corrupted_poc = u32::MAX; - - for i in 0..self.max_dpb_size as usize { - let entry = &self.entries[i]; - if entry.state != DpbState::InUse { - continue; - } - - if entry.marking == MarkingState::ShortTerm && !entry.corrupted { - num_short_term += 1; - if (entry.pic_order_cnt as u32) < oldest_st_poc { - oldest_st_poc = entry.pic_order_cnt as u32; - oldest_st_idx = Some(i); - } - } - - if entry.marking == MarkingState::LongTerm && !entry.corrupted { - num_long_term += 1; - if (entry.pic_order_cnt as u32) < oldest_lt_poc { - oldest_lt_poc = entry.pic_order_cnt as u32; - oldest_lt_idx = Some(i); - } - } - - if entry.corrupted { - num_corrupted += 1; - if (entry.pic_order_cnt as u32) < oldest_corrupted_poc { - oldest_corrupted_poc = entry.pic_order_cnt as u32; - oldest_corrupted_idx = Some(i); - } - } - } - - let total_refs = num_short_term + num_long_term + num_corrupted; - - if !long_term_refs_present { - if total_refs > (self.max_dpb_size as usize - 1) { - // Remove oldest corrupted, or oldest short-term, or oldest long-term. - if num_corrupted > 0 && oldest_corrupted_poc < oldest_st_poc { - if let Some(idx) = oldest_corrupted_idx { - self.entries[idx].mark_unused(); - } - } else if num_short_term > 0 { - if let Some(idx) = oldest_st_idx { - self.entries[idx].mark_unused(); - } - } else if num_long_term > 0 - && let Some(idx) = oldest_lt_idx - { - self.entries[idx].mark_unused(); - } - } - } else { - // Balance LTR and STR (max 50% LTR) - let max_allowed_ltr = total_refs / 2; - - if total_refs > (self.max_dpb_size as usize - 1) { - if num_corrupted > 0 && oldest_corrupted_poc < oldest_st_poc { - if let Some(idx) = oldest_corrupted_idx { - self.entries[idx].mark_unused(); - } - } else if num_long_term > max_allowed_ltr { - if let Some(idx) = oldest_lt_idx { - self.entries[idx].mark_unused(); - } - } else if num_short_term > 0 - && let Some(idx) = oldest_st_idx - { - self.entries[idx].mark_unused(); - } - } - } - } - - /// Apply the Reference Picture Set to mark pictures. - /// - /// This categorizes reference pictures into: - /// - stCurrBefore: short-term refs with POC < current - /// - stCurrAfter: short-term refs with POC > current - /// - ltCurr: long-term refs used by current - /// - stFoll: short-term refs not used by current - /// - ltFoll: long-term refs not used by current - pub fn apply_reference_picture_set( - &mut self, - current_poc: i32, - pic_type: PictureType, - short_term_rps: &ShortTermRefPicSet, - num_long_term_pics: u32, - long_term_poc_lsb: &[u32], - used_by_curr_pic_lt: u32, - ) -> RefPicSet { - let mut ref_pic_set = RefPicSet::new(); - - if pic_type == PictureType::Idr { - // IDR has no references. - self.num_poc_st_curr_before = 0; - self.num_poc_st_curr_after = 0; - self.num_poc_st_foll = 0; - self.num_poc_lt_curr = 0; - self.num_poc_lt_foll = 0; - return ref_pic_set; - } - - // Calculate delta POC values. - let mut delta_poc_s0 = [0i32; MAX_DPB_SIZE]; - let mut delta_poc_s1 = [0i32; MAX_DPB_SIZE]; - - // S0: negative delta POCs. - for i in 0..short_term_rps.num_negative_pics as usize { - delta_poc_s0[i] = if i == 0 { - -(short_term_rps.delta_poc_s0_minus1[i] as i32 + 1) - } else { - delta_poc_s0[i - 1] - (short_term_rps.delta_poc_s0_minus1[i] as i32 + 1) - }; - } - - // S1: positive delta POCs. - for i in 0..short_term_rps.num_positive_pics as usize { - delta_poc_s1[i] = if i == 0 { - short_term_rps.delta_poc_s1_minus1[i] as i32 + 1 - } else { - delta_poc_s1[i - 1] + short_term_rps.delta_poc_s1_minus1[i] as i32 + 1 - }; - } - - // Build POC lists. - let mut poc_st_curr_before = [0i32; MAX_REF_LIST_SIZE]; - let mut poc_st_curr_after = [0i32; MAX_REF_LIST_SIZE]; - let mut poc_st_foll = [0i32; MAX_REF_LIST_SIZE]; - let mut poc_lt_curr = [0i32; MAX_REF_LIST_SIZE]; - - let mut j = 0usize; - let mut k = 0usize; - - // Process S0 (negative) - for (i, &delta) in delta_poc_s0 - .iter() - .enumerate() - .take(short_term_rps.num_negative_pics as usize) - { - let poc = current_poc + delta; - if (short_term_rps.used_by_curr_pic_s0_flag >> i) & 1 != 0 { - poc_st_curr_before[j] = poc; - j += 1; - } else { - poc_st_foll[k] = poc; - k += 1; - } - } - self.num_poc_st_curr_before = j as i8; - - j = 0; - // Process S1 (positive) - for (i, &delta) in delta_poc_s1 - .iter() - .enumerate() - .take(short_term_rps.num_positive_pics as usize) - { - let poc = current_poc + delta; - if (short_term_rps.used_by_curr_pic_s1_flag >> i) & 1 != 0 { - poc_st_curr_after[j] = poc; - j += 1; - } else { - poc_st_foll[k] = poc; - k += 1; - } - } - self.num_poc_st_curr_after = j as i8; - self.num_poc_st_foll = k as i8; - - // Process long-term references. - j = 0; - k = 0; - for (i, &poc_lsb) in long_term_poc_lsb - .iter() - .enumerate() - .take(num_long_term_pics as usize) - { - let poc_lt = poc_lsb as i32; - if (used_by_curr_pic_lt >> i) & 1 != 0 { - poc_lt_curr[j] = poc_lt; - j += 1; - } else { - k += 1; - } - } - self.num_poc_lt_curr = j as i8; - self.num_poc_lt_foll = k as i8; - - // Map POCs to DPB indices. - // stCurrBefore - for (i, &poc) in poc_st_curr_before - .iter() - .enumerate() - .take(self.num_poc_st_curr_before as usize) - { - ref_pic_set.st_curr_before[i] = -1; - for d in 0..self.max_dpb_size as usize { - let entry = &self.entries[d]; - if entry.state == DpbState::InUse - && entry.marking == MarkingState::ShortTerm - && entry.pic_order_cnt == poc - { - ref_pic_set.st_curr_before[i] = d as i8; - break; - } - } - } - ref_pic_set.num_st_curr_before = self.num_poc_st_curr_before as u8; - - // stCurrAfter - for (i, &poc) in poc_st_curr_after - .iter() - .enumerate() - .take(self.num_poc_st_curr_after as usize) - { - ref_pic_set.st_curr_after[i] = -1; - for d in 0..self.max_dpb_size as usize { - let entry = &self.entries[d]; - if entry.state == DpbState::InUse - && entry.marking == MarkingState::ShortTerm - && entry.pic_order_cnt == poc - { - ref_pic_set.st_curr_after[i] = d as i8; - break; - } - } - } - ref_pic_set.num_st_curr_after = self.num_poc_st_curr_after as u8; - - // ltCurr - for (i, &poc) in poc_lt_curr - .iter() - .enumerate() - .take(self.num_poc_lt_curr as usize) - { - ref_pic_set.lt_curr[i] = -1; - let mask = self.max_poc_lsb - 1; - for d in 0..self.max_dpb_size as usize { - let entry = &self.entries[d]; - if entry.state == DpbState::InUse - && entry.marking != MarkingState::Unused - && (entry.pic_order_cnt & mask) == poc - { - ref_pic_set.lt_curr[i] = d as i8; - // Mark as long-term. - break; - } - } - } - ref_pic_set.num_lt_curr = self.num_poc_lt_curr as u8; - - // Mark pictures not in RPS as unused. - let mut in_use = [false; MAX_DPB_SIZE]; - for i in 0..self.num_poc_st_curr_before as usize { - if ref_pic_set.st_curr_before[i] >= 0 { - in_use[ref_pic_set.st_curr_before[i] as usize] = true; - } - } - for i in 0..self.num_poc_st_curr_after as usize { - if ref_pic_set.st_curr_after[i] >= 0 { - in_use[ref_pic_set.st_curr_after[i] as usize] = true; - } - } - for i in 0..self.num_poc_lt_curr as usize { - if ref_pic_set.lt_curr[i] >= 0 { - in_use[ref_pic_set.lt_curr[i] as usize] = true; - } - } - // stFoll and ltFoll would also be marked as in_use - - for (i, used) in in_use.iter().enumerate().take(self.max_dpb_size as usize) { - if !used && self.entries[i].marking != MarkingState::Unused { - self.entries[i].mark_unused(); - } - } - - ref_pic_set - } - - /// Initialize the short-term RPS for a P/B frame. - /// - /// This builds the short-term reference picture set based on current DPB state. - pub fn initialize_short_term_rps( - &self, - current_poc: i32, - pic_type: PictureType, - current_temporal_id: i32, - num_ref_l0: u32, - num_ref_l1: u32, - ) -> ShortTermRefPicSet { - let mut rps = ShortTermRefPicSet::default(); - - if pic_type == PictureType::Idr { - return rps; - } - - // Collect short-term references. - let mut negative_refs: Vec<(i32, i32)> = Vec::new(); // (POC, delta_poc) - let mut positive_refs: Vec<(i32, i32)> = Vec::new(); - - for i in 0..self.max_dpb_size as usize { - let entry = &self.entries[i]; - if entry.state != DpbState::InUse { - continue; - } - if entry.marking != MarkingState::ShortTerm { - continue; - } - if entry.corrupted { - continue; - } - if entry.temporal_id > current_temporal_id { - continue; - } - - let delta_poc = entry.pic_order_cnt - current_poc; - if delta_poc < 0 { - negative_refs.push((entry.pic_order_cnt, delta_poc)); - } else if delta_poc > 0 && self.use_multiple_refs { - positive_refs.push((entry.pic_order_cnt, delta_poc)); - } - } - - // Sort negative refs by POC descending (closest to current first) - negative_refs.sort_by_key(|b| Reverse(b.0)); - - // Sort positive refs by POC ascending (closest to current first) - positive_refs.sort_by_key(|a| a.0); - - // Limit to DPB size - 1. - let max_refs = (self.max_dpb_size - 1) as usize; - while negative_refs.len() + positive_refs.len() > max_refs { - if !negative_refs.is_empty() { - negative_refs.pop(); - } else if !positive_refs.is_empty() { - positive_refs.pop(); - } - } - - // Build the RPS structure. - rps.num_negative_pics = negative_refs.len() as u8; - rps.num_positive_pics = positive_refs.len() as u8; - - let mut prev_delta = 0i32; - for (i, (_, delta_poc)) in negative_refs.iter().enumerate() { - rps.delta_poc_s0_minus1[i] = (prev_delta - delta_poc - 1) as u8; - // Mark as used if within num_ref_l0. - if i < num_ref_l0 as usize { - rps.used_by_curr_pic_s0_flag |= 1 << i; - } - prev_delta = *delta_poc; - } - - prev_delta = 0; - for (i, (_, delta_poc)) in positive_refs.iter().enumerate() { - rps.delta_poc_s1_minus1[i] = (*delta_poc - prev_delta - 1) as u8; - // Mark as used if within num_ref_l1. - if i < num_ref_l1 as usize { - rps.used_by_curr_pic_s1_flag |= 1 << i; - } - prev_delta = *delta_poc; - } - - rps - } - - /// Setup reference picture lists L0 and L1. - pub fn setup_reference_lists( - &self, - pic_type: PictureType, - ref_pic_set: &RefPicSet, - num_ref_l0: u32, - num_ref_l1: u32, - ) -> (ReferenceList, ReferenceList) { - H265ReferenceListBuilder::build_ref_lists( - &self.entries, - self.max_dpb_size, - pic_type, - &ref_pic_set.st_curr_before, - &ref_pic_set.st_curr_after, - &ref_pic_set.lt_curr, - num_ref_l0, - num_ref_l1, - self.use_multiple_refs, - ) - } - /// DPB bumping process. fn dpb_bumping(&mut self) { // Find picture with smallest POC that needs output. @@ -547,29 +81,6 @@ impl DpbH265 { } } } - - /// Get an entry by index. - pub fn get_entry(&self, index: usize) -> Option<&DpbEntry> { - if index < MAX_DPB_SIZE && self.entries[index].state == DpbState::InUse { - Some(&self.entries[index]) - } else { - None - } - } - - /// Get mutable entry by index. - pub fn get_entry_mut(&mut self, index: usize) -> Option<&mut DpbEntry> { - if index < MAX_DPB_SIZE { - Some(&mut self.entries[index]) - } else { - None - } - } - - /// Get the long-term flags. - pub fn long_term_flags(&self) -> u32 { - self.long_term_flags - } } impl DecodedPictureBufferTrait for DpbH265 { @@ -580,7 +91,6 @@ impl DecodedPictureBufferTrait for DpbH265 { } self.max_dpb_size = std::cmp::min(config.dpb_size as i8, MAX_DPB_SIZE as i8); - self.use_multiple_refs = config.use_multiple_references; self.max_poc_lsb = config.max_pic_order_cnt_lsb(); self.num_temporal_layers = config.num_temporal_layers; self.current_slot = -1; @@ -589,7 +99,6 @@ impl DecodedPictureBufferTrait for DpbH265 { self.num_poc_st_foll = 0; self.num_poc_lt_curr = 0; self.num_poc_lt_foll = 0; - self.refresh_pending = false; self.long_term_flags = 0; } @@ -736,10 +245,6 @@ impl DecodedPictureBufferTrait for DpbH265 { } } - fn current_slot(&self) -> i8 { - self.current_slot - } - fn is_full(&self) -> bool { let mut count = 0; for i in 0..self.max_dpb_size as usize { @@ -851,29 +356,4 @@ mod tests { assert_eq!(dpb.num_short_term_refs(), 2); } - - #[test] - fn test_h265_short_term_rps() { - let mut dpb = DpbH265::new(); - dpb.sequence_start(DpbConfig { - dpb_size: 4, - use_multiple_references: true, - ..Default::default() - }); - - // Add IDR at POC 0. - dpb.picture_start(PictureStartInfo { - frame_id: 0, - pic_order_cnt: 0, - pic_type: PictureType::Idr, - is_reference: true, - ..Default::default() - }); - dpb.picture_end(true); - - // Initialize RPS for P-frame at POC 2. - let rps = dpb.initialize_short_term_rps(2, PictureType::P, 0, 1, 0); - assert_eq!(rps.num_negative_pics, 1); - assert_eq!(rps.num_positive_pics, 0); - } } diff --git a/src/encoder/dpb/mod.rs b/src/encoder/dpb/mod.rs index da9975cc..4f41962b 100644 --- a/src/encoder/dpb/mod.rs +++ b/src/encoder/dpb/mod.rs @@ -1,25 +1,18 @@ //! Decoded Picture Buffer (DPB) management for video encoding. //! -//! This module provides full-featured DPB management for H.264 and H.265 video encoding, -//! including: -//! - B-frame support with categorization of frames before/after current frame in display order -//! - Complex GOP structures with hierarchical B-frames and multiple reference frames -//! - Long-term reference (LTR) support for keeping golden frames in the DPB -//! - Bitstream compliance for correctly populating slice headers and parameter sets +//! This module tracks the reference pictures of H.264 and H.265 encoding: +//! picture order counts, frame numbers and reference marking. //! -//! The DPB is codec-agnostic at the interface level, with codec-specific implementations. -//! for H.264 and H.265 in separate modules. +//! The DPB is codec-agnostic at the interface level, with codec-specific +//! implementations for H.264 and H.265 in separate modules. mod entry; mod h264; mod h265; -mod reference_lists; mod types; -pub use entry::{DpbEntry, MarkingState}; pub use h264::DpbH264; pub use h265::DpbH265; -pub use reference_lists::{ReferenceList, ReferencePicture}; pub use types::*; /// Maximum DPB size (16 slots for H.264, 15 for H.265, use 16 for both). @@ -28,15 +21,9 @@ pub const MAX_DPB_SIZE: usize = 16; /// Alias for MAX_DPB_SIZE for backwards compatibility. pub const MAX_DPB_SLOTS: usize = MAX_DPB_SIZE; -/// Maximum number of reference pictures in a list. -pub const MAX_REF_LIST_SIZE: usize = 16; - /// Default DPB size for encoding. pub const DEFAULT_DPB_SIZE: u32 = 4; -/// Default maximum number of reference frames. -pub const DEFAULT_MAX_NUM_REF_FRAMES: u32 = 4; - /// Picture type for encoding. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PictureType { @@ -46,32 +33,6 @@ pub enum PictureType { I, /// P picture (Predicted, uses L0 references). P, - /// B picture (Bi-predicted, uses L0 and L1 references). - B, -} - -impl PictureType { - /// Returns true if this is an IRAP (Intra Random Access Point) picture. - pub fn is_irap(&self) -> bool { - matches!(self, Self::Idr | Self::I) - } - - /// Returns true if this picture type uses reference pictures. - pub fn uses_references(&self) -> bool { - matches!(self, Self::P | Self::B) - } - - /// Returns true if this is a reference picture (can be used by other frames). - pub fn is_reference(&self) -> bool { - // B-frames can also be references in hierarchical B-frame structures. - // This is determined by the encoder, not the picture type alone. - matches!(self, Self::Idr | Self::I | Self::P) - } - - /// Returns true if this is a B picture. - pub fn is_b_frame(&self) -> bool { - matches!(self, Self::B) - } } /// Common DPB interface for both H.264 and H.265. @@ -89,9 +50,6 @@ pub trait DecodedPictureBufferTrait { /// End encoding a picture and update the DPB. fn picture_end(&mut self, is_reference: bool); - /// Get the current DPB slot index. - fn current_slot(&self) -> i8; - /// Check if the DPB is full. fn is_full(&self) -> bool; @@ -134,21 +92,3 @@ impl Default for DecodedPictureBuffer { Self::new() } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_picture_type() { - assert!(PictureType::Idr.is_irap()); - assert!(PictureType::I.is_irap()); - assert!(!PictureType::P.is_irap()); - assert!(!PictureType::B.is_irap()); - - assert!(!PictureType::Idr.uses_references()); - assert!(!PictureType::I.uses_references()); - assert!(PictureType::P.uses_references()); - assert!(PictureType::B.uses_references()); - } -} diff --git a/src/encoder/dpb/reference_lists.rs b/src/encoder/dpb/reference_lists.rs deleted file mode 100644 index 07c52620..00000000 --- a/src/encoder/dpb/reference_lists.rs +++ /dev/null @@ -1,536 +0,0 @@ -//! Reference picture list construction and management. - -use std::cmp::Reverse; - -use super::entry::{DpbEntry, DpbState, MarkingState}; -use super::{MAX_DPB_SIZE, MAX_REF_LIST_SIZE, PictureType}; - -/// A reference picture for building reference lists. -#[derive(Debug, Clone, Copy)] -pub struct ReferencePicture { - /// DPB slot index. - pub dpb_index: i8, - /// Picture Order Count. - pub poc: i32, - /// Frame number. - pub frame_num: u32, - /// Whether this is a long-term reference. - pub is_long_term: bool, - /// Long-term picture number (if long-term). - pub long_term_pic_num: i32, - /// Picture number (short-term). - pub pic_num: i32, -} - -impl Default for ReferencePicture { - fn default() -> Self { - Self { - dpb_index: -1, - poc: 0, - frame_num: 0, - is_long_term: false, - long_term_pic_num: 0, - pic_num: 0, - } - } -} - -/// Reference picture list (L0 or L1). -#[derive(Debug, Clone)] -pub struct ReferenceList { - /// Reference pictures in this list. - pub refs: [ReferencePicture; MAX_REF_LIST_SIZE], - /// Number of active references in this list. - pub count: u8, - /// Number of active references minus 1 (for slice header). - pub num_ref_idx_active_minus1: u8, -} - -impl Default for ReferenceList { - fn default() -> Self { - Self::new() - } -} - -impl ReferenceList { - /// Create an empty reference list. - pub fn new() -> Self { - Self { - refs: [ReferencePicture::default(); MAX_REF_LIST_SIZE], - count: 0, - num_ref_idx_active_minus1: 0, - } - } - - /// Clear the reference list. - pub fn clear(&mut self) { - self.count = 0; - self.num_ref_idx_active_minus1 = 0; - for r in &mut self.refs { - r.dpb_index = -1; - } - } - - /// Add a reference to the list. - pub fn add(&mut self, reference: ReferencePicture) { - if (self.count as usize) < MAX_REF_LIST_SIZE { - self.refs[self.count as usize] = reference; - self.count += 1; - } - } - - /// Get the DPB indices of references in this list. - pub fn dpb_indices(&self) -> Vec { - self.refs[..self.count as usize] - .iter() - .map(|r| r.dpb_index) - .collect() - } - - /// Check if a DPB index is in this list. - pub fn contains_dpb_index(&self, dpb_index: i8) -> bool { - self.refs[..self.count as usize] - .iter() - .any(|r| r.dpb_index == dpb_index) - } -} - -/// Reference list builder for H.264. -pub struct H264ReferenceListBuilder; - -impl H264ReferenceListBuilder { - /// Initialize reference picture list for P-frames (L0 only). - /// - /// For P-frames, L0 is initialized with short-term references sorted by. - /// descending PicNum, followed by long-term references sorted by ascending - /// LongTermPicNum. - pub fn init_ref_list_p_frame( - dpb: &[DpbEntry; MAX_DPB_SIZE], - dpb_size: i8, - current_frame_num: u32, - max_frame_num: u32, - ) -> ReferenceList { - let mut list = ReferenceList::new(); - let mut short_term_refs: Vec = Vec::new(); - let mut long_term_refs: Vec = Vec::new(); - - // Collect short-term and long-term references. - for (i, entry) in dpb.iter().enumerate().take(dpb_size as usize) { - if entry.state != DpbState::InUse { - continue; - } - - if entry.marking == MarkingState::ShortTerm - || entry.bottom_field_marking == MarkingState::ShortTerm - { - // Calculate frame_num_wrap. - let frame_num_wrap = if entry.frame_num > current_frame_num { - entry.frame_num as i32 - max_frame_num as i32 - } else { - entry.frame_num as i32 - }; - - short_term_refs.push(ReferencePicture { - dpb_index: i as i8, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: false, - long_term_pic_num: 0, - pic_num: frame_num_wrap, - }); - } - - if entry.marking == MarkingState::LongTerm - || entry.bottom_field_marking == MarkingState::LongTerm - { - long_term_refs.push(ReferencePicture { - dpb_index: i as i8, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: true, - long_term_pic_num: entry.long_term_frame_idx, - pic_num: 0, - }); - } - } - - // Sort short-term by descending PicNum. - short_term_refs.sort_by_key(|b| Reverse(b.pic_num)); - - // Sort long-term by ascending LongTermPicNum. - long_term_refs.sort_by_key(|a| a.long_term_pic_num); - - // Build L0: short-term first, then long-term. - for r in short_term_refs { - list.add(r); - } - for r in long_term_refs { - list.add(r); - } - - if list.count > 0 { - list.num_ref_idx_active_minus1 = list.count - 1; - } - - list - } - - /// Initialize reference picture lists for B-frames (L0 and L1). - /// - /// For B-frames: - /// - L0: Short-term refs with POC < current sorted by descending POC, - /// then POC > current sorted by ascending POC, then long-term refs. - /// - L1: Short-term refs with POC > current sorted by ascending POC, - /// then POC < current sorted by descending POC, then long-term refs. - pub fn init_ref_lists_b_frame( - dpb: &[DpbEntry; MAX_DPB_SIZE], - dpb_size: i8, - current_poc: i32, - ) -> (ReferenceList, ReferenceList) { - let mut list0 = ReferenceList::new(); - let mut list1 = ReferenceList::new(); - - let mut refs_before: Vec = Vec::new(); - let mut refs_after: Vec = Vec::new(); - let mut long_term_refs: Vec = Vec::new(); - - // Collect references categorized by POC. - for (i, entry) in dpb.iter().enumerate().take(dpb_size as usize) { - if entry.state != DpbState::InUse { - continue; - } - - if entry.marking == MarkingState::ShortTerm - || entry.bottom_field_marking == MarkingState::ShortTerm - { - let ref_pic = ReferencePicture { - dpb_index: i as i8, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: false, - long_term_pic_num: 0, - pic_num: entry.top_pic_num, - }; - - if entry.pic_order_cnt < current_poc { - refs_before.push(ref_pic); - } else if entry.pic_order_cnt > current_poc { - refs_after.push(ref_pic); - } - } - - if entry.marking == MarkingState::LongTerm - || entry.bottom_field_marking == MarkingState::LongTerm - { - long_term_refs.push(ReferencePicture { - dpb_index: i as i8, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: true, - long_term_pic_num: entry.long_term_frame_idx, - pic_num: 0, - }); - } - } - - // Sort refs_before by descending POC (closest to current first) - refs_before.sort_by_key(|b| Reverse(b.poc)); - - // Sort refs_after by ascending POC (closest to current first) - refs_after.sort_by_key(|a| a.poc); - - // Sort long-term by ascending LongTermPicNum. - long_term_refs.sort_by_key(|a| a.long_term_pic_num); - - // Build L0: refs_before, refs_after, long_term. - for r in &refs_before { - list0.add(*r); - } - for r in &refs_after { - list0.add(*r); - } - for r in &long_term_refs { - list0.add(*r); - } - - // Build L1: refs_after, refs_before, long_term. - for r in &refs_after { - list1.add(*r); - } - for r in &refs_before { - list1.add(*r); - } - for r in &long_term_refs { - list1.add(*r); - } - - if list0.count > 0 { - list0.num_ref_idx_active_minus1 = list0.count.saturating_sub(1); - } - if list1.count > 0 { - list1.num_ref_idx_active_minus1 = list1.count.saturating_sub(1); - } - - // H.264 spec: If L0 and L1 have same entries in same order, swap first two in L1. - if list0.count > 1 - && list1.count > 1 - && list0.count == list1.count - && Self::lists_identical(&list0, &list1) - { - list1.refs.swap(0, 1); - } - - (list0, list1) - } - - /// Check if two reference lists have identical entries. - fn lists_identical(l0: &ReferenceList, l1: &ReferenceList) -> bool { - if l0.count != l1.count { - return false; - } - for i in 0..l0.count as usize { - if l0.refs[i].dpb_index != l1.refs[i].dpb_index { - return false; - } - } - true - } -} - -/// Reference list builder for H.265. -pub struct H265ReferenceListBuilder; - -impl H265ReferenceListBuilder { - /// Build reference picture lists L0 and L1 from the Reference Picture Set. - /// - /// For P-frames: L0 only (from stCurrBefore, stCurrAfter, ltCurr). - /// For B-frames: L0 and L1 with different ordering. - #[allow(clippy::too_many_arguments)] - pub fn build_ref_lists( - dpb: &[DpbEntry; MAX_DPB_SIZE], - dpb_size: i8, - pic_type: PictureType, - st_curr_before: &[i8], - st_curr_after: &[i8], - lt_curr: &[i8], - num_ref_l0: u32, - num_ref_l1: u32, - use_multiple_refs: bool, - ) -> (ReferenceList, ReferenceList) { - let mut list0 = ReferenceList::new(); - let mut list1 = ReferenceList::new(); - - let num_poc_st_curr_before = st_curr_before.iter().filter(|&&x| x >= 0).count(); - let num_poc_st_curr_after = st_curr_after.iter().filter(|&&x| x >= 0).count(); - let num_poc_lt_curr = lt_curr.iter().filter(|&&x| x >= 0).count(); - let num_poc_total_curr = num_poc_st_curr_before + num_poc_st_curr_after + num_poc_lt_curr; - - // Set active reference counts. - list0.num_ref_idx_active_minus1 = if num_ref_l0 > 0 { - (num_ref_l0 - 1) as u8 - } else { - 0 - }; - list1.num_ref_idx_active_minus1 = if num_ref_l1 > 0 { - (num_ref_l1 - 1) as u8 - } else { - 0 - }; - - // Adjust based on available references. - if use_multiple_refs { - if (list0.num_ref_idx_active_minus1 as usize + 1) > num_poc_st_curr_before { - list0.num_ref_idx_active_minus1 = num_poc_st_curr_before.saturating_sub(1) as u8; - } - if pic_type == PictureType::B - && (list1.num_ref_idx_active_minus1 as usize + 1) > num_poc_st_curr_after - { - list1.num_ref_idx_active_minus1 = num_poc_st_curr_after.saturating_sub(1) as u8; - } - } - - // Build L0 for P and B frames. - if pic_type == PictureType::P || pic_type == PictureType::B { - let num_rps_curr_temp_list0 = std::cmp::max( - list0.num_ref_idx_active_minus1 as usize + 1, - num_poc_total_curr, - ); - - let mut r_idx = 0; - while r_idx < num_rps_curr_temp_list0 { - // Add stCurrBefore. - for &dpb_idx in st_curr_before.iter().filter(|&&x| x >= 0) { - if r_idx >= num_rps_curr_temp_list0 { - break; - } - if dpb_idx >= 0 && (dpb_idx as usize) < dpb_size as usize { - let entry = &dpb[dpb_idx as usize]; - list0.add(ReferencePicture { - dpb_index: dpb_idx, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: false, - long_term_pic_num: 0, - pic_num: entry.top_pic_num, - }); - } - r_idx += 1; - } - // Add stCurrAfter. - for &dpb_idx in st_curr_after.iter().filter(|&&x| x >= 0) { - if r_idx >= num_rps_curr_temp_list0 { - break; - } - if dpb_idx >= 0 && (dpb_idx as usize) < dpb_size as usize { - let entry = &dpb[dpb_idx as usize]; - list0.add(ReferencePicture { - dpb_index: dpb_idx, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: false, - long_term_pic_num: 0, - pic_num: entry.top_pic_num, - }); - } - r_idx += 1; - } - // Add ltCurr. - for &dpb_idx in lt_curr.iter().filter(|&&x| x >= 0) { - if r_idx >= num_rps_curr_temp_list0 { - break; - } - if dpb_idx >= 0 && (dpb_idx as usize) < dpb_size as usize { - let entry = &dpb[dpb_idx as usize]; - list0.add(ReferencePicture { - dpb_index: dpb_idx, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: true, - long_term_pic_num: entry.long_term_frame_idx, - pic_num: 0, - }); - } - r_idx += 1; - } - - // If we still need more refs, we'll loop and repeat. - if r_idx >= num_rps_curr_temp_list0 { - break; - } - } - } - - // Build L1 for B frames. - if pic_type == PictureType::B { - let num_rps_curr_temp_list1 = std::cmp::max( - list1.num_ref_idx_active_minus1 as usize + 1, - num_poc_total_curr, - ); - - let mut r_idx = 0; - while r_idx < num_rps_curr_temp_list1 { - // Add stCurrAfter first (future references) - for &dpb_idx in st_curr_after.iter().filter(|&&x| x >= 0) { - if r_idx >= num_rps_curr_temp_list1 { - break; - } - if dpb_idx >= 0 && (dpb_idx as usize) < dpb_size as usize { - let entry = &dpb[dpb_idx as usize]; - list1.add(ReferencePicture { - dpb_index: dpb_idx, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: false, - long_term_pic_num: 0, - pic_num: entry.top_pic_num, - }); - } - r_idx += 1; - } - // Add stCurrBefore. - for &dpb_idx in st_curr_before.iter().filter(|&&x| x >= 0) { - if r_idx >= num_rps_curr_temp_list1 { - break; - } - if dpb_idx >= 0 && (dpb_idx as usize) < dpb_size as usize { - let entry = &dpb[dpb_idx as usize]; - list1.add(ReferencePicture { - dpb_index: dpb_idx, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: false, - long_term_pic_num: 0, - pic_num: entry.top_pic_num, - }); - } - r_idx += 1; - } - // Add ltCurr. - for &dpb_idx in lt_curr.iter().filter(|&&x| x >= 0) { - if r_idx >= num_rps_curr_temp_list1 { - break; - } - if dpb_idx >= 0 && (dpb_idx as usize) < dpb_size as usize { - let entry = &dpb[dpb_idx as usize]; - list1.add(ReferencePicture { - dpb_index: dpb_idx, - poc: entry.pic_order_cnt, - frame_num: entry.frame_num, - is_long_term: true, - long_term_pic_num: entry.long_term_frame_idx, - pic_num: 0, - }); - } - r_idx += 1; - } - - if r_idx >= num_rps_curr_temp_list1 { - break; - } - } - } - - (list0, list1) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_reference_list_add() { - let mut list = ReferenceList::new(); - assert_eq!(list.count, 0); - - list.add(ReferencePicture { - dpb_index: 0, - poc: 0, - ..Default::default() - }); - assert_eq!(list.count, 1); - - list.add(ReferencePicture { - dpb_index: 1, - poc: 2, - ..Default::default() - }); - assert_eq!(list.count, 2); - } - - #[test] - fn test_reference_list_dpb_indices() { - let mut list = ReferenceList::new(); - list.add(ReferencePicture { - dpb_index: 2, - ..Default::default() - }); - list.add(ReferencePicture { - dpb_index: 5, - ..Default::default() - }); - - let indices = list.dpb_indices(); - assert_eq!(indices, vec![2, 5]); - } -} diff --git a/src/encoder/dpb/types.rs b/src/encoder/dpb/types.rs index 57cf55f6..e10c7c6a 100644 --- a/src/encoder/dpb/types.rs +++ b/src/encoder/dpb/types.rs @@ -1,18 +1,12 @@ //! DPB types and configuration structures. -use super::{DEFAULT_DPB_SIZE, DEFAULT_MAX_NUM_REF_FRAMES}; +use super::DEFAULT_DPB_SIZE; /// Configuration for DPB initialization. #[derive(Debug, Clone)] pub struct DpbConfig { /// Maximum DPB size in frames. pub dpb_size: u32, - /// Maximum number of reference frames. - pub max_num_ref_frames: u32, - /// Whether to use multiple references (enables full B-frame support). - pub use_multiple_references: bool, - /// Maximum number of long-term reference frames (0 to disable LTR). - pub max_long_term_refs: u32, /// Log2 of max_frame_num (for H.264 frame_num calculation). pub log2_max_frame_num_minus4: u8, /// Log2 of max_pic_order_cnt_lsb (for POC calculation). @@ -25,9 +19,6 @@ impl Default for DpbConfig { fn default() -> Self { Self { dpb_size: DEFAULT_DPB_SIZE, - max_num_ref_frames: DEFAULT_MAX_NUM_REF_FRAMES, - use_multiple_references: true, - max_long_term_refs: 0, log2_max_frame_num_minus4: 0, // max_frame_num = 16 log2_max_pic_order_cnt_lsb_minus4: 0, // max_poc_lsb = 16 num_temporal_layers: 1, @@ -66,10 +57,6 @@ pub struct PictureStartInfo { pub pic_output_flag: bool, /// Whether this is a reference picture. pub is_reference: bool, - /// For H.264: whether to use long-term reference flag (IDR only). - pub long_term_reference_flag: bool, - /// For H.264: adaptive_ref_pic_marking_mode_flag. - pub adaptive_ref_pic_marking_mode: bool, /// For H.264: no_output_of_prior_pics_flag. pub no_output_of_prior_pics_flag: bool, } @@ -85,150 +72,7 @@ impl Default for PictureStartInfo { timestamp: 0, pic_output_flag: true, is_reference: true, - long_term_reference_flag: false, - adaptive_ref_pic_marking_mode: false, no_output_of_prior_pics_flag: false, } } } - -/// Reference Picture Set for H.265 encoding. -/// -/// Contains indices into the DPB for reference pictures categorized by their. -/// position relative to the current picture in display order. -#[derive(Debug, Clone, Default)] -pub struct RefPicSet { - /// Short-term references with POC before current (for L0, closest first). - pub st_curr_before: [i8; super::MAX_REF_LIST_SIZE], - /// Short-term references with POC after current (for L1, closest first). - pub st_curr_after: [i8; super::MAX_REF_LIST_SIZE], - /// Long-term references used by current picture. - pub lt_curr: [i8; super::MAX_REF_LIST_SIZE], - /// Short-term references not used by current but kept in DPB. - pub st_foll: [i8; super::MAX_REF_LIST_SIZE], - /// Long-term references not used by current but kept in DPB. - pub lt_foll: [i8; super::MAX_REF_LIST_SIZE], - /// Count of st_curr_before entries. - pub num_st_curr_before: u8, - /// Count of st_curr_after entries. - pub num_st_curr_after: u8, - /// Count of lt_curr entries. - pub num_lt_curr: u8, - /// Count of st_foll entries. - pub num_st_foll: u8, - /// Count of lt_foll entries. - pub num_lt_foll: u8, -} - -impl RefPicSet { - /// Create a new empty reference picture set. - pub fn new() -> Self { - Self { - st_curr_before: [-1; super::MAX_REF_LIST_SIZE], - st_curr_after: [-1; super::MAX_REF_LIST_SIZE], - lt_curr: [-1; super::MAX_REF_LIST_SIZE], - st_foll: [-1; super::MAX_REF_LIST_SIZE], - lt_foll: [-1; super::MAX_REF_LIST_SIZE], - num_st_curr_before: 0, - num_st_curr_after: 0, - num_lt_curr: 0, - num_st_foll: 0, - num_lt_foll: 0, - } - } - - /// Clear all entries. - pub fn clear(&mut self) { - self.st_curr_before.fill(-1); - self.st_curr_after.fill(-1); - self.lt_curr.fill(-1); - self.st_foll.fill(-1); - self.lt_foll.fill(-1); - self.num_st_curr_before = 0; - self.num_st_curr_after = 0; - self.num_lt_curr = 0; - self.num_st_foll = 0; - self.num_lt_foll = 0; - } - - /// Get the total number of reference pictures used by the current picture. - pub fn num_poc_total_curr(&self) -> u8 { - self.num_st_curr_before + self.num_st_curr_after + self.num_lt_curr - } -} - -/// H.265 Short-term Reference Picture Set data. -#[derive(Debug, Clone, Default)] -pub struct ShortTermRefPicSet { - /// Number of negative delta POC values. - pub num_negative_pics: u8, - /// Number of positive delta POC values. - pub num_positive_pics: u8, - /// Delta POC S0 minus 1 values. - pub delta_poc_s0_minus1: [u8; super::MAX_REF_LIST_SIZE], - /// Used by current pic S0 flag (bitmask). - pub used_by_curr_pic_s0_flag: u16, - /// Delta POC S1 minus 1 values. - pub delta_poc_s1_minus1: [u8; super::MAX_REF_LIST_SIZE], - /// Used by current pic S1 flag (bitmask). - pub used_by_curr_pic_s1_flag: u16, - /// Whether to use inter_ref_pic_set_prediction_flag. - pub inter_ref_pic_set_prediction_flag: bool, -} - -/// H.264 Memory Management Control Operation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MmcoOperation { - /// End of MMCO list. - End, - /// Mark short-term picture as unused for reference. - UnmarkShortTerm { - /// difference_of_pic_nums_minus1. - difference_of_pic_nums_minus1: u32, - }, - /// Mark long-term picture as unused for reference. - UnmarkLongTerm { - /// long_term_pic_num. - long_term_pic_num: u32, - }, - /// Assign long-term frame index to short-term reference. - MarkLongTerm { - /// difference_of_pic_nums_minus1. - difference_of_pic_nums_minus1: u32, - /// long_term_frame_idx. - long_term_frame_idx: u32, - }, - /// Set max long-term frame index. - SetMaxLongTermIndex { - /// max_long_term_frame_idx_plus1. - max_long_term_frame_idx_plus1: u32, - }, - /// Mark all reference pictures as unused. - UnmarkAll, - /// Mark current picture as long-term. - MarkCurrentAsLongTerm { - /// long_term_frame_idx. - long_term_frame_idx: u32, - }, -} - -/// H.264 reference picture list modification entry. -#[derive(Debug, Clone, Copy)] -pub struct RefListModEntry { - /// Modification operation (0, 1, 2, 3). - pub modification_of_pic_nums_idc: u8, - /// abs_diff_pic_num_minus1 or long_term_pic_num. - pub value: u32, -} - -/// H.264 POC type. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum PocType { - /// POC type 0 (explicit signaling via pic_order_cnt_lsb). - #[default] - Type0, - /// POC type 1 (delta POC from frame_num). - Type1, - /// POC type 2 (derived from frame_num). - Type2, -} diff --git a/src/encoder/gop.rs b/src/encoder/gop.rs index da863ef4..8a1bae65 100644 --- a/src/encoder/gop.rs +++ b/src/encoder/gop.rs @@ -1,55 +1,33 @@ //! GOP (Group of Pictures) structure for video encoding. //! //! This module provides GOP structure management for H.264 and H.265 encoders. -//! The GOP structure determines frame types (IDR/I/P/B) and their ordering. +//! The GOP structure determines frame types (IDR or P). /// Frame type in GOP. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GopFrameType { /// IDR frame. Idr, - /// I frame (intra). - I, /// P frame (predicted). P, - /// B frame (bidirectional). - B, } impl GopFrameType { - /// Whether this frame type is a reference. - pub fn is_reference(&self) -> bool { - matches!(self, Self::Idr | Self::I | Self::P) - } - /// Whether this is an IDR frame. pub fn is_idr(&self) -> bool { matches!(self, Self::Idr) } - - /// Whether this is an intra frame (I or IDR). - pub fn is_intra(&self) -> bool { - matches!(self, Self::Idr | Self::I) - } } /// Position in the GOP. #[derive(Debug, Clone)] pub struct GopPosition { - /// Frame number within the GOP (0-based). - pub gop_frame_num: u32, - /// Overall frame index in the sequence. - pub frame_index: u64, /// Frame type. pub frame_type: GopFrameType, /// POC (Picture Order Count). pub pic_order_cnt: i32, - /// Display order (for B-frames). - pub display_order: u64, /// Whether this frame is a reference. pub is_reference: bool, - /// Temporal layer ID. - pub temporal_id: u8, } /// GOP structure manager. @@ -58,10 +36,6 @@ pub struct GopPosition { /// and their ordering. This is codec-agnostic and can be used for both /// H.264 and H.265 encoding. pub struct GopStructure { - /// GOP size (number of frames between IDR frames). - gop_size: u32, - /// Number of B-frames between I/P frames. - b_frame_count: u32, /// Current frame index. frame_index: u64, /// Current frame number. @@ -70,8 +44,6 @@ pub struct GopStructure { poc: i32, /// IDR period. idr_period: u32, - /// IDR frame count. - idr_count: u32, /// Max frame_num value (log2_max_frame_num_minus4 + 4). max_frame_num: u32, /// Max POC LSB value (log2_max_pic_order_cnt_lsb_minus4 + 4). @@ -81,36 +53,19 @@ pub struct GopStructure { } impl GopStructure { - /// Create a new GOP structure. - /// - /// # Arguments - /// * `gop_size` - Number of frames in each GOP. - /// * `b_frame_count` - Number of B-frames between I/P frames (set to 0 for I-P only). - /// * `idr_period` - Period between IDR frames (0 means only first frame is IDR). - pub fn new(gop_size: u32, b_frame_count: u32, idr_period: u32) -> Self { + /// An I-P GOP structure, with an IDR every `gop_size` frames. + pub fn new(gop_size: u32) -> Self { Self { - gop_size: gop_size.max(1), - b_frame_count, + idr_period: gop_size, frame_index: 0, frame_num: 0, poc: 0, - idr_period: if idr_period == 0 { - gop_size - } else { - idr_period - }, - idr_count: 0, max_frame_num: 16, // log2_max_frame_num_minus4=0 -> 2^4 = 16 max_poc_lsb: 16, // log2_max_pic_order_cnt_lsb_minus4=0 -> 2^4 = 16 force_idr: false, } } - /// Create a simple I-P GOP structure. - pub fn new_ip_only(gop_size: u32) -> Self { - Self::new(gop_size, 0, gop_size) - } - /// Set max frame_num (2^(log2_max_frame_num_minus4+4)). pub fn set_max_frame_num(&mut self, log2_max_frame_num_minus4: u8) { self.max_frame_num = 1 << (log2_max_frame_num_minus4 + 4); @@ -134,70 +89,28 @@ impl GopStructure { if is_idr { self.frame_num = 0; self.poc = 0; - self.idr_count += 1; } - // Determine frame type. let frame_type = if is_idr { GopFrameType::Idr - } else if self.b_frame_count > 0 { - // With B-frames, pattern is: I B B P B B P... - let gop_pos = (self.frame_index % self.gop_size as u64) as u32; - if gop_pos == 0 { - GopFrameType::Idr // Start of GOP is always IDR - } else if gop_pos.is_multiple_of(self.b_frame_count + 1) { - GopFrameType::P - } else { - GopFrameType::B - } } else { - // No B-frames, so I-P-P-P-... GopFrameType::P }; let position = GopPosition { - gop_frame_num: (self.frame_index % self.gop_size as u64) as u32, - frame_index: self.frame_index, frame_type, pic_order_cnt: self.poc, - display_order: self.frame_index, - is_reference: frame_type.is_reference(), - temporal_id: if frame_type == GopFrameType::B { 1 } else { 0 }, + is_reference: true, }; // Update counters for next frame. - if frame_type.is_reference() { - self.frame_num = (self.frame_num + 1) % self.max_frame_num; - } + self.frame_num = (self.frame_num + 1) % self.max_frame_num; self.poc = (self.poc + 2) % (self.max_poc_lsb as i32 * 2); self.frame_index += 1; position } - /// Get current frame number. - pub fn current_frame_num(&self) -> u32 { - self.frame_num - } - - /// Get current POC. - pub fn current_poc(&self) -> i32 { - self.poc - } - - /// Get total frames encoded. - pub fn total_frames(&self) -> u64 { - self.frame_index - } - - /// Reset the GOP structure. - pub fn reset(&mut self) { - self.frame_index = 0; - self.frame_num = 0; - self.poc = 0; - self.idr_count = 0; - } - /// Request that the next frame be an IDR frame. pub fn request_idr(&mut self) { self.force_idr = true; @@ -210,25 +123,22 @@ mod tests { #[test] fn test_ip_only_gop() { - let mut gop = GopStructure::new_ip_only(30); + let mut gop = GopStructure::new(30); // First frame should be IDR. let pos = gop.get_next_frame(); assert_eq!(pos.frame_type, GopFrameType::Idr); - assert_eq!(pos.frame_index, 0); assert!(pos.is_reference); // Next frames should be P. for i in 1..30 { let pos = gop.get_next_frame(); assert_eq!(pos.frame_type, GopFrameType::P, "Frame {i} should be P"); - assert_eq!(pos.frame_index, i); assert!(pos.is_reference); } // Frame 30 should be IDR (new GOP) let pos = gop.get_next_frame(); assert_eq!(pos.frame_type, GopFrameType::Idr); - assert_eq!(pos.frame_index, 30); } } diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index fce70b00..2cc32a8b 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -79,9 +79,6 @@ impl H264 { let mut dpb = DecodedPictureBuffer::new(); dpb.h264.sequence_start(DpbConfig { dpb_size: common.dpb_slot_count as u32, - max_num_ref_frames: if config.b_frame_count > 0 { 2 } else { 1 }, - use_multiple_references: config.b_frame_count > 0, - max_long_term_refs: 0, log2_max_frame_num_minus4: 4, log2_max_pic_order_cnt_lsb_minus4: 4, num_temporal_layers: 1, @@ -91,10 +88,6 @@ impl H264 { dpb, frame_num_syntax: 0, idr_pic_id: 0, - has_backward_reference: false, - backward_reference_frame_num: 0, - backward_reference_poc: 0, - backward_reference_dpb_slot: 2, l0_references: Vec::new(), reference_window, active_reference_count, diff --git a/src/encoder/h264/mod.rs b/src/encoder/h264/mod.rs index f1f0732e..9edaab2a 100644 --- a/src/encoder/h264/mod.rs +++ b/src/encoder/h264/mod.rs @@ -38,11 +38,6 @@ pub(crate) struct H264 { frame_num_syntax: u32, /// IDR picture id, toggled each IDR. idr_pic_id: u32, - /// Whether an L1 (backward) reference is available, for B-frames. - has_backward_reference: bool, - backward_reference_frame_num: u32, - backward_reference_poc: i32, - backward_reference_dpb_slot: u8, /// References held in the DPB, most-recent first. Each frame predicts /// from the first `active_reference_count`; the rest are kept for /// reference frame invalidation to fall back on. @@ -73,15 +68,12 @@ impl VideoCodec for H264 { // Reset the DPB for the new coded video sequence. let dpb_config = DpbConfig { dpb_size: common.dpb_slot_count as u32, - max_num_ref_frames: self.reference_window, - use_multiple_references: common.config.b_frame_count > 0, log2_max_frame_num_minus4: 4, log2_max_pic_order_cnt_lsb_minus4: 4, ..Default::default() }; self.dpb.h264.sequence_start(dpb_config); self.l0_references.clear(); - self.has_backward_reference = false; // An IDR resets the decoder's whole DPB, so any deferred // unused-reference markings are moot. self.pending_unmark_frame_nums.clear(); @@ -113,15 +105,13 @@ impl VideoCodec for H264 { let frame_num = self.frame_num_syntax; let pic_order_cnt = plan.pic_order_cnt(); - if plan.is_reference() && !plan.is_b_frame() { + if plan.is_reference() { self.frame_num_syntax = (frame_num + 1) % 256; } if plan.is_reference() { let pic_type = if plan.is_idr() { PictureType::Idr - } else if plan.is_b_frame() { - PictureType::B } else { PictureType::P }; @@ -181,8 +171,6 @@ impl VideoCodec for H264 { } } self.l0_references = survivors; - // A backward (L1) reference may also be tainted; only B-frames use it. - self.has_backward_reference = false; !self.l0_references.is_empty() } diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index eb8bf1ca..8d85252e 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -21,7 +21,6 @@ impl H264 { ) -> Result { let is_idr = plan.is_idr(); let is_reference = plan.is_reference(); - let is_b_frame = plan.is_b_frame(); let pic_order_cnt = plan.pic_order_cnt(); let frame_num = self.frame_num_syntax; @@ -103,16 +102,12 @@ impl H264 { let slice_type = if is_idr { ash::vk::native::StdVideoH264SliceType_STD_VIDEO_H264_SLICE_TYPE_I - } else if is_b_frame { - ash::vk::native::StdVideoH264SliceType_STD_VIDEO_H264_SLICE_TYPE_B } else { ash::vk::native::StdVideoH264SliceType_STD_VIDEO_H264_SLICE_TYPE_P }; let picture_type = if is_idr { ash::vk::native::StdVideoH264PictureType_STD_VIDEO_H264_PICTURE_TYPE_IDR - } else if is_b_frame { - ash::vk::native::StdVideoH264PictureType_STD_VIDEO_H264_PICTURE_TYPE_B } else { ash::vk::native::StdVideoH264PictureType_STD_VIDEO_H264_PICTURE_TYPE_P }; @@ -167,7 +162,7 @@ impl H264 { // STD_VIDEO_H264_NO_REFERENCE_PICTURE = 0xFF. const NO_REFERENCE_PICTURE: u8 = 0xFF; let mut ref_list0: [u8; 32] = [NO_REFERENCE_PICTURE; 32]; - let mut ref_list1: [u8; 32] = [NO_REFERENCE_PICTURE; 32]; + let ref_list1: [u8; 32] = [NO_REFERENCE_PICTURE; 32]; let ref_lists_info_flags = ash::vk::native::StdVideoEncodeH264ReferenceListsInfoFlags { _bitfield_align_1: [], @@ -178,16 +173,8 @@ impl H264 { ), }; - // Set up reference lists for P-frames and B-frames. - let (num_ref_l0, num_ref_l1) = if is_b_frame && self.has_backward_reference { - if let Some(first_ref) = self.l0_references.first() { - ref_list0[0] = first_ref.dpb_slot; - ref_list1[0] = self.backward_reference_dpb_slot; - (1, 1) - } else { - (0, 0) - } - } else if !is_idr && !self.l0_references.is_empty() { + // Set up the reference list for P-frames. + let num_ref_l0 = if !is_idr && !self.l0_references.is_empty() { let actual_count = self .l0_references .len() @@ -196,9 +183,9 @@ impl H264 { for (i, ref_info) in self.l0_references.iter().take(actual_count).enumerate() { ref_list0[i] = ref_info.dpb_slot; } - (actual_count, 0) + actual_count } else { - (0, 0) + 0 }; let ref_lists_info = ash::vk::native::StdVideoEncodeH264ReferenceListsInfo { @@ -208,11 +195,7 @@ impl H264 { } else { 0 }, - num_ref_idx_l1_active_minus1: if num_ref_l1 > 0 { - (num_ref_l1 - 1) as u8 - } else { - 0 - }, + num_ref_idx_l1_active_minus1: 0, RefPicList0: ref_list0, RefPicList1: ref_list1, refList0ModOpCount: 0, @@ -324,46 +307,6 @@ impl H264 { ); } - // Backward (L1) reference for B-frames. - let (backward_resource, backward_std_info) = if is_b_frame && self.has_backward_reference { - let image_view = common.dpb_image_views[self.backward_reference_dpb_slot as usize]; - let resource = vk::VideoPictureResourceInfoKHR::default() - .coded_offset(vk::Offset2D { x: 0, y: 0 }) - .coded_extent(coded_extent) - .base_array_layer(0) - .image_view_binding(image_view); - let std_info = ash::vk::native::StdVideoEncodeH264ReferenceInfo { - flags: std_reference_info_flags, - primary_pic_type: - ash::vk::native::StdVideoH264PictureType_STD_VIDEO_H264_PICTURE_TYPE_P, - FrameNum: self.backward_reference_frame_num, - PicOrderCnt: self.backward_reference_poc, - long_term_pic_num: 0, - long_term_frame_idx: 0, - temporal_id: 0, - }; - (Some(resource), Some(std_info)) - } else { - (None, None) - }; - - let mut backward_dpb_info = if let Some(ref std_info) = backward_std_info { - vk::VideoEncodeH264DpbSlotInfoKHR::default().std_reference_info(std_info) - } else { - vk::VideoEncodeH264DpbSlotInfoKHR::default() - }; - - let backward_ref_slot = if let Some(ref resource) = backward_resource { - Some( - vk::VideoReferenceSlotInfoKHR::default() - .slot_index(self.backward_reference_dpb_slot as i32) - .picture_resource(resource) - .push(&mut backward_dpb_info), - ) - } else { - None - }; - // Reference info for the setup slot (this frame). let std_reference_info = ash::vk::native::StdVideoEncodeH264ReferenceInfo { flags: std_reference_info_flags, @@ -395,14 +338,7 @@ impl H264 { // Reference slots for the encode command. let mut encode_ref_slots = Vec::new(); - if is_b_frame && self.has_backward_reference { - if let Some(l0) = l0_slots.first() { - encode_ref_slots.push(*l0); - } - if let Some(l1) = backward_ref_slot { - encode_ref_slots.push(l1); - } - } else if !is_idr { + if !is_idr { encode_ref_slots.extend_from_slice(&l0_slots); } @@ -419,14 +355,7 @@ impl H264 { // Reference slots for begin coding (setup slot is marked inactive, -1). let mut reference_slots_for_begin = vec![setup_slot_for_begin]; - if is_b_frame && self.has_backward_reference { - if let Some(l0) = l0_slots.first() { - reference_slots_for_begin.push(*l0); - } - if let Some(l1) = backward_ref_slot { - reference_slots_for_begin.push(l1); - } - } else if !is_idr { + if !is_idr { reference_slots_for_begin.extend_from_slice(&l0_slots); } @@ -459,8 +388,7 @@ impl H264 { let mut h264_rc_info = vk::VideoEncodeH264RateControlInfoKHR::default() .gop_frame_count(common.config.gop_size) - .idr_period(common.config.gop_size) - .consecutive_b_frame_count(common.config.b_frame_count); + .idr_period(common.config.gop_size); let mut rc_info = vk::VideoEncodeRateControlInfoKHR::default().rate_control_mode(rc.mode); if !rc.is_disabled() { diff --git a/src/encoder/h264/session_params.rs b/src/encoder/h264/session_params.rs index 6320341d..74c0a116 100644 --- a/src/encoder/h264/session_params.rs +++ b/src/encoder/h264/session_params.rs @@ -100,7 +100,7 @@ impl H264 { matrix_coefficients: desc.matrix_coefficients, num_units_in_tick: 0, time_scale: 0, - max_num_reorder_frames: if config.b_frame_count > 0 { 1 } else { 0 }, + max_num_reorder_frames: 0, max_dec_frame_buffering: window + 1, chroma_sample_loc_type_top_field: 0, chroma_sample_loc_type_bottom_field: 0, @@ -117,11 +117,7 @@ impl H264 { bit_depth_luma_minus8, bit_depth_chroma_minus8, log2_max_frame_num_minus4: 4, - pic_order_cnt_type: if config.b_frame_count > 0 { - ash::vk::native::StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_0 - } else { - ash::vk::native::StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_2 - }, + pic_order_cnt_type: ash::vk::native::StdVideoH264PocType_STD_VIDEO_H264_POC_TYPE_2, offset_for_non_ref_pic: 0, offset_for_top_to_bottom_field: 0, log2_max_pic_order_cnt_lsb_minus4: 4, diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index 88bfcac0..8fb8ddfd 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -77,9 +77,6 @@ impl H265 { let mut dpb = DecodedPictureBuffer::new(); dpb.h265.sequence_start(DpbConfig { dpb_size: common.dpb_slot_count as u32, - max_num_ref_frames: if config.b_frame_count > 0 { 2 } else { 1 }, - use_multiple_references: config.b_frame_count > 0, - max_long_term_refs: 0, log2_max_frame_num_minus4: 0, log2_max_pic_order_cnt_lsb_minus4: 4, num_temporal_layers: 1, @@ -88,9 +85,6 @@ impl H265 { let codec = H265 { dpb, header_data: None, - has_backward_reference: false, - backward_reference_poc: 0, - backward_reference_dpb_slot: 2, l0_references: Vec::new(), reference_window, active_reference_count, diff --git a/src/encoder/h265/mod.rs b/src/encoder/h265/mod.rs index 8eff3e17..aca6a2d0 100644 --- a/src/encoder/h265/mod.rs +++ b/src/encoder/h265/mod.rs @@ -33,9 +33,6 @@ pub(crate) struct H265 { dpb: DecodedPictureBuffer, /// Cached VPS/SPS/PPS header (invalidated when session params change). header_data: Option>, - has_backward_reference: bool, - backward_reference_poc: i32, - backward_reference_dpb_slot: u8, /// References held in the DPB, most-recent first. Each frame predicts /// from the first `active_reference_count`; the rest stay in its /// reference picture set, unused, for reference frame invalidation to fall @@ -68,15 +65,12 @@ impl VideoCodec for H265 { if plan.is_idr() { let dpb_config = DpbConfig { dpb_size: common.dpb_slot_count as u32, - max_num_ref_frames: self.reference_window, - use_multiple_references: common.config.b_frame_count > 0, log2_max_frame_num_minus4: 0, log2_max_pic_order_cnt_lsb_minus4: 4, ..Default::default() }; self.dpb.h265.sequence_start(dpb_config); self.l0_references.clear(); - self.has_backward_reference = false; // All DPB slots become inactive at the start of a coded video sequence. common.dpb_slot_active.fill(false); } @@ -111,8 +105,6 @@ impl VideoCodec for H265 { let pic_order_cnt = plan.pic_order_cnt(); let pic_type = if plan.is_idr() { PictureType::Idr - } else if plan.is_b_frame() { - PictureType::B } else { PictureType::P }; @@ -139,13 +131,11 @@ impl VideoCodec for H265 { self.l0_references.pop(); } - if !plan.is_b_frame() { - let used_slots: Vec = self.l0_references.iter().map(|r| r.dpb_slot).collect(); - for i in 0..common.dpb_slot_count as u8 { - if !used_slots.contains(&i) { - common.current_dpb_slot = i; - break; - } + let used_slots: Vec = self.l0_references.iter().map(|r| r.dpb_slot).collect(); + for i in 0..common.dpb_slot_count as u8 { + if !used_slots.contains(&i) { + common.current_dpb_slot = i; + break; } } } @@ -162,9 +152,6 @@ impl VideoCodec for H265 { // the decoder to follow — no extra bitstream marking is required. self.l0_references .retain(|r| r.display_order < first_lost_display_order); - // A backward (L1) reference may also be tainted; B-frames are the only - // consumer of it, so simply forget it on invalidation. - self.has_backward_reference = false; !self.l0_references.is_empty() } diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index a8172bb9..47323a57 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -22,7 +22,6 @@ impl H265 { ) -> Result { let is_idr = plan.is_idr(); let is_reference = plan.is_reference(); - let is_b_frame = plan.is_b_frame(); let pic_order_cnt = plan.pic_order_cnt(); let slot = common.pipeline.current(); @@ -55,15 +54,11 @@ impl H265 { let slice_type = if is_idr { ash::vk::native::StdVideoH265SliceType_STD_VIDEO_H265_SLICE_TYPE_I - } else if is_b_frame { - ash::vk::native::StdVideoH265SliceType_STD_VIDEO_H265_SLICE_TYPE_B } else { ash::vk::native::StdVideoH265SliceType_STD_VIDEO_H265_SLICE_TYPE_P }; let picture_type = if is_idr { ash::vk::native::StdVideoH265PictureType_STD_VIDEO_H265_PICTURE_TYPE_IDR - } else if is_b_frame { - ash::vk::native::StdVideoH265PictureType_STD_VIDEO_H265_PICTURE_TYPE_B } else { ash::vk::native::StdVideoH265PictureType_STD_VIDEO_H265_PICTURE_TYPE_P }; @@ -107,11 +102,8 @@ impl H265 { // Short-term reference picture set: negative (past) refs for P, plus a // positive (future) ref for B. let mut delta_poc_s0_minus1 = [0u16; 16]; - let mut delta_poc_s1_minus1 = [0u16; 16]; let mut num_negative_pics: u8 = 0; - let mut num_positive_pics: u8 = 0; let mut used_by_curr_pic_s0_flag: u16 = 0; - let mut used_by_curr_pic_s1_flag: u16 = 0; if !is_idr && !self.l0_references.is_empty() { // max_poc = 2^(log2_max_pic_order_cnt_lsb_minus4 + 4) * 2 = 512. @@ -137,12 +129,6 @@ impl H265 { } num_negative_pics += 1; } - if is_b_frame && self.has_backward_reference { - let delta_poc_l1 = self.backward_reference_poc - pic_order_cnt; - delta_poc_s1_minus1[0] = (delta_poc_l1 - 1).max(0) as u16; - num_positive_pics = 1; - used_by_curr_pic_s1_flag = 1; - } } let frame_rps = ash::vk::native::StdVideoH265ShortTermRefPicSet { @@ -158,14 +144,14 @@ impl H265 { abs_delta_rps_minus1: 0, used_by_curr_pic_flag: 0, used_by_curr_pic_s0_flag, - used_by_curr_pic_s1_flag, + used_by_curr_pic_s1_flag: 0, reserved1: 0, reserved2: 0, reserved3: 0, num_negative_pics, - num_positive_pics, + num_positive_pics: 0, delta_poc_s0_minus1, - delta_poc_s1_minus1, + delta_poc_s1_minus1: [0; 16], }; let empty_rps = ash::vk::native::StdVideoH265ShortTermRefPicSet { @@ -209,26 +195,18 @@ impl H265 { const NO_REFERENCE_PICTURE: u8 = 0xFF; let mut ref_list0: [u8; 15] = [NO_REFERENCE_PICTURE; 15]; - let mut ref_list1: [u8; 15] = [NO_REFERENCE_PICTURE; 15]; + let ref_list1: [u8; 15] = [NO_REFERENCE_PICTURE; 15]; - let (num_ref_l0, num_ref_l1) = if is_b_frame && self.has_backward_reference { - if let Some(first_ref) = self.l0_references.first() { - ref_list0[0] = first_ref.dpb_slot; - ref_list1[0] = self.backward_reference_dpb_slot; - (1, 1) - } else { - (0, 0) - } - } else if !is_idr && !self.l0_references.is_empty() { + let num_ref_l0 = if !is_idr && !self.l0_references.is_empty() { let count = used_references.len(); for (i, ref_info) in used_references.iter().enumerate() { if i < 15 { ref_list0[i] = ref_info.dpb_slot; } } - (count.min(15), 0) + count.min(15) } else { - (0, 0) + 0 }; let ref_lists_info = ash::vk::native::StdVideoEncodeH265ReferenceListsInfo { @@ -244,11 +222,7 @@ impl H265 { } else { 0 }, - num_ref_idx_l1_active_minus1: if num_ref_l1 > 0 { - (num_ref_l1 - 1) as u8 - } else { - 0 - }, + num_ref_idx_l1_active_minus1: 0, RefPicList0: ref_list0, RefPicList1: ref_list1, list_entry_l0: [0; 15], @@ -343,7 +317,6 @@ impl H265 { reference_slots_for_begin.push(setup_slot_for_begin); let has_l0_ref = !is_idr && !self.l0_references.is_empty(); - let has_l1_ref = is_b_frame && self.has_backward_reference; if has_l0_ref { for ref_info in used_references { @@ -361,22 +334,6 @@ impl H265 { std_ref_infos.push(std_info); } } - if has_l1_ref { - ref_resources.push( - vk::VideoPictureResourceInfoKHR::default() - .coded_offset(vk::Offset2D { x: 0, y: 0 }) - .coded_extent(coded_extent) - .base_array_layer(0) - .image_view_binding( - common.dpb_image_views[self.backward_reference_dpb_slot as usize], - ), - ); - let mut std_info = std_reference_info; - std_info.PicOrderCntVal = self.backward_reference_poc; - std_info.pic_type = - ash::vk::native::StdVideoH265PictureType_STD_VIDEO_H265_PICTURE_TYPE_P; - std_ref_infos.push(std_info); - } for std_info in &std_ref_infos { h265_slot_infos @@ -398,14 +355,6 @@ impl H265 { } } } - if has_l1_ref && let Some(h265_slot_info) = slot_infos_iter.next() { - let ref_slot = vk::VideoReferenceSlotInfoKHR::default() - .slot_index(self.backward_reference_dpb_slot as i32) - .picture_resource(&ref_resources[stored_indices_count]) - .push(h265_slot_info); - reference_slots.push(ref_slot); - reference_slots_for_begin.push(ref_slot); - } // Rate control. let qp_min = if rc.is_disabled() { rc.qp as i32 } else { 26 }; @@ -433,8 +382,7 @@ impl H265 { let mut h265_rc_info = vk::VideoEncodeH265RateControlInfoKHR::default() .gop_frame_count(common.config.gop_size) - .idr_period(common.config.gop_size) - .consecutive_b_frame_count(common.config.b_frame_count); + .idr_period(common.config.gop_size); let mut rc_info = vk::VideoEncodeRateControlInfoKHR::default().rate_control_mode(rc.mode); if !rc.is_disabled() { rc_info = rc_info diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index b732aa14..b15757c6 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -3,24 +3,21 @@ //! This module provides: //! - Core encoder types and configuration (`EncodeConfig`, `EncodedPacket`, etc.) //! - GOP structure management (`gop` module) - reusable for H.264/H.265. -//! - Frame reordering for B-frame support (`reorder` module) - reusable for H.264/H.265. mod api; -pub mod av1; -pub mod bitwriter; +pub(crate) mod av1; pub(crate) mod codec; -pub mod dpb; +pub(crate) mod dpb; #[cfg(feature = "dmabuf")] pub mod external; mod frame; -pub mod gop; -pub mod h264; -pub mod h265; +pub(crate) mod gop; +pub(crate) mod h264; +pub(crate) mod h265; pub(crate) mod input; pub(crate) mod pipeline; pub(crate) mod profile; -pub mod reorder; -pub mod resources; +pub(crate) mod resources; pub(crate) mod rgb; mod source; mod source_image; @@ -410,9 +407,6 @@ pub struct EncodeConfig { pub frame_rate_denominator: u32, /// GOP size (distance between IDR frames). pub gop_size: u32, - /// Number of consecutive B-frames. Always 0: no codec encodes B-frames - /// yet, so there is no setter. - pub(crate) b_frame_count: u32, /// Maximum number of reference frames kept in the DPB: the window that /// [`Encoder::invalidate_reference_frames`](crate::Encoder::invalidate_reference_frames) /// can recover from. Each frame predicts from only as many of the newest @@ -464,7 +458,6 @@ impl EncodeConfig { frame_rate_numerator: DEFAULT_FRAME_RATE, frame_rate_denominator: 1, gop_size: DEFAULT_GOP_SIZE, - b_frame_count: 0, // Start without B-frames for simplicity. max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, @@ -491,7 +484,6 @@ impl EncodeConfig { frame_rate_numerator: DEFAULT_FRAME_RATE, frame_rate_denominator: 1, gop_size: DEFAULT_GOP_SIZE, - b_frame_count: 0, // Start without B-frames for simplicity. max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, @@ -518,7 +510,6 @@ impl EncodeConfig { frame_rate_numerator: DEFAULT_FRAME_RATE, frame_rate_denominator: 1, gop_size: DEFAULT_GOP_SIZE, - b_frame_count: 0, // Start without B-frames for simplicity. max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, @@ -806,7 +797,6 @@ mod tests { assert_eq!(config.bit_depth, BitDepth::Eight); assert_eq!(config.rate_control, RateControl::Cqp { qp: 26 }); assert_eq!(config.gop_size, 30); - assert_eq!(config.b_frame_count, 0); assert_eq!(config.frame_rate_numerator, 30); assert_eq!(config.frame_rate_denominator, 1); } @@ -878,7 +868,6 @@ mod tests { // AV1 uses a 0-255 quantizer index. assert_eq!(config.rate_control, RateControl::Cqp { qp: 128 }); assert_eq!(config.gop_size, 30); - assert_eq!(config.b_frame_count, 0); assert_eq!(config.frame_rate_numerator, 30); assert_eq!(config.frame_rate_denominator, 1); } diff --git a/src/encoder/reorder.rs b/src/encoder/reorder.rs deleted file mode 100644 index 87bd62e8..00000000 --- a/src/encoder/reorder.rs +++ /dev/null @@ -1,328 +0,0 @@ -//! Frame reordering buffer for B-frame support. -//! -//! This module handles frame reordering for video encoding with B-frames. -//! Frames arrive in display order but must be encoded in a different order. -//! because B-frames reference both past and future frames. -//! -//! This is codec-agnostic and can be used for both H.264 and H.265 encoding. -//! -//! For example, with 1 B-frame between P-frames: -//! - Display order: I(0), B(1), P(2), B(3), P(4), ... -//! - Encode order: I(0), P(2), B(1), P(4), B(3), ... -//! -//! The P-frame must be encoded before the B-frame that references it. - -use super::gop::GopFrameType; - -/// Represents a buffered frame waiting to be encoded. -#[derive(Clone)] -pub struct BufferedFrame { - /// Raw frame data (YUV). - pub data: Vec, - /// Display order (input order). - pub display_order: u64, - /// Frame type determined by GOP structure. - pub frame_type: GopFrameType, - /// Picture Order Count for this frame. - pub poc: i32, - /// Whether this frame should be used as a reference. - pub is_reference: bool, -} - -/// Frame ready to be encoded (output from reorder buffer). -pub struct FrameToEncode { - /// Raw frame data. - pub data: Vec, - /// Display order. - pub display_order: u64, - /// Encode order (order in which this frame is encoded). - pub encode_order: u64, - /// Frame type. - pub frame_type: GopFrameType, - /// POC. - pub poc: i32, - /// Whether this is a reference frame. - pub is_reference: bool, - /// For B-frames: display order of the forward reference (L0). - pub forward_ref_display_order: Option, - /// For B-frames: display order of the backward reference (L1). - pub backward_ref_display_order: Option, -} - -/// Frame reorder buffer for B-frame encoding. -/// -/// Handles the reordering of frames from display order to encode order. -/// This is codec-agnostic and works for both H.264 and H.265 encoding. -pub struct FrameReorderBuffer { - /// Number of consecutive B-frames between anchor frames. - b_frame_count: u32, - /// Buffer holding frames waiting to be encoded. - buffer: Vec, - /// Next encode order to assign. - next_encode_order: u64, - /// Display order of the last anchor frame (I or P). - last_anchor_display_order: Option, - /// Whether we're flushing (no more input frames). - flushing: bool, -} - -impl FrameReorderBuffer { - /// Create a new frame reorder buffer. - /// - /// # Arguments - /// * `b_frame_count` - Number of consecutive B-frames between I/P frames. - pub fn new(b_frame_count: u32) -> Self { - Self { - b_frame_count, - buffer: Vec::with_capacity((b_frame_count + 2) as usize), - next_encode_order: 0, - last_anchor_display_order: None, - flushing: false, - } - } - - /// Submit a frame in display order. - /// - /// Returns frames ready to encode (may be 0 or more). - pub fn submit_frame( - &mut self, - data: Vec, - display_order: u64, - frame_type: GopFrameType, - poc: i32, - is_reference: bool, - ) -> Vec { - // Add frame to buffer. - self.buffer.push(BufferedFrame { - data, - display_order, - frame_type, - poc, - is_reference, - }); - - // Get frames ready to encode. - self.get_frames_to_encode() - } - - /// Flush remaining frames when encoding ends. - /// - /// Call this when there are no more input frames to get any remaining. - /// buffered frames. - pub fn flush(&mut self) -> Vec { - self.flushing = true; - self.get_frames_to_encode() - } - - /// Get frames that are ready to be encoded. - fn get_frames_to_encode(&mut self) -> Vec { - if self.b_frame_count == 0 { - // No B-frames: just output frames in order. - return self.drain_all_frames(); - } - - let mut result = Vec::new(); - - // For B-frame encoding, we need to wait for the anchor frame. - // before we can encode the B-frames that precede it. - // - // Pattern: I, B, B, P, B, B, P, ... - // When we receive P, we can encode P first, then the B-frames before it. - - loop { - // Find if we have an anchor frame (I or P) after some B-frames. - let anchor_pos = self.buffer.iter().position(|f| { - f.frame_type == GopFrameType::Idr - || f.frame_type == GopFrameType::I - || f.frame_type == GopFrameType::P - }); - - match anchor_pos { - Some(0) => { - // First frame is anchor - output it immediately. - let frame = self.buffer.remove(0); - let encode_order = self.next_encode_order; - self.next_encode_order += 1; - self.last_anchor_display_order = Some(frame.display_order); - - result.push(FrameToEncode { - data: frame.data, - display_order: frame.display_order, - encode_order, - frame_type: frame.frame_type, - poc: frame.poc, - is_reference: frame.is_reference, - forward_ref_display_order: None, - backward_ref_display_order: None, - }); - } - Some(pos) if pos > 0 => { - // We have B-frames followed by an anchor. - // First, output the anchor (it needs to be encoded before B-frames) - let anchor = self.buffer.remove(pos); - let anchor_display_order = anchor.display_order; - let anchor_encode_order = self.next_encode_order; - self.next_encode_order += 1; - - result.push(FrameToEncode { - data: anchor.data, - display_order: anchor.display_order, - encode_order: anchor_encode_order, - frame_type: anchor.frame_type, - poc: anchor.poc, - is_reference: anchor.is_reference, - forward_ref_display_order: self.last_anchor_display_order, - backward_ref_display_order: None, - }); - - // Now output the B-frames that were before the anchor. - // They reference the previous anchor (L0) and this anchor (L1) - let forward_ref = self.last_anchor_display_order; - let backward_ref = Some(anchor_display_order); - - // Drain the B-frames (now at positions 0..pos-1 since we removed anchor) - for _ in 0..(pos) { - if self.buffer.is_empty() { - break; - } - let b_frame = self.buffer.remove(0); - let encode_order = self.next_encode_order; - self.next_encode_order += 1; - - result.push(FrameToEncode { - data: b_frame.data, - display_order: b_frame.display_order, - encode_order, - frame_type: b_frame.frame_type, - poc: b_frame.poc, - is_reference: b_frame.is_reference, - forward_ref_display_order: forward_ref, - backward_ref_display_order: backward_ref, - }); - } - - self.last_anchor_display_order = Some(anchor_display_order); - } - None if self.flushing && !self.buffer.is_empty() => { - // Flushing: output remaining B-frames. - // They can only reference the previous anchor (L0 only) - let forward_ref = self.last_anchor_display_order; - - while !self.buffer.is_empty() { - let frame = self.buffer.remove(0); - let encode_order = self.next_encode_order; - self.next_encode_order += 1; - - result.push(FrameToEncode { - data: frame.data, - display_order: frame.display_order, - encode_order, - frame_type: frame.frame_type, - poc: frame.poc, - is_reference: frame.is_reference, - forward_ref_display_order: forward_ref, - // No backward ref available when flushing. - backward_ref_display_order: None, - }); - } - } - _ => break, // No complete group ready - } - } - - result - } - - /// Drain all frames without B-frame reordering. - fn drain_all_frames(&mut self) -> Vec { - self.buffer - .drain(..) - .map(|frame| { - let encode_order = self.next_encode_order; - self.next_encode_order += 1; - - let forward_ref = if frame.frame_type == GopFrameType::P { - self.last_anchor_display_order - } else { - None - }; - - if frame.frame_type.is_reference() { - self.last_anchor_display_order = Some(frame.display_order); - } - - FrameToEncode { - data: frame.data, - display_order: frame.display_order, - encode_order, - frame_type: frame.frame_type, - poc: frame.poc, - is_reference: frame.is_reference, - forward_ref_display_order: forward_ref, - backward_ref_display_order: None, - } - }) - .collect() - } - - /// Check if buffer is empty. - pub fn is_empty(&self) -> bool { - self.buffer.is_empty() - } - - /// Get number of buffered frames. - pub fn len(&self) -> usize { - self.buffer.len() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_no_b_frames() { - let mut buffer = FrameReorderBuffer::new(0); - - // Submit I, P, P. - let frames = buffer.submit_frame(vec![1], 0, GopFrameType::Idr, 0, true); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].display_order, 0); - assert_eq!(frames[0].encode_order, 0); - - let frames = buffer.submit_frame(vec![2], 1, GopFrameType::P, 2, true); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].display_order, 1); - assert_eq!(frames[0].encode_order, 1); - } - - #[test] - fn test_one_b_frame() { - let mut buffer = FrameReorderBuffer::new(1); - - // Submit I, B, P in display order. - let frames = buffer.submit_frame(vec![1], 0, GopFrameType::Idr, 0, true); - // I is output immediately. - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].frame_type, GopFrameType::Idr); - assert_eq!(frames[0].encode_order, 0); - - // Submit B - should buffer. - let frames = buffer.submit_frame(vec![2], 1, GopFrameType::B, 2, false); - assert_eq!(frames.len(), 0); // B is buffered - - // Submit P - should output P then B. - let frames = buffer.submit_frame(vec![3], 2, GopFrameType::P, 4, true); - assert_eq!(frames.len(), 2); - // First: P (encode order 1) - assert_eq!(frames[0].frame_type, GopFrameType::P); - assert_eq!(frames[0].display_order, 2); - assert_eq!(frames[0].encode_order, 1); - // Second: B (encode order 2) - assert_eq!(frames[1].frame_type, GopFrameType::B); - assert_eq!(frames[1].display_order, 1); - assert_eq!(frames[1].encode_order, 2); - assert_eq!(frames[1].forward_ref_display_order, Some(0)); // References I - assert_eq!(frames[1].backward_ref_display_order, Some(2)); // References P - } -} From eff2d3e00cb41957c571084f75442b9d1f5ad518 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 07:50:01 +0100 Subject: [PATCH 2/9] refactor(image)!: replace the InputImage uploads with upload and upload_to --- CHANGELOG.md | 12 +- README.md | 18 +- benches/encode.rs | 17 +- examples/encode.rs | 17 +- src/encoder/api.rs | 16 +- src/image.rs | 498 +++++++++++++------------------------------- src/lib.rs | 20 +- tests/dmabuf.rs | 18 +- tests/rfi.rs | 13 +- tests/roundtrip.rs | 14 +- tests/submit.rs | 25 +-- tests/upload.rs | 139 +++++++++++++ tests/verify_all.rs | 14 +- 13 files changed, 336 insertions(+), 485 deletions(-) create mode 100644 tests/upload.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ef403d..a5235bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,9 +62,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `EncodeConfig::with_b_frames` and the public `b_frame_count` field are removed. No codec encodes B-frames yet, and every encoder refused a count above 0 at runtime. -- **Breaking:** `InputImage::upload_yuv420_to` and `upload_yuv444_to` take an - `InputSlot` instead of a `vk::Image`. `InputImage` uploads wait until the - frames that read the image are done with it. +- **Breaking:** `InputImage` has two uploads, `upload` to its own image and + `upload_to` an encoder's `InputSlot`, which take a `YuvLayout`: `I420`, + `Nv12`, `I420P10` or `I444`. They replace `upload_yuv420`, `upload_nv12`, + `upload_yuv420_10bit`, `upload_yuv420_to` and `upload_yuv444_to`, which + covered only some of the combinations. A layout the image cannot hold is + an error; `upload_nv12` panicked for a 10-bit or 4:4:4 image. The uploads + wait until the frames that read the image are done with it. +- **Breaking:** `InputImage::new` no longer takes a `Codec`, which it did not + use. `InputImage::for_encoder` makes one for an encoder's frames. - **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the encode API: `EncodeConfig::with_color` replaces `with_color_description`, `Encoder::reconfigure` replaces `set_color_description`, and diff --git a/README.md b/README.md index 26284fdf..f1377dd5 100644 --- a/README.md +++ b/README.md @@ -96,10 +96,7 @@ fn main() -> Result<(), Box> { ### Encoding Video ```rust -use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControl, - VideoContextBuilder, -}; +use pixelforge::{Codec, EncodeConfig, Encoder, InputImage, RateControl, VideoContextBuilder}; fn main() -> Result<(), Box> { let context = VideoContextBuilder::new() @@ -115,20 +112,13 @@ fn main() -> Result<(), Box> { .with_frame_rate(30, 1) .with_gop_size(60); - // Create an InputImage helper for uploading YUV data to the GPU. - let mut input_image = InputImage::new( - context.clone(), - Codec::H264, - 1920, - 1080, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; let mut encoder = Encoder::new(context, config)?; + // Create an InputImage helper for uploading YUV data to the GPU. + let mut input_image = InputImage::for_encoder(&encoder)?; // For each frame: upload YUV data and encode. // let yuv_data: &[u8] = ...; // YUV420 frame data - // input_image.upload_yuv420(yuv_data)?; + // input_image.upload(YuvLayout::I420, yuv_data)?; // let packet = encoder.submit(Frame::image(input_image.source()))?; Ok(()) diff --git a/benches/encode.rs b/benches/encode.rs index 30441748..7995d6b5 100644 --- a/benches/encode.rs +++ b/benches/encode.rs @@ -9,8 +9,8 @@ //! cargo bench --bench encode use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, EncoderTuningMode, Frame, InputImage, - PixelFormat, RateControl, VideoContextBuilder, + Codec, EncodeConfig, Encoder, EncoderTuningMode, Frame, InputImage, RateControl, + VideoContextBuilder, YuvLayout, }; use std::fs::File; use std::io::Read; @@ -134,16 +134,9 @@ fn run_encode_test( println!("Config: {:?}, GOP={}", config.rate_control, config.gop_size); - // Create input image for uploading frames. - let mut input_image = InputImage::new( - context.clone(), - codec, - WIDTH, - HEIGHT, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; let mut encoder = Encoder::new(context.clone(), config)?; + // Create input image for uploading frames. + let mut input_image = InputImage::for_encoder(&encoder)?; let mut total_bytes = 0; let mut gpu_durations: Vec = Vec::new(); @@ -169,7 +162,7 @@ fn run_encode_test( let frame = &yuv_data[i * frame_size..(i + 1) * frame_size]; // Upload YUV420 data to the input image - input_image.upload_yuv420(frame)?; + input_image.upload(YuvLayout::I420, frame)?; pending.push_back(encoder.submit(Frame::image(input_image.source()))?); // drain before submitting when at capacity diff --git a/examples/encode.rs b/examples/encode.rs index b61231f8..05336b6a 100644 --- a/examples/encode.rs +++ b/examples/encode.rs @@ -10,8 +10,8 @@ //! The codec defaults to `h264`. use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, - VideoContextBuilder, + Codec, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, + YuvLayout, }; use std::fs::File; use std::io::{Read, Write}; @@ -101,16 +101,9 @@ fn main() -> Result<(), Box> { config.rate_control, config.gop_size ); - // Create input image for uploading frames. - let mut input_image = InputImage::new( - context.clone(), - codec, - WIDTH, - HEIGHT, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; let mut encoder = Encoder::new(context, config)?; + // Create input image for uploading frames. + let mut input_image = InputImage::for_encoder(&encoder)?; let mut output = File::create(output_name)?; let mut total_bytes = 0; @@ -139,7 +132,7 @@ fn main() -> Result<(), Box> { let frame = &yuv_data[i * frame_size..(i + 1) * frame_size]; // Upload YUV420 data to the input image. - input_image.upload_yuv420(frame)?; + input_image.upload(YuvLayout::I420, frame)?; // Submit the frame (async) and keep the pipeline at most ~2 deep. pending.push_back(encoder.submit(Frame::image(input_image.source()))?); diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 9f6cccc6..019e2384 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -146,21 +146,14 @@ impl Encoder { /// an error: submit the RGB frames from a [`SourceImage`] instead. /// /// ```no_run - /// use pixelforge::{Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, VideoContext}; + /// use pixelforge::{EncodeConfig, Encoder, Frame, InputImage, VideoContext, YuvLayout}; /// /// # fn example(context: VideoContext) -> Result<(), Box> { - /// let mut encoder = Encoder::new(context.clone(), EncodeConfig::h264(1920, 1080))?; - /// let mut upload = InputImage::new( - /// context, - /// Codec::H264, - /// 1920, - /// 1080, - /// EncodeBitDepth::Eight, - /// PixelFormat::Yuv420, - /// )?; + /// let mut encoder = Encoder::new(context, EncodeConfig::h264(1920, 1080))?; + /// let mut upload = InputImage::for_encoder(&encoder)?; /// # let yuv_data = vec![0u8; 1920 * 1080 * 3 / 2]; /// let slot = encoder.input_slot()?; - /// upload.upload_yuv420_to(&slot, &yuv_data)?; + /// upload.upload_to(&slot, YuvLayout::I420, &yuv_data)?; /// let packet = pollster::block_on(encoder.submit(Frame::input(slot))?)?; /// # Ok(()) /// # } @@ -371,7 +364,6 @@ impl Encoder { &*self.inner } - #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] pub(crate) fn context(&self) -> &VideoContext { &self.context } diff --git a/src/image.rs b/src/image.rs index 3f7fabd9..88217d85 100644 --- a/src/image.rs +++ b/src/image.rs @@ -3,17 +3,47 @@ //! This module provides [`InputImage`], a helper for uploading YUV data from the CPU //! to the GPU. The image it creates is a transfer-only staging image (not a Vulkan Video //! image) and must be copied into an encoder's input image before encoding. -//! Use [`InputImage::upload_yuv420`] to upload to the internal image, then submit +//! Use [`InputImage::upload`] to upload to the internal image, then submit //! [`InputImage::source`] with [`Frame::image`](crate::Frame::image). -//! Alternatively, use [`InputImage::upload_yuv420_to`] / [`InputImage::upload_yuv444_to`] -//! to upload directly into an encoder's [`InputSlot`] (from +//! Alternatively, use [`InputImage::upload_to`] to upload directly into an +//! encoder's [`InputSlot`] (from //! [`Encoder::input_slot`](crate::encoder::Encoder::input_slot)). -use crate::encoder::{BitDepth, Codec, Handoff, InputSlot, PixelFormat, SourceImage}; +use crate::encoder::{BitDepth, Encoder, Handoff, InputSlot, PixelFormat, Source, SourceImage}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; +/// How the YUV samples handed to [`InputImage::upload`] are laid out in +/// memory. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum YuvLayout { + /// 8-bit 4:2:0 planes: Y, then U, then V (yuv420p). For a 10-bit image, + /// each sample becomes the high 8 bits of the 10. + I420, + /// 8-bit 4:2:0: a Y plane, then an interleaved UV plane. For an 8-bit + /// image only. + Nv12, + /// 10-bit 4:2:0 planes: Y, then U, then V, each sample a little-endian + /// 16-bit word with the value in its low 10 bits (yuv420p10le). For a + /// 10-bit image only. + I420P10, + /// 8-bit 4:4:4 planes: Y, then U, then V (yuv444p). For a 10-bit image, + /// each sample becomes the high 8 bits of the 10. + I444, +} + +impl YuvLayout { + /// The size of one `width` by `height` frame in this layout, in bytes. + pub fn frame_size(&self, width: u32, height: u32) -> usize { + let luma = (width * height) as usize; + match self { + Self::I420 | Self::Nv12 => luma * 3 / 2, + Self::I420P10 | Self::I444 => luma * 3, + } + } +} + /// A GPU image for staging YUV frame data before encoding. /// /// This struct owns a transfer-only Vulkan image (no video profile, no @@ -23,12 +53,11 @@ use ash::vk; /// /// The image is **not** directly usable as a Vulkan Video encode source. To /// encode, either: -/// - Upload to this image and submit [`Self::source`] with +/// - [`upload`](Self::upload) to this image and submit [`Self::source`] with /// [`Frame::image`](crate::Frame::image), which copies it into the encoder's /// input image, or -/// - Use [`upload_yuv420_to`](Self::upload_yuv420_to) / -/// [`upload_yuv444_to`](Self::upload_yuv444_to) to upload directly into the -/// encoder's input image, through an [`InputSlot`]. +/// - [`upload_to`](Self::upload_to) the encoder's input image directly, +/// through an [`InputSlot`]. pub struct InputImage { /// Keeps track of the frames that read `image`. source: SourceImage, @@ -52,6 +81,26 @@ pub struct InputImage { } impl InputImage { + /// A staging image for the frames of `encoder`: its size, pixel format + /// and bit depth. + /// + /// An error for an encoder with an RGB [`Source`]: this uploads YUV. + pub fn for_encoder(encoder: &Encoder) -> Result { + let config = encoder.config(); + if config.source != Source::Yuv { + return Err(PixelForgeError::InvalidInput( + "an InputImage uploads YUV, and the encoder's source is RGB".to_string(), + )); + } + Self::new( + encoder.context().clone(), + config.dimensions.width, + config.dimensions.height, + config.bit_depth, + config.pixel_format, + ) + } + /// Create a new staging image for uploading YUV frame data. /// /// Creates a transfer-only image suitable for staging YUV data before @@ -60,14 +109,12 @@ impl InputImage { /// /// # Arguments /// * `context` - The Vulkan video context - /// * `codec` - Unused. Kept for API compatibility. /// * `width` - Image width in pixels /// * `height` - Image height in pixels /// * `bit_depth` - Bit depth for the image (8-bit or 10-bit) /// * `pixel_format` - Pixel format (YUV420 or YUV444) pub fn new( context: VideoContext, - _codec: Codec, width: u32, height: u32, bit_depth: BitDepth, @@ -250,246 +297,68 @@ impl InputImage { Ok(input) } - /// Upload YUV420 (I420) data to this image. - /// - /// The data is expected in I420 format (Y plane, then U plane, then V plane). - /// This method converts it to NV12 (8-bit) or P010 (10-bit) format and uploads - /// it to the GPU. + /// Upload a frame of `layout` data to this image. /// - /// For 10-bit images, 8-bit input data is expanded to 10-bit by left-shifting - /// each sample by 6 bits (filling in the 6 lower bits with the high bits). - pub fn upload_yuv420(&mut self, yuv_data: &[u8]) -> Result<()> { - let expected_size = (self.width * self.height * 3 / 2) as usize; - if yuv_data.len() < expected_size { - return Err(PixelForgeError::InvalidInput(format!( - "YUV data too small: expected {} bytes, got {}", - expected_size, - yuv_data.len() - ))); - } - - let device = self.context.device(); - let width = self.width as usize; - let height = self.height as usize; - let y_size = width * height; - - // Map staging buffer and convert I420 to NV12/P010. - let data_ptr = unsafe { - device.map_memory( - self.staging_memory, - 0, - self.staging_size as vk::DeviceSize, - vk::MemoryMapFlags::empty(), - ) - } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; - - unsafe { - match self.bit_depth { - BitDepth::Eight => { - // 8-bit NV12 format - let dst = - std::slice::from_raw_parts_mut(data_ptr as *mut u8, self.staging_size); - - // Copy Y plane directly. - dst[..y_size].copy_from_slice(&yuv_data[..y_size]); - - // Interleave U and V planes to create NV12 UV plane. - let u_plane = &yuv_data[y_size..y_size + y_size / 4]; - let v_plane = &yuv_data[y_size + y_size / 4..]; - for i in 0..y_size / 4 { - dst[y_size + i * 2] = u_plane[i]; - dst[y_size + i * 2 + 1] = v_plane[i]; - } - } - BitDepth::Ten => { - // 10-bit P010 format: each sample is stored as 16-bit little-endian - // with the 10-bit value in the high bits (bits 6-15), and bits 0-5 as padding. - // - // To convert 8-bit (0-255) to 10-bit (0-1023): - // - Multiply by ~4 to scale the range: (val * 1023) / 255 ≈ val * 4 - // - This is equivalent to: (val << 2) to get 10-bit value - // - Then shift left by 6 to place in P010 format: (val << 2) << 6 = val << 8 - // - // For better precision, we can also fill in the 2 LSBs of the 10-bit value - // using the 2 MSBs of the original 8-bit value: (val << 8) | (val >> 6 << 6) - // Simplified: val << 8 puts the 8-bit value in bits 8-15, which is correct for P010. - let dst = - std::slice::from_raw_parts_mut(data_ptr as *mut u16, self.staging_size / 2); - - // Convert and copy Y plane (expand 8-bit to 10-bit P010). - for i in 0..y_size { - let val = yuv_data[i] as u16; - // Shift left by 8 to place 8-bit value in bits 8-15 (P010 format). - // This effectively scales 0-255 to 0-65280 with proper bit alignment. - dst[i] = val << 8; - } - - // Interleave U and V planes to create P010 UV plane. - let u_plane = &yuv_data[y_size..y_size + y_size / 4]; - let v_plane = &yuv_data[y_size + y_size / 4..]; - for i in 0..y_size / 4 { - let u_val = u_plane[i] as u16; - let v_val = v_plane[i] as u16; - dst[y_size + i * 2] = u_val << 8; - dst[y_size + i * 2 + 1] = v_val << 8; - } - } - } - - device.unmap_memory(self.staging_memory); - } - - // Record and submit copy commands. - self.copy_staging_to_image()?; - - Ok(()) + /// Submit it afterwards with [`Frame::image`](crate::Frame::image) and + /// [`Self::source`]. This waits until the frames submitted from the image + /// before are done with it. + pub fn upload(&mut self, layout: YuvLayout, data: &[u8]) -> Result<()> { + self.write_staging(layout, data)?; + self.copy_staging_to_image() } - /// Upload NV12 data directly to this image. + /// Upload a frame of `layout` data into an encoder's input image, so the + /// encode reads it with no further copy. /// - /// The data is expected in NV12 format (Y plane, then interleaved UV plane). - pub fn upload_nv12(&mut self, nv12_data: &[u8]) -> Result<()> { - let expected_size = (self.width * self.height * 3 / 2) as usize; - if nv12_data.len() < expected_size { - return Err(PixelForgeError::InvalidInput(format!( - "NV12 data too small: expected {} bytes, got {}", - expected_size, - nv12_data.len() - ))); - } - - let device = self.context.device(); - - // Map staging buffer and copy directly. - let data_ptr = unsafe { - device.map_memory( - self.staging_memory, - 0, - self.staging_size as vk::DeviceSize, - vk::MemoryMapFlags::empty(), - ) - } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; - - unsafe { - let dst = std::slice::from_raw_parts_mut(data_ptr as *mut u8, self.staging_size); - dst.copy_from_slice(&nv12_data[..self.staging_size]); - device.unmap_memory(self.staging_memory); - } - - // Record and submit copy commands. - self.copy_staging_to_image()?; - - Ok(()) + /// Submit the slot with [`Frame::input`](crate::Frame::input) afterwards. + pub fn upload_to(&mut self, slot: &InputSlot, layout: YuvLayout, data: &[u8]) -> Result<()> { + self.write_staging(layout, data)?; + self.copy_staging_to_image_internal( + slot.image(), + vk::ImageLayout::UNDEFINED, + vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, + ) } - /// Upload native 10-bit YUV420 (I420 10-bit) data to this image. - /// - /// The data is expected in yuv420p10le format: - /// - Y plane: width * height * 2 bytes (16-bit little-endian samples) - /// - U plane: (width/2) * (height/2) * 2 bytes (16-bit little-endian samples) - /// - V plane: (width/2) * (height/2) * 2 bytes (16-bit little-endian samples) - /// - /// The 10-bit values are stored in the lower 10 bits of each 16-bit sample. - /// This method converts them to P010 format (10-bit in upper bits) for the encoder. - /// - /// This method is only valid for 10-bit InputImage instances. - pub fn upload_yuv420_10bit(&mut self, yuv_data: &[u8]) -> Result<()> { - if self.bit_depth != BitDepth::Ten { - return Err(PixelForgeError::InvalidInput( - "upload_yuv420_10bit can only be used with 10-bit InputImage".to_string(), - )); - } - - // 10-bit YUV420 size: Y (w*h*2) + U (w*h/4*2) + V (w*h/4*2) = w*h*3 bytes - let expected_size = (self.width * self.height * 3) as usize; - if yuv_data.len() < expected_size { + /// Convert `data` into the staging buffer, in the image's format. + fn write_staging(&mut self, layout: YuvLayout, data: &[u8]) -> Result<()> { + let fits = matches!( + (layout, self.pixel_format, self.bit_depth), + (YuvLayout::I420, PixelFormat::Yuv420, _) + | (YuvLayout::Nv12, PixelFormat::Yuv420, BitDepth::Eight) + | (YuvLayout::I420P10, PixelFormat::Yuv420, BitDepth::Ten) + | (YuvLayout::I444, PixelFormat::Yuv444, _) + ); + if !fits { return Err(PixelForgeError::InvalidInput(format!( - "YUV420 10-bit data too small: expected {} bytes, got {}", - expected_size, - yuv_data.len() + "{layout:?} data cannot be uploaded to a {:?} {:?}-bit image", + self.pixel_format, self.bit_depth ))); } - - let device = self.context.device(); - let width = self.width as usize; - let height = self.height as usize; - let y_size = width * height; // Number of Y samples - - // Map staging buffer. - let data_ptr = unsafe { - device.map_memory( - self.staging_memory, - 0, - self.staging_size as vk::DeviceSize, - vk::MemoryMapFlags::empty(), - ) - } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; - - unsafe { - // Source is yuv420p10le (10-bit in lower bits of 16-bit words) - // Destination is P010 (10-bit in upper bits of 16-bit words) - let src = - std::slice::from_raw_parts(yuv_data.as_ptr() as *const u16, expected_size / 2); - let dst = std::slice::from_raw_parts_mut(data_ptr as *mut u16, self.staging_size / 2); - - // Convert and copy Y plane (shift 10-bit values to upper bits for P010). - for i in 0..y_size { - // yuv420p10le has 10-bit value in bits 0-9, P010 needs it in bits 6-15 - dst[i] = src[i] << 6; - } - - // Convert and interleave U and V planes to create P010 UV plane. - // Source U plane starts at y_size, V plane at y_size + y_size/4 - let u_offset = y_size; - let v_offset = y_size + y_size / 4; - for i in 0..y_size / 4 { - let u_val = src[u_offset + i]; - let v_val = src[v_offset + i]; - dst[y_size + i * 2] = u_val << 6; - dst[y_size + i * 2 + 1] = v_val << 6; - } - - device.unmap_memory(self.staging_memory); - } - - // Record and submit copy commands. - self.copy_staging_to_image()?; - - Ok(()) - } - - /// Upload YUV420 (I420) data to an external image. - /// - /// Same as `upload_yuv420()` but copies the data into an encoder's input - /// image instead of this InputImage's own image, so the encode reads it - /// with no further copy. Submit the slot with - /// [`Frame::input`](crate::Frame::input) afterwards. - pub fn upload_yuv420_to(&mut self, slot: &InputSlot, yuv_data: &[u8]) -> Result<()> { - if self.pixel_format != PixelFormat::Yuv420 { - return Err(PixelForgeError::InvalidInput( - "upload_yuv420_to can only be used with YUV420 InputImage".to_string(), - )); - } - - let expected_size = (self.width * self.height * 3 / 2) as usize; - if yuv_data.len() < expected_size { + let expected_size = layout.frame_size(self.width, self.height); + if data.len() < expected_size { return Err(PixelForgeError::InvalidInput(format!( - "YUV data too small: expected {} bytes, got {}", - expected_size, - yuv_data.len() + "{layout:?} data too small: expected {expected_size} bytes, got {}", + data.len() ))); } - let device = self.context.device(); - let width = self.width as usize; - let height = self.height as usize; - let y_size = width * height; + let luma = (self.width * self.height) as usize; + let chroma = match self.pixel_format { + PixelFormat::Yuv444 => luma, + _ => luma / 4, + }; + let ten_bit = self.bit_depth == BitDepth::Ten; + let luma_bytes = if ten_bit { luma * 2 } else { luma }; + // The UV plane of a 4:4:4 image starts 4-byte aligned, as + // `copy_staging_to_image_internal` copies it. + let uv_offset = match self.pixel_format { + PixelFormat::Yuv444 => crate::align4(luma_bytes), + _ => luma_bytes, + }; - // Map staging buffer and convert I420 to NV12/P010. - let data_ptr = unsafe { + let device = self.context.device(); + let ptr = unsafe { device.map_memory( self.staging_memory, 0, @@ -498,137 +367,54 @@ impl InputImage { ) } .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + // SAFETY: the mapping covers `staging_size` bytes and nothing else + // uses it until it is unmapped below. + let dst = unsafe { std::slice::from_raw_parts_mut(ptr as *mut u8, self.staging_size) }; + let put16 = |dst: &mut [u8], index: usize, value: u16| { + dst[index * 2..index * 2 + 2].copy_from_slice(&value.to_ne_bytes()); + }; - unsafe { - match self.bit_depth { - BitDepth::Eight => { - let dst = - std::slice::from_raw_parts_mut(data_ptr as *mut u8, self.staging_size); - dst[..y_size].copy_from_slice(&yuv_data[..y_size]); - let u_plane = &yuv_data[y_size..y_size + y_size / 4]; - let v_plane = &yuv_data[y_size + y_size / 4..]; - for i in 0..y_size / 4 { - dst[y_size + i * 2] = u_plane[i]; - dst[y_size + i * 2 + 1] = v_plane[i]; + match layout { + YuvLayout::Nv12 => dst[..expected_size].copy_from_slice(&data[..expected_size]), + YuvLayout::I420 | YuvLayout::I444 => { + let (y, rest) = data.split_at(luma); + let (u, v) = rest.split_at(chroma); + if ten_bit { + // An 8-bit sample in the high byte is the same value in + // P010's 10 high bits. + for (i, &s) in y.iter().enumerate() { + put16(dst, i, (s as u16) << 8); } - } - BitDepth::Ten => { - let dst = - std::slice::from_raw_parts_mut(data_ptr as *mut u16, self.staging_size / 2); - for i in 0..y_size { - let val = yuv_data[i] as u16; - dst[i] = val << 8; + let uv = &mut dst[uv_offset..]; + for i in 0..chroma { + put16(uv, i * 2, (u[i] as u16) << 8); + put16(uv, i * 2 + 1, (v[i] as u16) << 8); } - let u_plane = &yuv_data[y_size..y_size + y_size / 4]; - let v_plane = &yuv_data[y_size + y_size / 4..]; - for i in 0..y_size / 4 { - let u_val = u_plane[i] as u16; - let v_val = v_plane[i] as u16; - dst[y_size + i * 2] = u_val << 8; - dst[y_size + i * 2 + 1] = v_val << 8; + } else { + dst[..luma].copy_from_slice(y); + let uv = &mut dst[uv_offset..]; + for i in 0..chroma { + uv[i * 2] = u[i]; + uv[i * 2 + 1] = v[i]; } } } - - device.unmap_memory(self.staging_memory); - } - - // Record and submit copy commands to the target image. - self.copy_staging_to_image_internal( - slot.image(), - vk::ImageLayout::UNDEFINED, - vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, - )?; - - Ok(()) - } - - /// Upload YUV444 (planar) data into an encoder's input image, like - /// [`Self::upload_yuv420_to`]. - pub fn upload_yuv444_to(&mut self, slot: &InputSlot, yuv_data: &[u8]) -> Result<()> { - if self.pixel_format != PixelFormat::Yuv444 { - return Err(PixelForgeError::InvalidInput( - "upload_yuv444_to can only be used with YUV444 InputImage".to_string(), - )); - } - - let plane_size = (self.width * self.height) as usize; - let expected_size = plane_size * 3; // Y + U + V at full resolution - if yuv_data.len() < expected_size { - return Err(PixelForgeError::InvalidInput(format!( - "YUV444 data too small: expected {} bytes, got {}", - expected_size, - yuv_data.len() - ))); - } - - let device = self.context.device(); - - // Map staging buffer. - let data_ptr = unsafe { - device.map_memory( - self.staging_memory, - 0, - self.staging_size as vk::DeviceSize, - vk::MemoryMapFlags::empty(), - ) - } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; - - unsafe { - match self.bit_depth { - BitDepth::Eight => { - // 8-bit semi-planar YUV444: Y plane, then interleaved UV plane. - let dst = - std::slice::from_raw_parts_mut(data_ptr as *mut u8, self.staging_size); - - // Copy Y plane directly. - dst[..plane_size].copy_from_slice(&yuv_data[..plane_size]); - - // Align UV plane offset to 4 bytes for VkBufferImageCopy compliance. - let uv_offset = crate::align4(plane_size); - - // Interleave U and V planes for semi-planar format. - let u_plane = &yuv_data[plane_size..plane_size * 2]; - let v_plane = &yuv_data[plane_size * 2..plane_size * 3]; - for i in 0..plane_size { - dst[uv_offset + i * 2] = u_plane[i]; - dst[uv_offset + i * 2 + 1] = v_plane[i]; - } + YuvLayout::I420P10 => { + // The source has the 10 bits in the low bits of each sample, + // P010 in the high bits. + let sample = |i: usize| u16::from_le_bytes([data[i * 2], data[i * 2 + 1]]) << 6; + for i in 0..luma { + put16(dst, i, sample(i)); } - BitDepth::Ten => { - // 10-bit semi-planar YUV444: Expand 8-bit to 16-bit and interleave UV. - let dst = - std::slice::from_raw_parts_mut(data_ptr as *mut u16, self.staging_size / 2); - - // Convert Y plane. - for i in 0..plane_size { - dst[i] = (yuv_data[i] as u16) << 8; - } - - // Align UV plane offset to 4 bytes for VkBufferImageCopy compliance. - let uv_offset_u16 = crate::align4(plane_size * 2) / 2; - - // Interleave and convert U and V planes. - let u_plane = &yuv_data[plane_size..plane_size * 2]; - let v_plane = &yuv_data[plane_size * 2..plane_size * 3]; - for i in 0..plane_size { - dst[uv_offset_u16 + i * 2] = (u_plane[i] as u16) << 8; - dst[uv_offset_u16 + i * 2 + 1] = (v_plane[i] as u16) << 8; - } + let uv = &mut dst[uv_offset..]; + for i in 0..chroma { + put16(uv, i * 2, sample(luma + i)); + put16(uv, i * 2 + 1, sample(luma + chroma + i)); } } - - device.unmap_memory(self.staging_memory); } - // Record and submit copy commands. - self.copy_staging_to_image_internal( - slot.image(), - vk::ImageLayout::UNDEFINED, - vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, - )?; - + unsafe { device.unmap_memory(self.staging_memory) }; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index b3c62f3a..d3261dcc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,10 +90,7 @@ //! ## Encoding Video //! //! ```rust,no_run -//! use pixelforge::{ -//! Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControl, -//! VideoContextBuilder, -//! }; +//! use pixelforge::{Codec, EncodeConfig, Encoder, InputImage, RateControl, VideoContextBuilder}; //! //! fn main() -> Result<(), Box> { //! let context = VideoContextBuilder::new() @@ -109,20 +106,13 @@ //! .with_frame_rate(30, 1) //! .with_gop_size(60); //! -//! // Create an InputImage helper for uploading YUV data to the GPU. -//! let mut input_image = InputImage::new( -//! context.clone(), -//! Codec::H264, -//! 1920, -//! 1080, -//! EncodeBitDepth::Eight, -//! PixelFormat::Yuv420, -//! )?; //! let mut encoder = Encoder::new(context, config)?; +//! // Create an InputImage helper for uploading YUV data to the GPU. +//! let mut input_image = InputImage::for_encoder(&encoder)?; //! //! // For each frame: upload YUV data and encode. //! // let yuv_data: &[u8] = ...; // YUV420 frame data -//! // input_image.upload_yuv420(yuv_data)?; +//! // input_image.upload(YuvLayout::I420, yuv_data)?; //! // let packet = encoder.submit(Frame::image(input_image.source()))?; //! //! Ok(()) @@ -417,7 +407,7 @@ pub use encoder::{ StreamColor, }; pub use error::PixelForgeError; -pub use image::InputImage; +pub use image::{InputImage, YuvLayout}; pub use sync::TimelinePoint; pub use vulkan::VideoContextBuilder; diff --git a/tests/dmabuf.rs b/tests/dmabuf.rs index ff376950..6590d61a 100644 --- a/tests/dmabuf.rs +++ b/tests/dmabuf.rs @@ -17,9 +17,9 @@ use ash::vk::TaggedStructure; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::encoder::external::drm_format; use pixelforge::{ - Codec, ColorSpec, Conversion, ConversionPreference, DmaBuf, DmaBufPlane, EncodeBitDepth, - EncodeConfig, Encoder, ExternalImage, Frame, Handoff, InputFormat, InputImage, PixelFormat, - RateControl, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, + Codec, ColorSpec, Conversion, ConversionPreference, DmaBuf, DmaBufPlane, EncodeConfig, Encoder, + ExternalImage, Frame, Handoff, InputFormat, InputImage, RateControl, RgbSource, Source, + SourceImage, VideoContext, VideoContextBuilder, YuvLayout, }; use std::os::fd::{FromRawFd, OwnedFd}; @@ -427,19 +427,11 @@ fn encode_reference( config: EncodeConfig, frames: &[Vec], ) -> Result, Box> { - let (w, h) = (config.dimensions.width, config.dimensions.height); let mut encoder = Encoder::new(context.clone(), config.clone())?; - let mut input = InputImage::new( - context.clone(), - config.codec, - w, - h, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; + let mut input = InputImage::for_encoder(&encoder)?; let mut stream = Vec::new(); for frame in frames { - input.upload_nv12(frame)?; + input.upload(YuvLayout::Nv12, frame)?; stream.extend(pollster::block_on(encoder.submit(Frame::image(input.source()))?)?.data); } Ok(stream) diff --git a/tests/rfi.rs b/tests/rfi.rs index 14d5f7fc..c50335b7 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -21,7 +21,7 @@ mod common; use pixelforge::{ Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, - VideoContextBuilder, + VideoContextBuilder, YuvLayout, }; use std::collections::VecDeque; use std::fs::File; @@ -110,14 +110,7 @@ fn run_codec( .with_gop_size(0); let mut encoder = Encoder::new(context.clone(), config)?; - let mut input_image = InputImage::new( - context.clone(), - codec, - WIDTH, - HEIGHT, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; + let mut input_image = InputImage::for_encoder(&encoder)?; let mut yuv_data = Vec::new(); File::open(input_path)?.read_to_end(&mut yuv_data)?; @@ -154,7 +147,7 @@ fn run_codec( } let slot = encoder.input_slot()?; - input_image.upload_yuv420_to(&slot, &yuv_data[start..end])?; + input_image.upload_to(&slot, YuvLayout::I420, &yuv_data[start..end])?; pending.push_back(encoder.submit(Frame::input(slot))?); while pending.len() > 2 { drain_one(&mut pending, &mut output_file, &mut recovery_is_key)?; diff --git a/tests/roundtrip.rs b/tests/roundtrip.rs index b26622be..6bedd536 100644 --- a/tests/roundtrip.rs +++ b/tests/roundtrip.rs @@ -22,8 +22,7 @@ use common::{Readback, decode_stream, write_nv12}; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, - VideoContextBuilder, + Codec, EncodeConfig, Encoder, Frame, InputImage, RateControl, VideoContextBuilder, YuvLayout, }; const WIDTH: u32 = 320; @@ -59,20 +58,13 @@ fn roundtrip_h264() -> Result<(), Box> { .with_frame_rate(30, 1) .with_gop_size(GOP); - let mut input_image = InputImage::new( - context.clone(), - Codec::H264, - WIDTH, - HEIGHT, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; let mut encoder = Encoder::new(context.clone(), config)?; + let mut input_image = InputImage::for_encoder(&encoder)?; let mut bitstream = Vec::new(); let mut pending: VecDeque = VecDeque::new(); for i in 0..frame_count { - input_image.upload_yuv420(&yuv[i * frame_size..(i + 1) * frame_size])?; + input_image.upload(YuvLayout::I420, &yuv[i * frame_size..(i + 1) * frame_size])?; pending.push_back(encoder.submit(Frame::image(input_image.source()))?); while pending.len() > 2 { let packet = pollster::block_on(pending.pop_front().expect("non-empty"))?; diff --git a/tests/submit.rs b/tests/submit.rs index 7375145c..6a85546c 100644 --- a/tests/submit.rs +++ b/tests/submit.rs @@ -10,7 +10,7 @@ use ash::vk; use pixelforge::{ Codec, ColorSpec, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, Handoff, InputFormat, InputImage, PixelFormat, RgbSource, Source, SourceImage, VideoContext, - VideoContextBuilder, + VideoContextBuilder, YuvLayout, }; const WIDTH: u32 = 320; @@ -29,14 +29,7 @@ fn context() -> Result> { fn an_input_slot_is_valid_until_the_next_submission() -> Result<(), Box> { let context = context()?; let mut encoder = Encoder::new(context.clone(), EncodeConfig::h264(WIDTH, HEIGHT))?; - let input = InputImage::new( - context.clone(), - Codec::H264, - WIDTH, - HEIGHT, - EncodeBitDepth::Eight, - PixelFormat::Yuv420, - )?; + let input = InputImage::for_encoder(&encoder)?; let stale = encoder.input_slot()?; let packet = encoder.submit(Frame::image(input.source()))?; @@ -61,7 +54,6 @@ fn a_source_image_needs_pixels() -> Result<(), Box> { let context = context()?; let input = InputImage::new( context.clone(), - Codec::H264, WIDTH, HEIGHT, EncodeBitDepth::Eight, @@ -111,14 +103,7 @@ fn repeat_encodes_the_last_frame_again() -> Result<(), Box Result<(), Box Result<(), Box Result> { + common::init_logging(); + Ok(VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(Codec::H265) + .build()?) +} + +/// A 4:2:0 picture as 8-bit planes: Y, U and V. +fn planes(frame: usize) -> (Vec, Vec, Vec) { + let (w, h) = (WIDTH as usize, HEIGHT as usize); + let y = (0..w * h) + .map(|i| ((i % w) + (i / w) * 3 + frame * 16) as u8) + .collect(); + let u = (0..w * h / 4).map(|i| ((i % (w / 2)) * 2) as u8).collect(); + let v = (0..w * h / 4) + .map(|i| ((i / (w / 2)) * 5 + frame * 8) as u8) + .collect(); + (y, u, v) +} + +/// The picture in `layout`. +fn frame(layout: YuvLayout, index: usize) -> Vec { + let (y, u, v) = planes(index); + match layout { + YuvLayout::I420 => [y, u, v].concat(), + YuvLayout::Nv12 => { + let uv = u.iter().zip(&v).flat_map(|(&u, &v)| [u, v]); + y.into_iter().chain(uv).collect() + } + // The same values, in the high 8 of 10 bits. + YuvLayout::I420P10 => [y, u, v] + .concat() + .into_iter() + .flat_map(|s| ((s as u16) << 2).to_le_bytes()) + .collect(), + YuvLayout::I444 => unreachable!("a 4:2:0 test"), + } +} + +/// Encode the frames in `layout`, half uploaded to the input image's own +/// image and half straight into a slot. +fn encode( + context: &VideoContext, + depth: EncodeBitDepth, + layout: YuvLayout, +) -> Result, Box> { + let config = EncodeConfig::h265(WIDTH, HEIGHT).with_bit_depth(depth); + let mut encoder = Encoder::new(context.clone(), config)?; + let mut input = InputImage::for_encoder(&encoder)?; + let mut stream = Vec::new(); + for i in 0..FRAMES { + let data = frame(layout, i); + let packet = if i % 2 == 0 { + input.upload(layout, &data)?; + encoder.submit(Frame::image(input.source()))? + } else { + let slot = encoder.input_slot()?; + input.upload_to(&slot, layout, &data)?; + encoder.submit(Frame::input(slot))? + }; + stream.extend(pollster::block_on(packet)?.data); + } + Ok(stream) +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn nv12_and_i420_upload_the_same_frame() -> Result<(), Box> { + let context = context()?; + let i420 = encode(&context, EncodeBitDepth::Eight, YuvLayout::I420)?; + let nv12 = encode(&context, EncodeBitDepth::Eight, YuvLayout::Nv12)?; + assert!(!i420.is_empty()); + assert_eq!( + i420, nv12, + "NV12 and I420 of one picture encode differently" + ); + Ok(()) +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn ten_bit_and_eight_bit_upload_the_same_frame() -> Result<(), Box> { + let context = context()?; + let i420 = match encode(&context, EncodeBitDepth::Ten, YuvLayout::I420) { + Ok(stream) => stream, + Err(e) => { + eprintln!("skipping: no 10-bit H.265 encoder ({e})"); + return Ok(()); + } + }; + let i420p10 = encode(&context, EncodeBitDepth::Ten, YuvLayout::I420P10)?; + assert!(!i420.is_empty()); + assert_eq!( + i420, i420p10, + "I420P10 and I420 of one picture encode differently" + ); + Ok(()) +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn a_layout_must_fit_the_image() -> Result<(), Box> { + let context = context()?; + let encoder = Encoder::new(context.clone(), EncodeConfig::h265(WIDTH, HEIGHT))?; + let mut input = InputImage::for_encoder(&encoder)?; + let size = |layout: YuvLayout| layout.frame_size(WIDTH, HEIGHT); + for layout in [YuvLayout::I420P10, YuvLayout::I444] { + assert!( + input.upload(layout, &vec![0; size(layout)]).is_err(), + "{layout:?} was uploaded to an 8-bit 4:2:0 image" + ); + } + assert!( + input + .upload(YuvLayout::I420, &vec![0; size(YuvLayout::I420) - 1]) + .is_err(), + "a short frame was uploaded" + ); + input.upload(YuvLayout::I420, &vec![0; size(YuvLayout::I420)])?; + Ok(()) +} diff --git a/tests/verify_all.rs b/tests/verify_all.rs index 5ed73e96..4c6c927a 100644 --- a/tests/verify_all.rs +++ b/tests/verify_all.rs @@ -12,7 +12,7 @@ mod common; use pixelforge::{ Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, - VideoContextBuilder, + VideoContextBuilder, YuvLayout, }; use std::fs::File; use std::io::{Read, Write}; @@ -138,8 +138,7 @@ fn run_test( Err(e) => return Err(format!("Failed to create encoder: {}", e).into()), }; - let mut input_image = - InputImage::new(context.clone(), codec, WIDTH, HEIGHT, depth, format)?; + let mut input_image = InputImage::for_encoder(&encoder)?; let input_path = match format { PixelFormat::Yuv420 => format!("testdata/test_frames_{}x{}_yuv420p.yuv", WIDTH, HEIGHT), @@ -174,11 +173,12 @@ fn run_test( // copy issues (InputImage uses the transfer queue, encoder uses the // video encode queue which doesn't support transfer ops). let slot = encoder.input_slot()?; - match format { - PixelFormat::Yuv420 => input_image.upload_yuv420_to(&slot, frame)?, - PixelFormat::Yuv444 => input_image.upload_yuv444_to(&slot, frame)?, + let layout = match format { + PixelFormat::Yuv420 => YuvLayout::I420, + PixelFormat::Yuv444 => YuvLayout::I444, _ => return Err("Unsupported format".into()), - } + }; + input_image.upload_to(&slot, layout, frame)?; pending.push_back(encoder.submit(Frame::input(slot))?); while pending.len() > 2 { From abb158a0175f8837f5e4cfabc8cc702e99c3dac9 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 07:57:14 +0100 Subject: [PATCH 3/9] refactor(encoder)!: add EncodeConfig::new and FrameRate, and tidy the config exports --- CHANGELOG.md | 10 +++ README.md | 2 +- src/converter/device_tests.rs | 7 +- src/encoder/av1/record.rs | 4 +- src/encoder/h264/record.rs | 4 +- src/encoder/h265/record.rs | 4 +- src/encoder/mod.rs | 118 +++++++++++++-------------------- src/lib.rs | 9 ++- tests/encode_format_support.rs | 20 ++---- tests/rfi.rs | 4 +- tests/rgb_encode.rs | 6 +- tests/submit.rs | 4 +- tests/upload.rs | 14 ++-- tests/verify_all.rs | 28 ++++---- 14 files changed, 105 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5235bde..bd381176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add `EncodeConfig::new(codec, width, height)`, for a codec chosen at + runtime. - Add RGB input: `EncodeConfig::with_source(Source::Rgb(..))`. The encoder converts RGB frames to the YUV its colour description describes, through `VK_VALVE_video_encode_rgb_conversion` where the device can and the colour @@ -71,6 +73,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 wait until the frames that read the image are done with it. - **Breaking:** `InputImage::new` no longer takes a `Codec`, which it did not use. `InputImage::for_encoder` makes one for an encoder's frames. +- **Breaking:** `EncodeConfig::frame_rate` (a `FrameRate`) replaces + `frame_rate_numerator` and `frame_rate_denominator`. `with_frame_rate` is + unchanged. +- **Breaking:** the encoder's `BitDepth` is exported as `BitDepth`, not + `EncodeBitDepth`. `Dimensions` and `FrameRate` are exported from the crate + root. +- **Breaking:** the `DEFAULT_*` encoder constants are no longer public. + `EncodeConfig::new` and `RateControl::default_for` apply them. - **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the encode API: `EncodeConfig::with_color` replaces `with_color_description`, `Encoder::reconfigure` replaces `set_color_description`, and diff --git a/README.md b/README.md index f1377dd5..2c78a62c 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ let context = VideoContextBuilder::new() // SDR desktop content, encoded as HDR10 for an HDR streaming session. let config = EncodeConfig::h265(1920, 1080) - .with_bit_depth(pixelforge::EncodeBitDepth::Ten) + .with_bit_depth(pixelforge::BitDepth::Ten) .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) .with_color(StreamColor::Hdr10 { range: ColorRange::Full, diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index e98907ac..d97cfd48 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -36,9 +36,8 @@ use super::{ }; use crate::encoder::input::SourceOwnership; use crate::{ - Codec, ColorDescription, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, - Handoff, RateControl, RgbSource, Source, SourceImage, StreamColor, VideoContext, - VideoContextBuilder, + BitDepth, Codec, ColorDescription, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, + RateControl, RgbSource, Source, SourceImage, StreamColor, VideoContext, VideoContextBuilder, }; use ash::vk; @@ -194,7 +193,7 @@ fn encoders( let ten = Encoder::new( context.clone(), EncodeConfig::h265(WIDTH, HEIGHT) - .with_bit_depth(EncodeBitDepth::Ten) + .with_bit_depth(BitDepth::Ten) .with_frame_rate(30, 1), ) .ok(); diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 4b99b4ca..90cb8cd0 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -298,8 +298,8 @@ impl Av1 { let rc_layer_info = vk::VideoEncodeRateControlLayerInfoKHR::default() .average_bitrate(rc.average_bitrate as u64) .max_bitrate(rc.max_bitrate as u64) - .frame_rate_numerator(common.config.frame_rate_numerator) - .frame_rate_denominator(common.config.frame_rate_denominator) + .frame_rate_numerator(common.config.frame_rate.numerator) + .frame_rate_denominator(common.config.frame_rate.denominator) .push(&mut av1_rc_layer_info); let rc_layers = [rc_layer_info]; diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index 8d85252e..8387ab7d 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -381,8 +381,8 @@ impl H264 { let rc_layer_info = vk::VideoEncodeRateControlLayerInfoKHR::default() .average_bitrate(rc.average_bitrate as u64) .max_bitrate(rc.max_bitrate as u64) - .frame_rate_numerator(common.config.frame_rate_numerator) - .frame_rate_denominator(common.config.frame_rate_denominator) + .frame_rate_numerator(common.config.frame_rate.numerator) + .frame_rate_denominator(common.config.frame_rate.denominator) .push(&mut h264_rc_layer_info); let rc_layers = [rc_layer_info]; diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index 47323a57..c418a768 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -375,8 +375,8 @@ impl H265 { let rc_layer_info = vk::VideoEncodeRateControlLayerInfoKHR::default() .average_bitrate(rc.average_bitrate as u64) .max_bitrate(rc.max_bitrate as u64) - .frame_rate_numerator(common.config.frame_rate_numerator) - .frame_rate_denominator(common.config.frame_rate_denominator) + .frame_rate_numerator(common.config.frame_rate.numerator) + .frame_rate_denominator(common.config.frame_rate.denominator) .push(&mut h265_rc_layer_info); let rc_layers = [rc_layer_info]; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index b15757c6..459265bd 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -30,22 +30,22 @@ use crate::converter::ColorRange; // Default encoder configuration constants. /// Default frame rate (frames per second). -pub const DEFAULT_FRAME_RATE: u32 = 30; +pub(crate) const DEFAULT_FRAME_RATE: u32 = 30; /// Default GOP (Group of Pictures) size. -pub const DEFAULT_GOP_SIZE: u32 = 30; +pub(crate) const DEFAULT_GOP_SIZE: u32 = 30; /// Default QP (quantization parameter) for H.264. -pub const DEFAULT_H264_QP: u32 = 26; +pub(crate) const DEFAULT_H264_QP: u32 = 26; /// Default QP (quantization parameter) for H.265. -pub const DEFAULT_H265_QP: u32 = 28; +pub(crate) const DEFAULT_H265_QP: u32 = 28; /// Default quantizer index for AV1. -pub const DEFAULT_AV1_Q_INDEX: u32 = 128; +pub(crate) const DEFAULT_AV1_Q_INDEX: u32 = 128; /// Default maximum number of reference frames. -pub const DEFAULT_MAX_REFERENCE_FRAMES: u32 = 4; +pub(crate) const DEFAULT_MAX_REFERENCE_FRAMES: u32 = 4; /// Video codec types. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -280,12 +280,23 @@ pub enum FrameType { } /// Video dimensions. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Dimensions { + /// Width in pixels. pub width: u32, + /// Height in pixels. pub height: u32, } +/// Frames per second, as a fraction: 30000/1001 for 29.97. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameRate { + /// Frames. + pub numerator: u32, + /// Seconds. + pub denominator: u32, +} + /// What the stream says about its colours: for an RGB source, also what the /// encoder converts to. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -401,10 +412,8 @@ pub struct EncodeConfig { pub bit_depth: BitDepth, /// How the encoder spends bits. Checked by [`Encoder::new`]. pub rate_control: RateControl, - /// Frame rate numerator. - pub frame_rate_numerator: u32, - /// Frame rate denominator. - pub frame_rate_denominator: u32, + /// Frames per second. + pub frame_rate: FrameRate, /// GOP size (distance between IDR frames). pub gop_size: u32, /// Maximum number of reference frames kept in the DPB: the window that @@ -444,19 +453,21 @@ pub struct EncodeConfig { } impl EncodeConfig { - /// Create a new H.264 encode configuration with default settings. - pub fn h264(width: u32, height: u32) -> Self { + /// Create a new `codec` encode configuration with default settings. + pub fn new(codec: Codec, width: u32, height: u32) -> Self { assert!(width > 0, "width must be non-zero"); assert!(height > 0, "height must be non-zero"); Self { - codec: Codec::H264, + codec, dimensions: Dimensions { width, height }, pixel_format: PixelFormat::Yuv420, bit_depth: BitDepth::Eight, - rate_control: RateControl::default_for(Codec::H264), - frame_rate_numerator: DEFAULT_FRAME_RATE, - frame_rate_denominator: 1, + rate_control: RateControl::default_for(codec), + frame_rate: FrameRate { + numerator: DEFAULT_FRAME_RATE, + denominator: 1, + }, gop_size: DEFAULT_GOP_SIZE, max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, @@ -470,56 +481,19 @@ impl EncodeConfig { } } + /// Create a new H.264 encode configuration with default settings. + pub fn h264(width: u32, height: u32) -> Self { + Self::new(Codec::H264, width, height) + } + /// Create a new H.265/HEVC encode configuration with default settings. pub fn h265(width: u32, height: u32) -> Self { - assert!(width > 0, "width must be non-zero"); - assert!(height > 0, "height must be non-zero"); - - Self { - codec: Codec::H265, - dimensions: Dimensions { width, height }, - pixel_format: PixelFormat::Yuv420, - bit_depth: BitDepth::Eight, - rate_control: RateControl::default_for(Codec::H265), - frame_rate_numerator: DEFAULT_FRAME_RATE, - frame_rate_denominator: 1, - gop_size: DEFAULT_GOP_SIZE, - max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, - virtual_buffer_size_ms: 1000, - initial_virtual_buffer_size_ms: 1000, - color: StreamColor::default(), - source: Source::Yuv, - conversion: ConversionPreference::Auto, - encode_usage_hint: EncodeUsageHint::Default, - encode_content_hint: EncodeContentHint::Default, - encoder_tuning_mode: EncoderTuningMode::Default, - } + Self::new(Codec::H265, width, height) } /// Create a new AV1 encode configuration with default settings. pub fn av1(width: u32, height: u32) -> Self { - assert!(width > 0, "width must be non-zero"); - assert!(height > 0, "height must be non-zero"); - - Self { - codec: Codec::AV1, - dimensions: Dimensions { width, height }, - pixel_format: PixelFormat::Yuv420, - bit_depth: BitDepth::Eight, - rate_control: RateControl::default_for(Codec::AV1), - frame_rate_numerator: DEFAULT_FRAME_RATE, - frame_rate_denominator: 1, - gop_size: DEFAULT_GOP_SIZE, - max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, - virtual_buffer_size_ms: 1000, - initial_virtual_buffer_size_ms: 1000, - color: StreamColor::default(), - source: Source::Yuv, - conversion: ConversionPreference::Auto, - encode_usage_hint: EncodeUsageHint::Default, - encode_content_hint: EncodeContentHint::Default, - encoder_tuning_mode: EncoderTuningMode::Default, - } + Self::new(Codec::AV1, width, height) } /// Set how the encoder spends bits. @@ -542,8 +516,10 @@ impl EncodeConfig { /// Set the frame rate. pub fn with_frame_rate(mut self, numerator: u32, denominator: u32) -> Self { - self.frame_rate_numerator = numerator; - self.frame_rate_denominator = denominator; + self.frame_rate = FrameRate { + numerator, + denominator, + }; self } @@ -797,8 +773,8 @@ mod tests { assert_eq!(config.bit_depth, BitDepth::Eight); assert_eq!(config.rate_control, RateControl::Cqp { qp: 26 }); assert_eq!(config.gop_size, 30); - assert_eq!(config.frame_rate_numerator, 30); - assert_eq!(config.frame_rate_denominator, 1); + assert_eq!(config.frame_rate.numerator, 30); + assert_eq!(config.frame_rate.denominator, 1); } #[test] @@ -838,8 +814,8 @@ mod tests { fn test_with_frame_rate() { let config = EncodeConfig::h264(1920, 1080).with_frame_rate(60, 1); - assert_eq!(config.frame_rate_numerator, 60); - assert_eq!(config.frame_rate_denominator, 1); + assert_eq!(config.frame_rate.numerator, 60); + assert_eq!(config.frame_rate.denominator, 1); } #[test] @@ -868,8 +844,8 @@ mod tests { // AV1 uses a 0-255 quantizer index. assert_eq!(config.rate_control, RateControl::Cqp { qp: 128 }); assert_eq!(config.gop_size, 30); - assert_eq!(config.frame_rate_numerator, 30); - assert_eq!(config.frame_rate_denominator, 1); + assert_eq!(config.frame_rate.numerator, 30); + assert_eq!(config.frame_rate.denominator, 1); } #[test] @@ -883,7 +859,7 @@ mod tests { assert_eq!(config.codec, Codec::AV1); assert_eq!(config.rate_control, RateControl::Cqp { qp: 100 }); assert_eq!(config.gop_size, 60); - assert_eq!(config.frame_rate_numerator, 60); + assert_eq!(config.frame_rate.numerator, 60); assert_eq!(config.max_reference_frames, 2); } @@ -907,7 +883,7 @@ mod tests { } ); assert_eq!(config.gop_size, 120); - assert_eq!(config.frame_rate_numerator, 60); + assert_eq!(config.frame_rate.numerator, 60); } } diff --git a/src/lib.rs b/src/lib.rs index d3261dcc..45bf780d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,7 +200,7 @@ //! //! // SDR desktop content, encoded as HDR10 for an HDR streaming session. //! let config = EncodeConfig::h265(1920, 1080) -//! .with_bit_depth(pixelforge::EncodeBitDepth::Ten) +//! .with_bit_depth(pixelforge::BitDepth::Ten) //! .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) //! .with_color(StreamColor::Hdr10 { //! range: ColorRange::Full, @@ -397,10 +397,9 @@ pub use decoder::{ #[cfg(feature = "dmabuf")] pub use encoder::external::{DmaBuf, DmaBufPlane, ExternalImage, InputFormats, InputModifier}; pub use encoder::{ - BitDepth as EncodeBitDepth, Codec, ColorDescription, DEFAULT_AV1_Q_INDEX, DEFAULT_FRAME_RATE, - DEFAULT_GOP_SIZE, DEFAULT_H264_QP, DEFAULT_H265_QP, DEFAULT_MAX_REFERENCE_FRAMES, EncodeConfig, - EncodeContentHint, EncodeFuture, EncodeUsageHint, EncodedPacket, Encoder, EncoderTuningMode, - Frame, FrameType, PixelFormat, RateControl, + BitDepth, Codec, ColorDescription, Dimensions, EncodeConfig, EncodeContentHint, EncodeFuture, + EncodeUsageHint, EncodedPacket, Encoder, EncoderTuningMode, Frame, FrameRate, FrameType, + PixelFormat, RateControl, }; pub use encoder::{ Conversion, ConversionPreference, Handoff, InputSlot, RgbSource, Source, SourceImage, diff --git a/tests/encode_format_support.rs b/tests/encode_format_support.rs index 12b99a9b..85d1aa8c 100644 --- a/tests/encode_format_support.rs +++ b/tests/encode_format_support.rs @@ -4,9 +4,9 @@ #[allow(dead_code)] mod common; -use pixelforge::{Codec, EncodeBitDepth, EncodeConfig, Encoder, PixelFormat, VideoContextBuilder}; +use pixelforge::{BitDepth, Codec, EncodeConfig, Encoder, PixelFormat, VideoContextBuilder}; -fn config(codec: Codec, pixel_format: PixelFormat, bit_depth: EncodeBitDepth) -> EncodeConfig { +fn config(codec: Codec, pixel_format: PixelFormat, bit_depth: BitDepth) -> EncodeConfig { match codec { Codec::H264 => EncodeConfig::h264(1280, 720), Codec::H265 => EncodeConfig::h265(1280, 720), @@ -27,7 +27,7 @@ fn query_matches_encoder_creation() { for codec in [Codec::H264, Codec::H265, Codec::AV1] { for pixel_format in [PixelFormat::Yuv420, PixelFormat::Yuv444] { - for bit_depth in [EncodeBitDepth::Eight, EncodeBitDepth::Ten] { + for bit_depth in [BitDepth::Eight, BitDepth::Ten] { let supported = context.supports_encode_format(codec, pixel_format, bit_depth); let created = Encoder::new(context.clone(), config(codec, pixel_format, bit_depth)).is_ok(); @@ -53,7 +53,7 @@ fn eight_bit_420_matches_supports_encode() { .expect("no Vulkan video device"); for codec in [Codec::H264, Codec::H265, Codec::AV1] { assert_eq!( - context.supports_encode_format(codec, PixelFormat::Yuv420, EncodeBitDepth::Eight), + context.supports_encode_format(codec, PixelFormat::Yuv420, BitDepth::Eight), context.supports_encode(codec), "{codec:?}" ); @@ -69,18 +69,10 @@ fn unsupported_codec_or_format_is_false() { .build() .expect("no Vulkan video device"); // The encoder has no 4:2:2 H.265 profile. - assert!(!context.supports_encode_format( - Codec::H265, - PixelFormat::Yuv422, - EncodeBitDepth::Eight - )); + assert!(!context.supports_encode_format(Codec::H265, PixelFormat::Yuv422, BitDepth::Eight)); for codec in [Codec::H264, Codec::H265, Codec::AV1] { if !context.supports_encode(codec) { - assert!(!context.supports_encode_format( - codec, - PixelFormat::Yuv420, - EncodeBitDepth::Eight - )); + assert!(!context.supports_encode_format(codec, PixelFormat::Yuv420, BitDepth::Eight)); } } } diff --git a/tests/rfi.rs b/tests/rfi.rs index c50335b7..e7bdda67 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -20,7 +20,7 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, + BitDepth, Codec, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, YuvLayout, }; use std::collections::VecDeque; @@ -105,7 +105,7 @@ fn run_codec( } .with_rate_control(RateControl::Cqp { qp: 10 }) .with_pixel_format(PixelFormat::Yuv420) - .with_bit_depth(EncodeBitDepth::Eight) + .with_bit_depth(BitDepth::Eight) // Infinite GOP so the recovery can only come from RFI, never a periodic IDR. .with_gop_size(0); diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index c4b41db0..5ca50107 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -16,7 +16,7 @@ use common::{Readback, decode_stream}; use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ - Codec, ColorRange, ColorSpec, Conversion, ConversionPreference, EncodeBitDepth, EncodeConfig, + BitDepth, Codec, ColorRange, ColorSpec, Conversion, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, InputFormat, RateControl, RgbSource, Source, SourceImage, StreamColor, VideoContext, VideoContextBuilder, }; @@ -273,9 +273,9 @@ fn rgb_query_matches_encoder_creation() -> Result<(), Box InputFormat::RGBA16F, ] { let bit_depth = if format == InputFormat::ABGR2101010 { - EncodeBitDepth::Ten + BitDepth::Ten } else { - EncodeBitDepth::Eight + BitDepth::Eight }; for &(name, stream) in &colors { let color = if matches!(stream, StreamColor::Hdr10 { .. }) { diff --git a/tests/submit.rs b/tests/submit.rs index 6a85546c..10ed16b8 100644 --- a/tests/submit.rs +++ b/tests/submit.rs @@ -8,7 +8,7 @@ mod common; use ash::vk; use pixelforge::{ - Codec, ColorSpec, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, Handoff, + BitDepth, Codec, ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, InputFormat, InputImage, PixelFormat, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, YuvLayout, }; @@ -56,7 +56,7 @@ fn a_source_image_needs_pixels() -> Result<(), Box> { context.clone(), WIDTH, HEIGHT, - EncodeBitDepth::Eight, + BitDepth::Eight, PixelFormat::Yuv420, )?; for layout in [vk::ImageLayout::UNDEFINED, vk::ImageLayout::PREINITIALIZED] { diff --git a/tests/upload.rs b/tests/upload.rs index 6f69a8f2..5c5626b2 100644 --- a/tests/upload.rs +++ b/tests/upload.rs @@ -7,8 +7,8 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, VideoContext, - VideoContextBuilder, YuvLayout, + BitDepth, Codec, EncodeConfig, Encoder, Frame, InputImage, VideoContext, VideoContextBuilder, + YuvLayout, }; const WIDTH: u32 = 320; @@ -59,7 +59,7 @@ fn frame(layout: YuvLayout, index: usize) -> Vec { /// image and half straight into a slot. fn encode( context: &VideoContext, - depth: EncodeBitDepth, + depth: BitDepth, layout: YuvLayout, ) -> Result, Box> { let config = EncodeConfig::h265(WIDTH, HEIGHT).with_bit_depth(depth); @@ -85,8 +85,8 @@ fn encode( #[ignore = "requires a Vulkan Video device"] fn nv12_and_i420_upload_the_same_frame() -> Result<(), Box> { let context = context()?; - let i420 = encode(&context, EncodeBitDepth::Eight, YuvLayout::I420)?; - let nv12 = encode(&context, EncodeBitDepth::Eight, YuvLayout::Nv12)?; + let i420 = encode(&context, BitDepth::Eight, YuvLayout::I420)?; + let nv12 = encode(&context, BitDepth::Eight, YuvLayout::Nv12)?; assert!(!i420.is_empty()); assert_eq!( i420, nv12, @@ -99,14 +99,14 @@ fn nv12_and_i420_upload_the_same_frame() -> Result<(), Box Result<(), Box> { let context = context()?; - let i420 = match encode(&context, EncodeBitDepth::Ten, YuvLayout::I420) { + let i420 = match encode(&context, BitDepth::Ten, YuvLayout::I420) { Ok(stream) => stream, Err(e) => { eprintln!("skipping: no 10-bit H.265 encoder ({e})"); return Ok(()); } }; - let i420p10 = encode(&context, EncodeBitDepth::Ten, YuvLayout::I420P10)?; + let i420p10 = encode(&context, BitDepth::Ten, YuvLayout::I420P10)?; assert!(!i420.is_empty()); assert_eq!( i420, i420p10, diff --git a/tests/verify_all.rs b/tests/verify_all.rs index 4c6c927a..cc48a6e7 100644 --- a/tests/verify_all.rs +++ b/tests/verify_all.rs @@ -11,7 +11,7 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, + BitDepth, Codec, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, YuvLayout, }; use std::fs::File; @@ -49,18 +49,18 @@ fn verify_all() -> Result<(), Box> { let dir = common::scratch_dir("verify_all"); let combinations = [ - (Codec::H264, EncodeBitDepth::Eight, PixelFormat::Yuv420), - (Codec::H264, EncodeBitDepth::Eight, PixelFormat::Yuv444), - (Codec::H264, EncodeBitDepth::Ten, PixelFormat::Yuv420), - (Codec::H264, EncodeBitDepth::Ten, PixelFormat::Yuv444), - (Codec::H265, EncodeBitDepth::Eight, PixelFormat::Yuv420), - (Codec::H265, EncodeBitDepth::Eight, PixelFormat::Yuv444), - (Codec::H265, EncodeBitDepth::Ten, PixelFormat::Yuv420), - (Codec::H265, EncodeBitDepth::Ten, PixelFormat::Yuv444), - (Codec::AV1, EncodeBitDepth::Eight, PixelFormat::Yuv420), - (Codec::AV1, EncodeBitDepth::Eight, PixelFormat::Yuv444), - (Codec::AV1, EncodeBitDepth::Ten, PixelFormat::Yuv420), - (Codec::AV1, EncodeBitDepth::Ten, PixelFormat::Yuv444), + (Codec::H264, BitDepth::Eight, PixelFormat::Yuv420), + (Codec::H264, BitDepth::Eight, PixelFormat::Yuv444), + (Codec::H264, BitDepth::Ten, PixelFormat::Yuv420), + (Codec::H264, BitDepth::Ten, PixelFormat::Yuv444), + (Codec::H265, BitDepth::Eight, PixelFormat::Yuv420), + (Codec::H265, BitDepth::Eight, PixelFormat::Yuv444), + (Codec::H265, BitDepth::Ten, PixelFormat::Yuv420), + (Codec::H265, BitDepth::Ten, PixelFormat::Yuv444), + (Codec::AV1, BitDepth::Eight, PixelFormat::Yuv420), + (Codec::AV1, BitDepth::Eight, PixelFormat::Yuv444), + (Codec::AV1, BitDepth::Ten, PixelFormat::Yuv420), + (Codec::AV1, BitDepth::Ten, PixelFormat::Yuv444), ]; let context = VideoContextBuilder::new() @@ -105,7 +105,7 @@ fn run_test( context: &pixelforge::VideoContext, dir: &std::path::Path, codec: Codec, - depth: EncodeBitDepth, + depth: BitDepth, format: PixelFormat, ) -> Result> { // AV1 uses .obu extension for raw OBU streams (with temporal delimiters). From a53dca0a256d7369e78abb64409404142fa3b5e1 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 08:04:23 +0100 Subject: [PATCH 4/9] refactor(error)!: replace the string error variants with typed ones --- CHANGELOG.md | 12 ++++ src/converter/mod.rs | 14 ++-- src/converter/pipeline.rs | 24 +++---- src/decoder/common.rs | 13 ++-- src/decoder/frames.rs | 8 +-- src/decoder/h264/session.rs | 9 +-- src/decoder/mod.rs | 4 +- src/decoder/pipeline.rs | 22 +++---- src/decoder/transfer.rs | 4 +- src/encoder/av1/record.rs | 2 +- src/encoder/av1/session_params.rs | 7 +- src/encoder/codec.rs | 5 +- src/encoder/external.rs | 39 +++++------ src/encoder/h264/record.rs | 2 +- src/encoder/h264/session_params.rs | 8 +-- src/encoder/h265/record.rs | 2 +- src/encoder/h265/session_params.rs | 8 +-- src/encoder/pipeline.rs | 14 ++-- src/encoder/resources.rs | 77 ++++++++++------------ src/error.rs | 102 ++++++++++++----------------- src/image.rs | 42 ++++++------ src/video.rs | 58 ++++++++-------- src/vulkan.rs | 6 +- src/vulkan/encode_device.rs | 2 +- 24 files changed, 224 insertions(+), 260 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd381176..07268293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 root. - **Breaking:** the `DEFAULT_*` encoder constants are no longer public. `EncodeConfig::new` and `RateControl::default_for` apply them. +- **Breaking:** `PixelForgeError` has fewer, typed variants. A failed + Vulkan call is `Vulkan { call, result }`, with the function's name and its + `vk::Result`, in place of `DeviceCreation`, `VideoSessionCreation`, + `SessionParametersCreation`, `MemoryAllocation`, `ResourceCreation`, + `CommandBuffer`, `Synchronization`, `QueryPool` and `Vulkan(vk::Result)`. + A codec or memory type the device lacks is `NoSuitableDevice`, which + replaces `CodecNotSupported`. New are `Driver`, for a driver that reports + success with no usable result, `Io`, for a failed OS call, and + `Cancelled`, for a frame whose encoder shut down. `From` is + removed. +- Every failed Vulkan call on a lost device is `PixelForgeError::DeviceLost`. + Some were reported as another variant. - **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the encode API: `EncodeConfig::with_color` replaces `with_color_description`, `Encoder::reconfigure` replaces `set_color_description`, and diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 417a56ac..aaac0ec1 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -683,14 +683,14 @@ impl ColorConverter { unsafe { device .reset_command_buffer(self.command_buffer, vk::CommandBufferResetFlags::empty()) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; let begin_info = vk::CommandBufferBeginInfo::default() .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT); device .begin_command_buffer(self.command_buffer, &begin_info) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; // --- Phase 1: Transition source image for shader read --- @@ -899,7 +899,7 @@ impl ColorConverter { device .end_command_buffer(self.command_buffer) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; } let (semaphore, value) = self.timeline.pending_signal(); @@ -920,11 +920,11 @@ impl ColorConverter { unsafe { device .reset_fences(&[self.fence]) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetFences"))?; self.context .sync2() .queue_submit2(self.context.compute_queue(), &[submit_info], self.fence) - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; + .map_err(PixelForgeError::vk("vkQueueSubmit2"))?; } self.timeline.commit(); self.in_flight = true; @@ -942,7 +942,7 @@ impl ColorConverter { .device() .wait_for_fences(&[self.fence], true, u64::MAX) } - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; + .map_err(PixelForgeError::vk("vkWaitForFences"))?; self.in_flight = false; } self.destroy_src_view(); @@ -965,7 +965,7 @@ impl ColorConverter { }); let view = unsafe { self.context.device().create_image_view(&view_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("source image view: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateImageView"))?; self.src_view = Some(view); Ok(view) diff --git a/src/converter/pipeline.rs b/src/converter/pipeline.rs index 3eb332f9..d5fda178 100644 --- a/src/converter/pipeline.rs +++ b/src/converter/pipeline.rs @@ -68,7 +68,7 @@ pub fn create_converter( .bindings(&bindings); let descriptor_set_layout = unsafe { device.create_descriptor_set_layout(&layout_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateDescriptorSetLayout"))?; // Create pipeline layout with push constants. let push_constant_range = vk::PushConstantRange::default() @@ -81,14 +81,14 @@ pub fn create_converter( .push_constant_ranges(std::slice::from_ref(&push_constant_range)); let pipeline_layout = unsafe { device.create_pipeline_layout(&pipeline_layout_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreatePipelineLayout"))?; // Create compute shader module. let shader_code = get_spirv_code()?; let shader_info = vk::ShaderModuleCreateInfo::default().code(&shader_code); let shader_module = unsafe { device.create_shader_module(&shader_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateShaderModule"))?; // Create compute pipeline. let entry_point = std::ffi::CString::new("main").unwrap(); @@ -104,7 +104,7 @@ pub fn create_converter( let pipeline = unsafe { device.create_compute_pipelines(vk::PipelineCache::null(), &[pipeline_info], None) } - .map_err(|(_, e)| PixelForgeError::ResourceCreation(e.to_string()))?[0]; + .map_err(|(_, e)| PixelForgeError::vulkan("vkCreateComputePipelines", e))?[0]; // Destroy shader module (no longer needed after pipeline creation) unsafe { device.destroy_shader_module(shader_module, None) }; @@ -124,7 +124,7 @@ pub fn create_converter( .address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE); let sampler = unsafe { device.create_sampler(&sampler_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("sampler creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateSampler"))?; // Create output buffer (device local for compute shader output, transfer // source for image copy). @@ -148,7 +148,7 @@ pub fn create_converter( .flags(vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER); let command_pool = unsafe { device.create_command_pool(&pool_info, None) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateCommandPool"))?; // Allocate command buffer. let alloc_info = vk::CommandBufferAllocateInfo::default() @@ -157,12 +157,12 @@ pub fn create_converter( .command_buffer_count(1); let command_buffer = unsafe { device.allocate_command_buffers(&alloc_info) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?[0]; + .map_err(PixelForgeError::vk("vkAllocateCommandBuffers"))?[0]; // Create fence for synchronization. let fence_info = vk::FenceCreateInfo::default(); let fence = unsafe { device.create_fence(&fence_info, None) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateFence"))?; Ok(ColorConverter { context, @@ -198,7 +198,7 @@ fn create_buffer( .sharing_mode(vk::SharingMode::EXCLUSIVE); let buffer = unsafe { device.create_buffer(&buffer_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("buffer creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateBuffer"))?; let mem_requirements = unsafe { device.get_buffer_memory_requirements(buffer) }; @@ -208,7 +208,7 @@ fn create_buffer( properties, ) else { unsafe { device.destroy_buffer(buffer, None) }; - return Err(PixelForgeError::MemoryAllocation(format!( + return Err(PixelForgeError::NoSuitableDevice(format!( "No suitable memory type for buffer with properties {:?}", properties ))); @@ -222,7 +222,7 @@ fn create_buffer( Ok(m) => m, Err(e) => { unsafe { device.destroy_buffer(buffer, None) }; - return Err(PixelForgeError::MemoryAllocation(e.to_string())); + return Err(PixelForgeError::vulkan("vkAllocateMemory", e)); } }; @@ -231,7 +231,7 @@ fn create_buffer( device.destroy_buffer(buffer, None); device.free_memory(memory, None); } - return Err(PixelForgeError::MemoryAllocation(e.to_string())); + return Err(PixelForgeError::vulkan("vkBindBufferMemory", e)); } Ok((buffer, memory)) diff --git a/src/decoder/common.rs b/src/decoder/common.rs index e55db5c5..4db02a5e 100644 --- a/src/decoder/common.rs +++ b/src/decoder/common.rs @@ -62,7 +62,7 @@ pub(crate) fn query_decode_caps( ) }; if result != vk::Result::SUCCESS { - return Err(PixelForgeError::CodecNotSupported(format!( + return Err(PixelForgeError::NoSuitableDevice(format!( "decode profile not supported: {:?}", result ))); @@ -301,9 +301,8 @@ impl DecoderCommon { let video_decode_fn = ash::khr::video_decode_queue::Device::load(context.instance(), context.device()); - let command_pool = create_command_pool(&context, decode_queue_family, "decode")?; - let transfer_pool = - create_command_pool(&context, context.transfer_queue_family(), "decode transfer")?; + let command_pool = create_command_pool(&context, decode_queue_family)?; + let transfer_pool = create_command_pool(&context, context.transfer_queue_family())?; let (pipeline, frames_rx) = DecodePipeline::new(&context, command_pool, transfer_pool)?; Ok(Self { @@ -407,7 +406,7 @@ impl DecoderCommon { self.video_queue_fn .create_video_session(&session_create_info, None) } - .map_err(|e| PixelForgeError::VideoSessionCreation(format!("{:?}", e)))?; + .map_err(PixelForgeError::vk("vkCreateVideoSessionKHR"))?; let session_memory = allocate_session_memory(&self.context, session, &self.video_queue_fn)?; @@ -657,12 +656,12 @@ impl DecoderCommon { unsafe { device .reset_command_buffer(command_buffer, vk::CommandBufferResetFlags::empty()) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; let begin_info = vk::CommandBufferBeginInfo::default() .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT); device .begin_command_buffer(command_buffer, &begin_info) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; } Ok(()) } diff --git a/src/decoder/frames.rs b/src/decoder/frames.rs index 9e17866a..ca63ab4e 100644 --- a/src/decoder/frames.rs +++ b/src/decoder/frames.rs @@ -589,7 +589,7 @@ fn create_pool_image( create_info.push(&mut format_list) }; let image = unsafe { context.device().create_image(&create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("reorder pool image: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateImage"))?; let reqs = unsafe { context.device().get_image_memory_requirements(image) }; let memory_type_index = find_memory_type( @@ -598,15 +598,15 @@ fn create_pool_image( vk::MemoryPropertyFlags::DEVICE_LOCAL, ) .ok_or_else(|| { - PixelForgeError::MemoryAllocation("No device-local memory for reorder pool".to_string()) + PixelForgeError::NoSuitableDevice("No device-local memory for reorder pool".to_string()) })?; let alloc_info = vk::MemoryAllocateInfo::default() .allocation_size(reqs.size) .memory_type_index(memory_type_index); let memory = unsafe { context.device().allocate_memory(&alloc_info, None) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateMemory"))?; unsafe { context.device().bind_image_memory(image, memory, 0) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBindImageMemory"))?; Ok((image, memory)) } diff --git a/src/decoder/h264/session.rs b/src/decoder/h264/session.rs index a0cbb728..858490e8 100644 --- a/src/decoder/h264/session.rs +++ b/src/decoder/h264/session.rs @@ -402,7 +402,7 @@ impl H264Decoder { .or_else(|| supported.first()) .copied() .ok_or_else(|| { - PixelForgeError::CodecNotSupported( + PixelForgeError::NoSuitableDevice( "H.264 decode: no supported picture format for this profile".to_string(), ) })?; @@ -598,12 +598,7 @@ impl H264Decoder { .video_queue_fn .create_video_session_parameters(&create_info, None) } - .map_err(|e| { - PixelForgeError::SessionParametersCreation(format!( - "H.264 decode session parameters: {:?}", - e - )) - }) + .map_err(PixelForgeError::vk("vkCreateVideoSessionParametersKHR")) } /// Split `data` into pictures, absorbing any parameter sets encountered. diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index cbf0f803..b6462202 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -393,7 +393,7 @@ impl Decoder { /// and [`VideoContext::supports_decode`]). pub fn new(context: VideoContext, config: DecodeConfig) -> Result { if !context.supports_decode(config.codec) { - return Err(PixelForgeError::CodecNotSupported(format!( + return Err(PixelForgeError::NoSuitableDevice(format!( "{:?} decode is not supported by the selected device", config.codec ))); @@ -401,7 +401,7 @@ impl Decoder { let mut inner: Box = match config.codec { Codec::H264 => Box::new(h264::H264Decoder::create(context, &config)?), other => { - return Err(PixelForgeError::CodecNotSupported(format!( + return Err(PixelForgeError::InvalidInput(format!( "{:?} decoding is not implemented yet", other ))); diff --git a/src/decoder/pipeline.rs b/src/decoder/pipeline.rs index 93c5a24e..fdedec97 100644 --- a/src/decoder/pipeline.rs +++ b/src/decoder/pipeline.rs @@ -160,7 +160,10 @@ impl DecodePipeline { let completion_thread = std::thread::Builder::new() .name("pixelforge-decode-completion".to_string()) .spawn(move || run_completion_thread(thread_device, work_rx, thread_sync, frames_tx)) - .map_err(|e| PixelForgeError::CommandBuffer(format!("spawn completion thread: {e}")))?; + .map_err(|source| PixelForgeError::Io { + what: "spawning the completion thread", + source, + })?; Ok(( Self { @@ -250,10 +253,10 @@ impl DecodePipeline { unsafe { device .end_command_buffer(slot.decode_command_buffer) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; device .reset_fences(&[slot.decode_fence]) - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetFences"))?; } // A timeline wait on value 0 is satisfied immediately, so the first @@ -288,7 +291,7 @@ impl DecodePipeline { unsafe { sync2 .queue_submit2(decode_queue, &[submit], slot.decode_fence) - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization))?; + .map_err(PixelForgeError::vk("vkQueueSubmit2"))?; } self.decodes.commit(); @@ -312,10 +315,10 @@ impl DecodePipeline { unsafe { device .end_command_buffer(slot.transfer_command_buffer) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; device .reset_fences(&[slot.transfer_fence]) - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetFences"))?; } let waits = [ @@ -344,7 +347,7 @@ impl DecodePipeline { unsafe { sync2 .queue_submit2(transfer_queue, &[submit], slot.transfer_fence) - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization))?; + .map_err(PixelForgeError::vk("vkQueueSubmit2"))?; } self.copies.commit(); @@ -451,10 +454,7 @@ fn run_completion_thread( if let Err(e) = wait && let Some(tx) = &frames_tx { - let _ = tx.unbounded_send(Err(PixelForgeError::vk( - e, - PixelForgeError::Synchronization, - ))); + let _ = tx.unbounded_send(Err(PixelForgeError::vulkan("vkWaitForFences", e))); } } diff --git a/src/decoder/transfer.rs b/src/decoder/transfer.rs index 86752bd2..2bb16df9 100644 --- a/src/decoder/transfer.rs +++ b/src/decoder/transfer.rs @@ -55,14 +55,14 @@ impl DecoderCommon { unsafe { device .reset_command_buffer(command_buffer, vk::CommandBufferResetFlags::empty()) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; device .begin_command_buffer( command_buffer, &vk::CommandBufferBeginInfo::default() .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), ) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; } let to_transfer = [ diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 90cb8cd0..80780d19 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -431,7 +431,7 @@ impl Av1 { device .end_command_buffer(command_buffer) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; } common.submit_frame() diff --git a/src/encoder/av1/session_params.rs b/src/encoder/av1/session_params.rs index 5034d68e..682a3299 100644 --- a/src/encoder/av1/session_params.rs +++ b/src/encoder/av1/session_params.rs @@ -167,12 +167,7 @@ impl Av1 { common .video_queue_fn .create_video_session_parameters(&session_params_create_info, None) - .map_err(|e| { - PixelForgeError::SessionParametersCreation(format!( - "Failed to create AV1 session parameters: {:?}", - e - )) - }) + .map_err(PixelForgeError::vk("vkCreateVideoSessionParametersKHR")) } } diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 0e5e7beb..74c37bb1 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -900,10 +900,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result ) }; if result != vk::Result::SUCCESS { - return Err(PixelForgeError::VideoSessionCreation(format!( - "{:?}", - result - ))); + return Err(PixelForgeError::vulkan("vkCreateVideoSessionKHR", result)); } let session_memory = allocate_session_memory(context, session, &video_queue_fn)?; diff --git a/src/encoder/external.rs b/src/encoder/external.rs index 3f93469e..c7fca2cf 100644 --- a/src/encoder/external.rs +++ b/src/encoder/external.rs @@ -243,7 +243,7 @@ impl ExternalImage { .memory(self.memory) .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let fd = unsafe { memory_fd.get_memory_fd(&info) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF export: {e}")))?; + .map_err(PixelForgeError::vk("vkGetMemoryFdKHR"))?; // SAFETY: vkGetMemoryFdKHR hands over a new fd the caller owns. let fd = unsafe { OwnedFd::from_raw_fd(fd) }; let planes = (0..self.plane_count) @@ -737,7 +737,7 @@ impl Encoder { info = info.push(&mut profile_list); } let image = unsafe { device.create_image(&info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF image: {e}")))?; + .map_err(PixelForgeError::vk("vkCreateImage"))?; let memory = match self.bind_external_memory(image, import) { Ok(memory) => memory, @@ -823,14 +823,14 @@ impl Encoder { unsafe { memory_fd.get_memory_fd_properties(handle, buffer.fd.as_raw_fd(), &mut fd_props) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF fd: {e}")))?; + .map_err(PixelForgeError::vk("vkGetMemoryFdPropertiesKHR"))?; let type_index = context .find_memory_type( requirements.memory_type_bits & fd_props.memory_type_bits, vk::MemoryPropertyFlags::empty(), ) .ok_or_else(|| { - PixelForgeError::ResourceCreation( + PixelForgeError::NoSuitableDevice( "DMA-BUF import: no memory type fits".to_string(), ) })?; @@ -842,7 +842,10 @@ impl Encoder { let fd = buffer .fd .try_clone() - .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF fd: {e}")))?; + .map_err(|source| PixelForgeError::Io { + what: "duplicating the DMA-BUF fd", + source, + })?; let raw = fd.into_raw_fd(); let mut import_info = vk::ImportMemoryFdInfoKHR::default() .handle_type(handle) @@ -852,9 +855,8 @@ impl Encoder { .memory_type_index(type_index) .push(&mut import_info) .push(&mut dedicated); - unsafe { device.allocate_memory(&info, None) }.map_err(|e| { - PixelForgeError::ResourceCreation(format!("DMA-BUF import: {e}")) - })? + unsafe { device.allocate_memory(&info, None) } + .map_err(PixelForgeError::vk("vkAllocateMemory"))? } None => { let type_index = context @@ -869,7 +871,7 @@ impl Encoder { ) }) .ok_or_else(|| { - PixelForgeError::ResourceCreation( + PixelForgeError::NoSuitableDevice( "DMA-BUF allocation: no memory type fits".to_string(), ) })?; @@ -879,16 +881,13 @@ impl Encoder { .memory_type_index(type_index) .push(&mut export) .push(&mut dedicated); - unsafe { device.allocate_memory(&info, None) }.map_err(|e| { - PixelForgeError::ResourceCreation(format!("DMA-BUF allocation: {e}")) - })? + unsafe { device.allocate_memory(&info, None) } + .map_err(PixelForgeError::vk("vkAllocateMemory"))? } }; if let Err(e) = unsafe { device.bind_image_memory(image, memory, 0) } { unsafe { device.free_memory(memory, None) }; - return Err(PixelForgeError::ResourceCreation(format!( - "DMA-BUF bind: {e}" - ))); + return Err(PixelForgeError::vulkan("vkBindImageMemory", e)); } Ok(memory) } @@ -964,14 +963,14 @@ impl SyncFiles { for _ in 0..slots { let semaphore = unsafe { device.create_semaphore(&vk::SemaphoreCreateInfo::default(), None) } - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateSemaphore"))?; files.acquire.push(semaphore); let mut export = vk::ExportSemaphoreCreateInfo::default() .handle_types(vk::ExternalSemaphoreHandleTypeFlags::SYNC_FD); let semaphore = unsafe { device.create_semaphore(&vk::SemaphoreCreateInfo::default().push(&mut export), None) } - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateSemaphore"))?; files.release.push(semaphore); } Ok(files) @@ -993,9 +992,7 @@ impl SyncFiles { .fd(raw); // Never closed here on failure, as for memory imports. if let Err(e) = unsafe { self.fd_fn.import_semaphore_fd(&info) } { - return Err(PixelForgeError::Synchronization(format!( - "sync_file import: {e}" - ))); + return Err(PixelForgeError::vulkan("vkImportSemaphoreFdKHR", e)); } Ok(vk::SemaphoreSubmitInfo::default() .semaphore(semaphore) @@ -1017,7 +1014,7 @@ impl SyncFiles { .semaphore(self.release[slot]) .handle_type(vk::ExternalSemaphoreHandleTypeFlags::SYNC_FD); let fd = unsafe { self.fd_fn.get_semaphore_fd(&info) } - .map_err(|e| PixelForgeError::Synchronization(format!("sync_file export: {e}")))?; + .map_err(PixelForgeError::vk("vkGetSemaphoreFdKHR"))?; if fd < 0 { return Ok(None); } diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index 8387ab7d..440e05ef 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -466,7 +466,7 @@ impl H264 { device .end_command_buffer(command_buffer) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; } let future = common.submit_frame()?; diff --git a/src/encoder/h264/session_params.rs b/src/encoder/h264/session_params.rs index 74c0a116..af56ee6f 100644 --- a/src/encoder/h264/session_params.rs +++ b/src/encoder/h264/session_params.rs @@ -199,10 +199,10 @@ impl H264 { ) }; if result != vk::Result::SUCCESS { - return Err(PixelForgeError::SessionParametersCreation(format!( - "{:?}", - result - ))); + return Err(PixelForgeError::vulkan( + "vkCreateVideoSessionParametersKHR", + result, + )); } Ok(session_params) } diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index c418a768..82514949 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -484,7 +484,7 @@ impl H265 { device .end_command_buffer(command_buffer) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; } common.submit_frame() diff --git a/src/encoder/h265/session_params.rs b/src/encoder/h265/session_params.rs index e6f8cabb..9f708a2c 100644 --- a/src/encoder/h265/session_params.rs +++ b/src/encoder/h265/session_params.rs @@ -355,10 +355,10 @@ impl H265 { ) }; if result != vk::Result::SUCCESS { - return Err(PixelForgeError::SessionParametersCreation(format!( - "{:?}", - result - ))); + return Err(PixelForgeError::vulkan( + "vkCreateVideoSessionParametersKHR", + result, + )); } Ok(session_params) } diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 05be333b..c818fe78 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -108,12 +108,7 @@ impl Future for EncodeFuture { })), // The sender was dropped without sending. The completion thread only // drops a sender after sending, so this indicates encoder teardown. - Poll::Ready(Err(oneshot::Canceled)) => { - Poll::Ready(Err(PixelForgeError::CommandBuffer( - "encode cancelled: encoder shut down before the frame was read back" - .to_string(), - ))) - } + Poll::Ready(Err(oneshot::Canceled)) => Poll::Ready(Err(PixelForgeError::Cancelled)), Poll::Pending => Poll::Pending, } } @@ -344,7 +339,10 @@ impl EncodePipeline { .spawn(move || { run_completion_thread(thread_device, work_rx, thread_sync); }) - .map_err(|e| PixelForgeError::CommandBuffer(format!("spawn readback thread: {e}")))?; + .map_err(|source| PixelForgeError::Io { + what: "spawning the readback thread", + source, + })?; Ok(Self { slots, @@ -423,7 +421,7 @@ impl EncodePipeline { ) = { let slot = &mut self.slots[slot_index]; let metadata = slot.pending_metadata.take().ok_or_else(|| { - PixelForgeError::CommandBuffer( + PixelForgeError::InvalidInput( "submit_current called without pending packet metadata".to_string(), ) })?; diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index e1a41beb..ca373ee1 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -109,7 +109,7 @@ pub(crate) fn create_encode_feedback_query_pool( .device() .create_query_pool(&query_pool_create_info, None) } - .map_err(|e| PixelForgeError::QueryPool(e.to_string())) + .map_err(PixelForgeError::vk("vkCreateQueryPool")) } pub(crate) fn create_encode_timestamp_query_pool(context: &VideoContext) -> Result { @@ -122,7 +122,7 @@ pub(crate) fn create_encode_timestamp_query_pool(context: &VideoContext) -> Resu .device() .create_query_pool(×tamp_query_pool_create_info, None) } - .map_err(|e| PixelForgeError::QueryPool(e.to_string())) + .map_err(PixelForgeError::vk("vkCreateQueryPool")) } /// Command resources for encoding operations. @@ -148,13 +148,13 @@ pub(crate) fn create_command_resources( encode_queue_family: u32, upload_queue_family: u32, ) -> Result { - let command_pool = create_command_pool(context, encode_queue_family, "encode")?; + let command_pool = create_command_pool(context, encode_queue_family)?; // The upload pool is the encode pool when one family does both. let upload_command_pool = if upload_queue_family == encode_queue_family { command_pool } else { - create_command_pool(context, upload_queue_family, "encode upload")? + create_command_pool(context, upload_queue_family)? }; let upload_command_buffer = allocate_command_buffers(context, upload_command_pool, 1)?[0]; @@ -222,7 +222,7 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam .sharing_mode(vk::SharingMode::EXCLUSIVE); let staging_buffer = unsafe { device.create_buffer(&buffer_create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("staging buffer: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateBuffer"))?; let mem_requirements = unsafe { device.get_buffer_memory_requirements(staging_buffer) }; let memory_type_index = find_memory_type( @@ -231,7 +231,7 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT, ) .ok_or_else(|| { - PixelForgeError::MemoryAllocation("No suitable memory type for staging buffer".to_string()) + PixelForgeError::NoSuitableDevice("No suitable memory type for staging buffer".to_string()) })?; let alloc_info = vk::MemoryAllocateInfo::default() @@ -239,15 +239,15 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam .memory_type_index(memory_type_index); let staging_memory = unsafe { device.allocate_memory(&alloc_info, None) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateMemory"))?; unsafe { device.bind_buffer_memory(staging_buffer, staging_memory, 0) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBindBufferMemory"))?; // Map and zero-fill. let data_ptr = unsafe { device.map_memory(staging_memory, 0, total_size, vk::MemoryMapFlags::empty()) } - .map_err(|e| PixelForgeError::MemoryAllocation(format!("map staging buffer: {}", e)))?; + .map_err(PixelForgeError::vk("vkMapMemory"))?; unsafe { ptr::write_bytes(data_ptr as *mut u8, 0, total_size as usize) }; unsafe { device.unmap_memory(staging_memory) }; @@ -255,12 +255,12 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam unsafe { device.reset_command_buffer(params.command_buffer, vk::CommandBufferResetFlags::empty()) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; let begin_info = vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT); unsafe { device.begin_command_buffer(params.command_buffer, &begin_info) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; // Transition image from UNDEFINED to TRANSFER_DST. let barrier = vk::ImageMemoryBarrier::default() @@ -387,25 +387,19 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam } unsafe { device.end_command_buffer(params.command_buffer) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; // Submit and wait. let submit_info = vk::SubmitInfo::default().command_buffers(std::slice::from_ref(¶ms.command_buffer)); unsafe { device.reset_fences(&[params.fence]) } - .map_err(|e| PixelForgeError::CommandBuffer(format!("reset fence: {}", e)))?; - unsafe { device.queue_submit(params.queue, &[submit_info], params.fence) }.map_err(|e| { - PixelForgeError::vk(e, |e| { - PixelForgeError::CommandBuffer(format!("submit clear: {e}")) - }) - })?; - unsafe { device.wait_for_fences(&[params.fence], true, u64::MAX) }.map_err(|e| { - PixelForgeError::vk(e, |e| { - PixelForgeError::CommandBuffer(format!("wait clear: {e}")) - }) - })?; + .map_err(PixelForgeError::vk("vkResetFences"))?; + unsafe { device.queue_submit(params.queue, &[submit_info], params.fence) } + .map_err(PixelForgeError::vk("vkQueueSubmit"))?; + unsafe { device.wait_for_fences(&[params.fence], true, u64::MAX) } + .map_err(PixelForgeError::vk("vkWaitForFences"))?; unsafe { device.reset_fences(&[params.fence]) } - .map_err(|e| PixelForgeError::CommandBuffer(format!("reset fence after clear: {}", e)))?; + .map_err(PixelForgeError::vk("vkResetFences"))?; // Clean up staging buffer. unsafe { @@ -460,12 +454,12 @@ pub(crate) fn upload_image_to_input( let command_buffer = params.upload_command_buffer; unsafe { device.reset_command_buffer(command_buffer, vk::CommandBufferResetFlags::empty()) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; let begin_info = vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT); unsafe { device.begin_command_buffer(command_buffer, &begin_info) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; let color = vk::ImageSubresourceRange { aspect_mask: vk::ImageAspectFlags::COLOR, @@ -616,7 +610,7 @@ pub(crate) fn upload_image_to_input( } unsafe { device.end_command_buffer(command_buffer) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; let command_buffers = [vk::CommandBufferSubmitInfo::default().command_buffer(command_buffer)]; let submit_info = vk::SubmitInfo2::default() @@ -629,7 +623,7 @@ pub(crate) fn upload_image_to_input( .sync2() .queue_submit2(params.upload_queue, &[submit_info], vk::Fence::null()) } - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; + .map_err(PixelForgeError::vk("vkQueueSubmit2"))?; Ok(()) } @@ -747,7 +741,7 @@ pub(crate) unsafe fn prepare_encode_command_buffer( unsafe { device .reset_command_buffer(command_buffer, vk::CommandBufferResetFlags::empty()) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; } let begin_info = @@ -755,7 +749,7 @@ pub(crate) unsafe fn prepare_encode_command_buffer( unsafe { device .begin_command_buffer(command_buffer, &begin_info) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; } unsafe { @@ -846,13 +840,13 @@ pub(crate) unsafe fn submit_encode_only( unsafe { device .reset_fences(&[fence]) - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetFences"))?; } unsafe { sync2 .queue_submit2(encode_queue, &[submit_info], fence) - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; + .map_err(PixelForgeError::vk("vkQueueSubmit2"))?; } Ok(()) @@ -882,7 +876,7 @@ pub(crate) unsafe fn wait_and_read_bitstream( unsafe { device .wait_for_fences(&[fence], true, u64::MAX) - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; + .map_err(PixelForgeError::vk("vkWaitForFences"))?; } // Read query results (offset + bytes_written). @@ -905,15 +899,15 @@ pub(crate) unsafe fn wait_and_read_bitstream( &mut query_results, vk::QueryResultFlags::WAIT, ) - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::QueryPool))?; + .map_err(PixelForgeError::vk("vkGetQueryPoolResults"))?; } let offset = query_results[0].offset as usize; let size = query_results[0].bytes_written as usize; if size == 0 { - return Err(PixelForgeError::QueryPool( - "Encoder produced 0 bytes".to_string(), + return Err(PixelForgeError::Driver( + "the encoder produced 0 bytes".to_string(), )); } @@ -1028,8 +1022,8 @@ pub(crate) fn get_encoded_session_params( match result { vk::Result::SUCCESS => { if data_size == 0 { - return Err(PixelForgeError::SessionParametersCreation( - "Encoded session parameters size is 0".to_string(), + return Err(PixelForgeError::Driver( + "encoded session parameters size is 0".to_string(), )); } data.truncate(data_size); @@ -1043,9 +1037,10 @@ pub(crate) fn get_encoded_session_params( data_size = data.len(); } err => { - return Err(PixelForgeError::SessionParametersCreation(format!( - "Failed to get encoded session parameters: {err:?}" - ))); + return Err(PixelForgeError::vulkan( + "vkGetEncodedVideoSessionParametersKHR", + err, + )); } } } diff --git a/src/error.rs b/src/error.rs index b928f169..f73b59dc 100644 --- a/src/error.rs +++ b/src/error.rs @@ -22,46 +22,10 @@ pub enum PixelForgeError { #[error("The Vulkan device was lost")] DeviceLost, - /// Vulkan logical device creation failed. - #[error("Failed to create Vulkan device: {0}")] - DeviceCreation(String), - - /// Video session creation failed. - #[error("Failed to create video session: {0}")] - VideoSessionCreation(String), - - /// Video session parameters creation failed. - #[error("Failed to create video session parameters: {0}")] - SessionParametersCreation(String), - - /// Memory allocation failed. - #[error("Memory allocation failed: {0}")] - MemoryAllocation(String), - - /// Resource creation failed (images, buffers, fences, command pools, etc.). - #[error("Failed to create resource: {0}")] - ResourceCreation(String), - - /// Command buffer operation failed. - #[error("Command buffer error: {0}")] - CommandBuffer(String), - /// Invalid input (dimensions, data size, format, etc.). #[error("Invalid input: {0}")] InvalidInput(String), - /// Codec not supported. - #[error("Codec not supported: {0}")] - CodecNotSupported(String), - - /// Synchronization primitive error. - #[error("Synchronization error: {0}")] - Synchronization(String), - - /// Query pool error. - #[error("Query pool error: {0}")] - QueryPool(String), - /// The encoded bitstream overflowed the destination buffer. #[error("Encoder bitstream buffer overflow (wrote {written} bytes into {capacity} bytes)")] BufferOverflow { @@ -71,27 +35,50 @@ pub enum PixelForgeError { capacity: usize, }, - /// Generic Vulkan error. - #[error("Vulkan error: {0}")] - Vulkan(ash::vk::Result), -} + /// A Vulkan call failed. A lost device is [`Self::DeviceLost`] instead. + #[error("{call} failed: {result}")] + Vulkan { + /// The Vulkan function that failed. + call: &'static str, + /// What it returned. + result: ash::vk::Result, + }, -impl From for PixelForgeError { - fn from(result: ash::vk::Result) -> Self { - Self::vk(result, |_| PixelForgeError::Vulkan(result)) - } + /// The driver reported success, but what it returned cannot be used, + /// such as an encode that wrote no bytes. + #[error("Driver error: {0}")] + Driver(String), + + /// An operating system call failed. + #[error("{what} failed: {source}")] + Io { + /// What failed. + what: &'static str, + /// The error. + #[source] + source: std::io::Error, + }, + + /// The encoder was dropped before the frame was read back. + #[error("Encode cancelled: the encoder shut down before the frame was read back")] + Cancelled, } impl PixelForgeError { - /// `wrap` the description of a failed Vulkan call, except a lost device, - /// which is always [`Self::DeviceLost`]. - pub(crate) fn vk(result: ash::vk::Result, wrap: impl FnOnce(String) -> Self) -> Self { + /// The error for `result` from the Vulkan function `call`: a lost device + /// is always [`Self::DeviceLost`]. + pub(crate) fn vulkan(call: &'static str, result: ash::vk::Result) -> Self { if result == ash::vk::Result::ERROR_DEVICE_LOST { Self::DeviceLost } else { - wrap(result.to_string()) + Self::Vulkan { call, result } } } + + /// [`Self::vulkan`] for `map_err`. + pub(crate) fn vk(call: &'static str) -> impl FnOnce(ash::vk::Result) -> Self { + move |result| Self::vulkan(call, result) + } } /// Result type for PixelForge operations. @@ -105,22 +92,15 @@ mod tests { #[test] fn a_lost_device_is_always_device_lost() { assert!(matches!( - PixelForgeError::vk( - vk::Result::ERROR_DEVICE_LOST, - PixelForgeError::CommandBuffer - ), - PixelForgeError::DeviceLost - )); - assert!(matches!( - PixelForgeError::from(vk::Result::ERROR_DEVICE_LOST), + PixelForgeError::vulkan("vkQueueSubmit", vk::Result::ERROR_DEVICE_LOST), PixelForgeError::DeviceLost )); assert!(matches!( - PixelForgeError::vk( - vk::Result::ERROR_OUT_OF_DEVICE_MEMORY, - PixelForgeError::CommandBuffer - ), - PixelForgeError::CommandBuffer(_) + PixelForgeError::vk("vkQueueSubmit")(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY), + PixelForgeError::Vulkan { + call: "vkQueueSubmit", + result: vk::Result::ERROR_OUT_OF_DEVICE_MEMORY + } )); } } diff --git a/src/image.rs b/src/image.rs index 88217d85..79b1423c 100644 --- a/src/image.rs +++ b/src/image.rs @@ -161,7 +161,7 @@ impl InputImage { .initial_layout(vk::ImageLayout::UNDEFINED); let image = unsafe { device.create_image(&image_create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("image creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateImage"))?; // Get memory requirements and allocate. let mem_requirements = unsafe { device.get_image_memory_requirements(image) }; @@ -172,7 +172,7 @@ impl InputImage { vk::MemoryPropertyFlags::DEVICE_LOCAL, ) .ok_or_else(|| { - PixelForgeError::MemoryAllocation("No suitable memory type for image".to_string()) + PixelForgeError::NoSuitableDevice("No suitable memory type for image".to_string()) })?; let alloc_info = vk::MemoryAllocateInfo::default() @@ -180,10 +180,10 @@ impl InputImage { .memory_type_index(memory_type_index); let memory = unsafe { device.allocate_memory(&alloc_info, None) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateMemory"))?; unsafe { device.bind_image_memory(image, memory, 0) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBindImageMemory"))?; // Create staging buffer for uploads. // Size depends on pixel format, bit depth, and alignment padding between planes. @@ -206,7 +206,7 @@ impl InputImage { .sharing_mode(vk::SharingMode::EXCLUSIVE); let staging_buffer = unsafe { device.create_buffer(&buffer_create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("buffer creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateBuffer"))?; let staging_mem_requirements = unsafe { device.get_buffer_memory_requirements(staging_buffer) }; @@ -217,7 +217,7 @@ impl InputImage { vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT, ) .ok_or_else(|| { - PixelForgeError::MemoryAllocation( + PixelForgeError::NoSuitableDevice( "No suitable memory type for staging buffer".to_string(), ) })?; @@ -227,10 +227,10 @@ impl InputImage { .memory_type_index(staging_memory_type); let staging_memory = unsafe { device.allocate_memory(&staging_alloc_info, None) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateMemory"))?; unsafe { device.bind_buffer_memory(staging_buffer, staging_memory, 0) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBindBufferMemory"))?; // Create command pool and buffer for transfers. let pool_info = vk::CommandPoolCreateInfo::default() @@ -238,7 +238,7 @@ impl InputImage { .flags(vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER); let command_pool = unsafe { device.create_command_pool(&pool_info, None) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkCreateCommandPool"))?; let alloc_info = vk::CommandBufferAllocateInfo::default() .command_pool(command_pool) @@ -246,13 +246,13 @@ impl InputImage { .command_buffer_count(1); let command_buffers = unsafe { device.allocate_command_buffers(&alloc_info) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateCommandBuffers"))?; let command_buffer = command_buffers[0]; // Create fence for synchronization. let fence_info = vk::FenceCreateInfo::default(); let fence = unsafe { device.create_fence(&fence_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("fence creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateFence"))?; let source = SourceImage::new( &context, @@ -289,7 +289,7 @@ impl InputImage { staging_size as vk::DeviceSize, vk::MemoryMapFlags::empty(), ) - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkMapMemory"))?; std::ptr::write_bytes(ptr as *mut u8, 0, staging_size); device.unmap_memory(staging_memory); } @@ -366,7 +366,7 @@ impl InputImage { vk::MemoryMapFlags::empty(), ) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkMapMemory"))?; // SAFETY: the mapping covers `staging_size` bytes and nothing else // uses it until it is unmapped below. let dst = unsafe { std::slice::from_raw_parts_mut(ptr as *mut u8, self.staging_size) }; @@ -457,12 +457,12 @@ impl InputImage { unsafe { device.reset_command_buffer(self.command_buffer, vk::CommandBufferResetFlags::empty()) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetCommandBuffer"))?; let begin_info = vk::CommandBufferBeginInfo::default() .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT); unsafe { device.begin_command_buffer(self.command_buffer, &begin_info) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBeginCommandBuffer"))?; // Transition image to transfer destination. let barrier = vk::ImageMemoryBarrier::default() @@ -621,24 +621,20 @@ impl InputImage { } unsafe { device.end_command_buffer(self.command_buffer) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; // Submit and wait. let submit_info = vk::SubmitInfo::default().command_buffers(std::slice::from_ref(&self.command_buffer)); unsafe { device.queue_submit(self.context.transfer_queue(), &[submit_info], self.fence) } - .map_err(|e| { - PixelForgeError::vk(e, |e| { - PixelForgeError::CommandBuffer(format!("Queue submit failed: {e}")) - }) - })?; + .map_err(PixelForgeError::vk("vkQueueSubmit"))?; unsafe { device.wait_for_fences(&[self.fence], true, u64::MAX) } - .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization))?; + .map_err(PixelForgeError::vk("vkWaitForFences"))?; unsafe { device.reset_fences(&[self.fence]) } - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(PixelForgeError::vk("vkResetFences"))?; Ok(()) } diff --git a/src/video.rs b/src/video.rs index 5cc6c52e..ad46ce92 100644 --- a/src/video.rs +++ b/src/video.rs @@ -86,7 +86,7 @@ fn allocate_and_bind_image_memory( vk::MemoryPropertyFlags::DEVICE_LOCAL, ) .ok_or_else(|| { - PixelForgeError::MemoryAllocation("No suitable memory type for image".to_string()) + PixelForgeError::NoSuitableDevice("No suitable memory type for image".to_string()) })?; let alloc_info = vk::MemoryAllocateInfo::default() @@ -94,13 +94,13 @@ fn allocate_and_bind_image_memory( .memory_type_index(memory_type_index); let memory = unsafe { context.device().allocate_memory(&alloc_info, None) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateMemory"))?; match unsafe { context.device().bind_image_memory(image, memory, 0) } { Ok(()) => Ok(memory), Err(e) => { unsafe { context.device().free_memory(memory, None) }; - Err(PixelForgeError::MemoryAllocation(e.to_string())) + Err(PixelForgeError::vulkan("vkBindImageMemory", e)) } } } @@ -142,7 +142,7 @@ fn create_video_image_raw( } let image = unsafe { context.device().create_image(&create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("image creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateImage"))?; match allocate_and_bind_image_memory(context, image) { Ok(memory) => Ok((image, memory)), @@ -186,7 +186,7 @@ pub(crate) fn create_video_image_view( }); unsafe { context.device().create_image_view(&view_create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("image view creation: {}", e))) + .map_err(PixelForgeError::vk("vkCreateImageView")) } /// The single-plane formats a multi-planar picture format's planes can be @@ -343,7 +343,7 @@ pub(crate) fn create_bitstream_buffer( .push(&mut profile_list); let buffer = unsafe { context.device().create_buffer(&create_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("buffer creation: {}", e)))?; + .map_err(PixelForgeError::vk("vkCreateBuffer"))?; let mem_requirements = unsafe { context.device().get_buffer_memory_requirements(buffer) }; @@ -353,7 +353,7 @@ pub(crate) fn create_bitstream_buffer( vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT, ) .ok_or_else(|| { - PixelForgeError::MemoryAllocation("No suitable memory type for buffer".to_string()) + PixelForgeError::NoSuitableDevice("No suitable memory type for buffer".to_string()) })?; let alloc_info = vk::MemoryAllocateInfo::default() @@ -361,10 +361,10 @@ pub(crate) fn create_bitstream_buffer( .memory_type_index(memory_type_index); let memory = unsafe { context.device().allocate_memory(&alloc_info, None) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkAllocateMemory"))?; unsafe { context.device().bind_buffer_memory(buffer, memory, 0) } - .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + .map_err(PixelForgeError::vk("vkBindBufferMemory"))?; Ok((buffer, memory)) } @@ -376,7 +376,7 @@ pub(crate) fn create_timeline_semaphore(context: &VideoContext) -> Result Result { +pub(crate) fn create_command_pool(context: &VideoContext, family: u32) -> Result { let pool_info = vk::CommandPoolCreateInfo::default() .queue_family_index(family) .flags(vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER); unsafe { context.device().create_command_pool(&pool_info, None) } - .map_err(|e| PixelForgeError::ResourceCreation(format!("{} command pool: {}", label, e))) + .map_err(PixelForgeError::vk("vkCreateCommandPool")) } /// Allocate `count` primary command buffers from `pool`. @@ -624,7 +624,7 @@ pub(crate) fn allocate_command_buffers( .level(vk::CommandBufferLevel::PRIMARY) .command_buffer_count(count); unsafe { context.device().allocate_command_buffers(&info) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string())) + .map_err(PixelForgeError::vk("vkAllocateCommandBuffers")) } /// A fence, created signaled when it will be waited on before its first submit. @@ -634,7 +634,7 @@ pub(crate) fn create_fence(context: &VideoContext, signaled: bool) -> Result Date: Thu, 5 Nov 2026 08:08:00 +0100 Subject: [PATCH 5/9] refactor(encoder)!: report IDR frames as FrameType::Idr and derive is_key_frame --- CHANGELOG.md | 3 +++ examples/encode.rs | 2 +- src/encoder/av1/mod.rs | 1 - src/encoder/codec.rs | 9 +++------ src/encoder/h264/mod.rs | 5 +---- src/encoder/h265/mod.rs | 5 +---- src/encoder/mod.rs | 22 ++++++++++++++-------- src/encoder/pipeline.rs | 2 -- tests/rfi.rs | 2 +- tests/rgb_encode.rs | 4 ++-- tests/submit.rs | 4 ++-- 11 files changed, 28 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07268293..b43fbd49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 removed. - Every failed Vulkan call on a lost device is `PixelForgeError::DeviceLost`. Some were reported as another variant. +- **Breaking:** `EncodedPacket::is_key_frame` is a method, not a field. An + IDR's `frame_type` is `FrameType::Idr`; it was `FrameType::I`. + `FrameType::Unknown`, which no packet had, is removed. - **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the encode API: `EncodeConfig::with_color` replaces `with_color_description`, `Encoder::reconfigure` replaces `set_color_description`, and diff --git a/examples/encode.rs b/examples/encode.rs index 05336b6a..3212bd7b 100644 --- a/examples/encode.rs +++ b/examples/encode.rs @@ -122,7 +122,7 @@ fn main() -> Result<(), Box> { packet.dts, packet.data.len(), packet.frame_type, - if packet.is_key_frame { " [KEY]" } else { "" } + if packet.is_key_frame() { " [KEY]" } else { "" } ); Ok::<(), Box>(()) }; diff --git a/src/encoder/av1/mod.rs b/src/encoder/av1/mod.rs index bb9843ac..2e2d3147 100644 --- a/src/encoder/av1/mod.rs +++ b/src/encoder/av1/mod.rs @@ -105,7 +105,6 @@ impl VideoCodec for Av1 { } Ok(PictureSetup { - frame_type: plan.frame_type(), header: Some(header), }) } diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 74c37bb1..717704b8 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -362,16 +362,14 @@ impl FramePlan { /// The stream-level frame type for packet metadata. pub fn frame_type(&self) -> FrameType { match self.gop.frame_type { - GopFrameType::Idr => FrameType::I, + GopFrameType::Idr => FrameType::Idr, GopFrameType::P => FrameType::P, } } } -/// The codec header (if any) and stream frame type produced by -/// [`VideoCodec::begin_picture`]. +/// The codec header (if any) produced by [`VideoCodec::begin_picture`]. pub(crate) struct PictureSetup { - pub frame_type: FrameType, /// Codec header to prepend (SPS/PPS, VPS/SPS/PPS, AV1 sequence header). /// `Some` only for frames that carry one (typically the IDR/key frame). pub header: Option>, @@ -506,8 +504,7 @@ impl CodecEncoder { let setup = self.codec.begin_picture(&mut self.common, &plan)?; self.common.set_pending_metadata(SlotPacketMetadata { - frame_type: setup.frame_type, - is_key_frame: plan.is_idr(), + frame_type: plan.frame_type(), pts: plan.display_order, dts: plan.encode_index, header: setup.header, diff --git a/src/encoder/h264/mod.rs b/src/encoder/h264/mod.rs index 9edaab2a..078a7cfa 100644 --- a/src/encoder/h264/mod.rs +++ b/src/encoder/h264/mod.rs @@ -85,10 +85,7 @@ impl VideoCodec for H264 { None }; - Ok(PictureSetup { - frame_type: plan.frame_type(), - header, - }) + Ok(PictureSetup { header }) } fn record_picture( diff --git a/src/encoder/h265/mod.rs b/src/encoder/h265/mod.rs index aca6a2d0..21d5f4f3 100644 --- a/src/encoder/h265/mod.rs +++ b/src/encoder/h265/mod.rs @@ -84,10 +84,7 @@ impl VideoCodec for H265 { None }; - Ok(PictureSetup { - frame_type: plan.frame_type(), - header, - }) + Ok(PictureSetup { header }) } fn record_picture( diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 459265bd..d1ed2b7c 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -275,8 +275,6 @@ pub enum FrameType { P, /// Bi-predicted frame. B, - /// Unknown frame type. - Unknown, } /// Video dimensions. @@ -622,8 +620,6 @@ pub struct EncodedPacket { pub data: Vec, /// Frame type. pub frame_type: FrameType, - /// Whether this is a keyframe. - pub is_key_frame: bool, /// Presentation timestamp. pub pts: u64, /// Decode timestamp. @@ -632,6 +628,13 @@ pub struct EncodedPacket { pub stats: EncodedPacketStats, } +impl EncodedPacket { + /// Whether a decoder can start at this packet: an IDR frame. + pub fn is_key_frame(&self) -> bool { + self.frame_type == FrameType::Idr + } +} + #[cfg(test)] mod tests { use super::*; @@ -897,7 +900,6 @@ mod tests { assert_ne!(FrameType::Idr, FrameType::I); assert_ne!(FrameType::I, FrameType::P); assert_ne!(FrameType::P, FrameType::B); - assert_ne!(FrameType::B, FrameType::Unknown); } } @@ -910,15 +912,19 @@ mod tests { let packet = EncodedPacket { data: vec![0x00, 0x00, 0x00, 0x01, 0x67], frame_type: FrameType::Idr, - is_key_frame: true, pts: 0, dts: 0, stats: EncodedPacketStats::default(), }; - assert!(packet.is_key_frame); - assert_eq!(packet.frame_type, FrameType::Idr); + assert!(packet.is_key_frame()); assert_eq!(packet.data.len(), 5); + + let packet = EncodedPacket { + frame_type: FrameType::P, + ..packet + }; + assert!(!packet.is_key_frame()); } } diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index c818fe78..126e7d95 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -125,7 +125,6 @@ pub(crate) const ENCODE_PIPELINE_DEPTH: usize = 2; /// when the completion thread reads the slot back. pub(crate) struct SlotPacketMetadata { pub frame_type: FrameType, - pub is_key_frame: bool, pub pts: u64, pub dts: u64, /// Codec header (SPS/PPS, VPS/SPS/PPS, or AV1 sequence header) to prepend. @@ -630,7 +629,6 @@ fn run_completion_thread( let packet = result.map(|()| EncodedPacket { data, frame_type: work.metadata.frame_type, - is_key_frame: work.metadata.is_key_frame, pts: work.metadata.pts, dts: work.metadata.dts, stats, diff --git a/tests/rfi.rs b/tests/rfi.rs index e7bdda67..d9f59690 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -126,7 +126,7 @@ fn run_codec( -> Result<(), Box> { let packet = pollster::block_on(pending.pop_front().unwrap())?; if packet.pts == INVALIDATE_AT { - *recovery_is_key = Some(packet.is_key_frame); + *recovery_is_key = Some(packet.is_key_frame()); } output_file.write_all(&packet.data)?; Ok(()) diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index 5ca50107..740ccc92 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -330,7 +330,7 @@ fn color_change_follows_into_the_conversion() -> Result<(), Box>()); assert!(keys[0] && keys[FRAMES as usize / 2], "key frames: {keys:?}"); @@ -381,7 +381,7 @@ fn a_format_change_switches_the_shader_in_place() -> Result<(), Box Result<(), Box Result<(), Box Date: Thu, 5 Nov 2026 08:14:45 +0100 Subject: [PATCH 6/9] refactor(encoder): share the encode command recording between codecs --- src/encoder/av1/record.rs | 125 +++++---------------------------- src/encoder/codec.rs | 139 ++++++++++++++++++++++++++++++++++++- src/encoder/h264/record.rs | 113 +++++------------------------- src/encoder/h265/record.rs | 119 +++++-------------------------- 4 files changed, 185 insertions(+), 311 deletions(-) diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 80780d19..432714aa 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -3,13 +3,10 @@ use super::Av1; -use crate::encoder::codec::{EncoderCommon, FramePlan, RateControlPlan}; +use crate::encoder::codec::{EncodeCommands, EncoderCommon, FramePlan, RateControlPlan}; use crate::encoder::pipeline::EncodeFuture; -use crate::encoder::resources::{ - end_timestamp, prepare_encode_command_buffer, record_dpb_barriers, - record_post_encode_dpb_barrier, reset_start_timestamp, -}; -use crate::error::{PixelForgeError, Result}; +use crate::encoder::resources::{prepare_encode_command_buffer, record_dpb_barriers}; +use crate::error::Result; use ash::vk; use ash::vk::TaggedStructure; use tracing::debug; @@ -29,7 +26,6 @@ impl Av1 { let slot = common.pipeline.current(); let command_buffer = slot.encode_command_buffer; let query_pool = slot.query_pool; - let timestamp_query_pool = slot.timestamp_query_pool; let bitstream_buffer = slot.bitstream_buffer; let bitstream_buffer_size = slot.bitstream_buffer_size; let input_image_view = common.source_view(); @@ -295,22 +291,6 @@ impl Av1 { .use_max_q_index(false); } - let rc_layer_info = vk::VideoEncodeRateControlLayerInfoKHR::default() - .average_bitrate(rc.average_bitrate as u64) - .max_bitrate(rc.max_bitrate as u64) - .frame_rate_numerator(common.config.frame_rate.numerator) - .frame_rate_denominator(common.config.frame_rate.denominator) - .push(&mut av1_rc_layer_info); - let rc_layers = [rc_layer_info]; - - let mut rc_info = vk::VideoEncodeRateControlInfoKHR::default().rate_control_mode(rc.mode); - if !rc.is_disabled() { - rc_info = rc_info - .layers(&rc_layers) - .virtual_buffer_size_in_ms(common.config.virtual_buffer_size_ms) - .initial_virtual_buffer_size_in_ms(common.config.initial_virtual_buffer_size_ms); - } - // Begin coding: setup slot (slot_index -1) plus any active reference slots. let mut all_reference_slots = Vec::new(); if is_reference { @@ -322,63 +302,15 @@ impl Av1 { } all_reference_slots.extend_from_slice(&reference_slots); - let is_first_frame = plan.is_first_frame(); - let should_reset_coding_state = is_first_frame || (is_key_frame && !rc.is_disabled()); // Clamp GOP values to at least 1; a value of 0 is undefined in // Vulkan and causes undefined behavior on some drivers (RADV). let gop_frames = common.config.gop_size.max(1); - let mut av1_rc_info = vk::VideoEncodeAV1RateControlInfoKHR::default() + let codec_rate_control = vk::VideoEncodeAV1RateControlInfoKHR::default() .gop_frame_count(gop_frames) .key_frame_period(gop_frames) .consecutive_bipredictive_frame_count(0) .temporal_layer_count(1); - // Reset and write start timestamp - reset_start_timestamp(common.device(), command_buffer, timestamp_query_pool); - - // Ash links pNext chains in place, so each command needs its own extension structs. - let mut begin_rc_info = rc_info; - let mut begin_av1_rc_info = av1_rc_info; - let begin_coding_info = if is_first_frame { - vk::VideoBeginCodingInfoKHR::default() - .video_session(common.session) - .video_session_parameters(common.session_params) - .reference_slots(&all_reference_slots) - .push(&mut begin_av1_rc_info) - } else { - vk::VideoBeginCodingInfoKHR::default() - .video_session(common.session) - .video_session_parameters(common.session_params) - .reference_slots(&all_reference_slots) - .push(&mut begin_rc_info) - .push(&mut begin_av1_rc_info) - }; - - unsafe { - common - .video_queue_fn - .cmd_begin_video_coding(command_buffer, &begin_coding_info); - } - - if should_reset_coding_state { - let mut quality_level_info = - vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); - let control_info = vk::VideoCodingControlInfoKHR::default() - .flags( - vk::VideoCodingControlFlagsKHR::RESET - | vk::VideoCodingControlFlagsKHR::ENCODE_RATE_CONTROL - | vk::VideoCodingControlFlagsKHR::ENCODE_QUALITY_LEVEL, - ) - .push(&mut rc_info) - .push(&mut av1_rc_info) - .push(&mut quality_level_info); - unsafe { - common - .video_queue_fn - .cmd_control_video_coding(command_buffer, &control_info); - } - } - let src_picture_resource = vk::VideoPictureResourceInfoKHR::default() .coded_offset(vk::Offset2D { x: 0, y: 0 }) .coded_extent(frame_extent) @@ -398,43 +330,18 @@ impl Av1 { } encode_info = encode_info.push(&mut av1_picture_info); - unsafe { - let device = common.device(); - device.cmd_begin_query( - command_buffer, - query_pool, - 0, - vk::QueryControlFlags::empty(), - ); - common - .video_encode_fn - .cmd_encode_video(command_buffer, &encode_info); - device.cmd_end_query(command_buffer, query_pool, 0); - - // Write end timestamp - end_timestamp(common.device(), command_buffer, timestamp_query_pool); - - record_post_encode_dpb_barrier( - device, - command_buffer, - &common.dpb_images, - common.use_layered_dpb, - current_dpb_slot, - ); - - let end_coding_info = vk::VideoEndCodingInfoKHR::default(); - common - .video_queue_fn - .cmd_end_video_coding(command_buffer, &end_coding_info); - - common.record_source_release(command_buffer); - - device - .end_command_buffer(command_buffer) - .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; - } - - common.submit_frame() + // A key frame with a bitrate controller resets it too. + let first_frame = plan.is_first_frame(); + common.record_encode(EncodeCommands { + begin_slots: &all_reference_slots, + encode_info: &encode_info, + rate_control: &rc, + codec_rate_control, + codec_layer: av1_rc_layer_info, + first_frame, + reset: first_frame || (is_key_frame && !rc.is_disabled()), + post_encode_barrier: true, + }) } /// Build `ref_frame_idx`, `ref_order_hint`, `primary_ref_frame` and diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 717704b8..610261d8 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -25,8 +25,9 @@ use crate::encoder::pipeline::{ use crate::encoder::profile::EncodeProfile; use crate::encoder::resources::{ EncoderTeardown, UploadParams, align_up, allocate_session_memory, create_command_resources, - create_dpb_images, destroy_encoder_resources, get_video_format, lcm, - query_supported_video_formats, upload_image_to_input, + create_dpb_images, destroy_encoder_resources, end_timestamp, get_video_format, lcm, + query_supported_video_formats, record_post_encode_dpb_barrier, reset_start_timestamp, + upload_image_to_input, }; use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControl, StreamColor}; @@ -312,6 +313,116 @@ impl EncoderCommon { Ok(future) } + /// Record one frame's encode into the current slot's command buffer, + /// from begin coding to the end of the command buffer, and submit it. + /// + /// The codec builds what differs: the reference slots, the encode info + /// with its picture info, and its rate control structs. + pub fn record_encode(&mut self, cmd: EncodeCommands<'_, R, L>) -> Result + where + R: for<'x> vk::Extends> + + for<'x> vk::Extends> + + TaggedStructure<'static> + + Copy, + L: for<'x> vk::Extends> + + TaggedStructure<'static>, + { + let slot = self.pipeline.current(); + let command_buffer = slot.encode_command_buffer; + let query_pool = slot.query_pool; + let timestamp_query_pool = slot.timestamp_query_pool; + let rc = cmd.rate_control; + + let mut codec_layer = cmd.codec_layer; + let rc_layers = [vk::VideoEncodeRateControlLayerInfoKHR::default() + .average_bitrate(rc.average_bitrate as u64) + .max_bitrate(rc.max_bitrate as u64) + .frame_rate_numerator(self.config.frame_rate.numerator) + .frame_rate_denominator(self.config.frame_rate.denominator) + .push(&mut codec_layer)]; + let mut rc_info = vk::VideoEncodeRateControlInfoKHR::default().rate_control_mode(rc.mode); + if !rc.is_disabled() { + rc_info = rc_info + .layers(&rc_layers) + .virtual_buffer_size_in_ms(self.config.virtual_buffer_size_ms) + .initial_virtual_buffer_size_in_ms(self.config.initial_virtual_buffer_size_ms); + } + + reset_start_timestamp(self.device(), command_buffer, timestamp_query_pool); + + // Ash links pNext chains in place, so each command needs its own + // extension structs. The first frame sets rate control with the + // control command below, after RESET, rather than here. + let mut begin_codec_rc = cmd.codec_rate_control; + let mut begin_rc = rc_info; + let mut begin_info = vk::VideoBeginCodingInfoKHR::default() + .video_session(self.session) + .video_session_parameters(self.session_params) + .reference_slots(cmd.begin_slots) + .push(&mut begin_codec_rc); + if !cmd.first_frame { + begin_info = begin_info.push(&mut begin_rc); + } + unsafe { + self.video_queue_fn + .cmd_begin_video_coding(command_buffer, &begin_info); + } + + // RESET + RATE_CONTROL + QUALITY_LEVEL in one control command (matches + // FFmpeg; required for AMD RADV). + if cmd.reset { + let mut codec_rc = cmd.codec_rate_control; + let mut quality_level_info = + vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let control_info = vk::VideoCodingControlInfoKHR::default() + .flags( + vk::VideoCodingControlFlagsKHR::RESET + | vk::VideoCodingControlFlagsKHR::ENCODE_RATE_CONTROL + | vk::VideoCodingControlFlagsKHR::ENCODE_QUALITY_LEVEL, + ) + .push(&mut rc_info) + .push(&mut codec_rc) + .push(&mut quality_level_info); + unsafe { + self.video_queue_fn + .cmd_control_video_coding(command_buffer, &control_info); + } + } + + unsafe { + let device = self.device(); + device.cmd_begin_query( + command_buffer, + query_pool, + 0, + vk::QueryControlFlags::empty(), + ); + self.video_encode_fn + .cmd_encode_video(command_buffer, cmd.encode_info); + device.cmd_end_query(command_buffer, query_pool, 0); + end_timestamp(device, command_buffer, timestamp_query_pool); + + if cmd.post_encode_barrier { + record_post_encode_dpb_barrier( + device, + command_buffer, + &self.dpb_images, + self.use_layered_dpb, + self.current_dpb_slot, + ); + } + + self.video_queue_fn + .cmd_end_video_coding(command_buffer, &vk::VideoEndCodingInfoKHR::default()); + self.record_source_release(command_buffer); + self.device() + .end_command_buffer(command_buffer) + .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; + } + + self.submit_frame() + } + /// Epilogue: advance to the next pipeline slot. pub fn advance(&mut self) { self.pipeline.advance(); @@ -375,6 +486,30 @@ pub(crate) struct PictureSetup { pub header: Option>, } +/// What differs between the codecs in one frame's encode commands, for +/// [`EncoderCommon::record_encode`]. +pub(crate) struct EncodeCommands<'a, R, L> { + /// The slots for begin coding: the setup slot as inactive (index -1), + /// if the frame is a reference, then the references it uses. + pub begin_slots: &'a [vk::VideoReferenceSlotInfoKHR<'a>], + /// The encode, with the codec's picture info chained in. + pub encode_info: &'a vk::VideoEncodeInfoKHR<'a>, + pub rate_control: &'a RateControlPlan, + /// The codec's rate control info, such as + /// `VideoEncodeH264RateControlInfoKHR`. + pub codec_rate_control: R, + /// The codec's rate control layer info, such as + /// `VideoEncodeH264RateControlLayerInfoKHR`. + pub codec_layer: L, + /// Whether this is the session's first frame. + pub first_frame: bool, + /// Whether to reset the coding state and set rate control and the + /// quality level with a control command. At least on the first frame. + pub reset: bool, + /// Whether to make the DPB write visible with a barrier after the encode. + pub post_encode_barrier: bool, +} + /// The resolved rate-control decision for a frame. /// /// The CQP/CBR/VBR selection logic is identical across codecs; only the default diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index 440e05ef..235d42e2 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -3,12 +3,10 @@ use super::H264; -use crate::encoder::codec::{EncoderCommon, FramePlan, RateControlPlan}; +use crate::encoder::codec::{EncodeCommands, EncoderCommon, FramePlan, RateControlPlan}; use crate::encoder::pipeline::EncodeFuture; -use crate::encoder::resources::{ - end_timestamp, prepare_encode_command_buffer, record_dpb_barriers, reset_start_timestamp, -}; -use crate::error::{PixelForgeError, Result}; +use crate::encoder::resources::{prepare_encode_command_buffer, record_dpb_barriers}; +use crate::error::Result; use ash::vk; use ash::vk::TaggedStructure; use tracing::debug; @@ -27,7 +25,6 @@ impl H264 { let slot = common.pipeline.current(); let command_buffer = slot.encode_command_buffer; let query_pool = slot.query_pool; - let timestamp_query_pool = slot.timestamp_query_pool; let bitstream_buffer = slot.bitstream_buffer; let bitstream_buffer_size = slot.bitstream_buffer_size; let input_image_view = common.source_view(); @@ -372,104 +369,26 @@ impl H264 { qp_p: qp_bounds_max, qp_b: qp_bounds_max, }; - - let mut h264_rc_layer_info = vk::VideoEncodeH264RateControlLayerInfoKHR::default() + let codec_layer = vk::VideoEncodeH264RateControlLayerInfoKHR::default() .use_min_qp(true) .min_qp(min_qp) .use_max_qp(true) .max_qp(max_qp); - let rc_layer_info = vk::VideoEncodeRateControlLayerInfoKHR::default() - .average_bitrate(rc.average_bitrate as u64) - .max_bitrate(rc.max_bitrate as u64) - .frame_rate_numerator(common.config.frame_rate.numerator) - .frame_rate_denominator(common.config.frame_rate.denominator) - .push(&mut h264_rc_layer_info); - let rc_layers = [rc_layer_info]; - - let mut h264_rc_info = vk::VideoEncodeH264RateControlInfoKHR::default() + let codec_rate_control = vk::VideoEncodeH264RateControlInfoKHR::default() .gop_frame_count(common.config.gop_size) .idr_period(common.config.gop_size); - let mut rc_info = vk::VideoEncodeRateControlInfoKHR::default().rate_control_mode(rc.mode); - if !rc.is_disabled() { - rc_info = rc_info - .layers(&rc_layers) - .virtual_buffer_size_in_ms(common.config.virtual_buffer_size_ms) - .initial_virtual_buffer_size_in_ms(common.config.initial_virtual_buffer_size_ms); - } - - // Reset and write start timestamp - reset_start_timestamp(common.device(), command_buffer, timestamp_query_pool); - - // For the first frame, configure rate control via the control command - // after RESET rather than in begin_coding. - let is_first_frame = plan.is_first_frame(); - let begin_info = if is_first_frame { - vk::VideoBeginCodingInfoKHR::default() - .video_session(common.session) - .video_session_parameters(common.session_params) - .reference_slots(&reference_slots_for_begin) - .push(&mut h264_rc_info) - } else { - vk::VideoBeginCodingInfoKHR::default() - .video_session(common.session) - .video_session_parameters(common.session_params) - .reference_slots(&reference_slots_for_begin) - .push(&mut h264_rc_info) - .push(&mut rc_info) - }; - - unsafe { - (common.video_queue_fn.fp().cmd_begin_video_coding_khr)(command_buffer, &begin_info); - } - - // RESET + RATE_CONTROL + QUALITY_LEVEL in one control command on the first - // frame (matches FFmpeg; required for AMD RADV). - if is_first_frame { - let mut quality_level_info = - vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); - let control_info = vk::VideoCodingControlInfoKHR::default() - .flags( - vk::VideoCodingControlFlagsKHR::RESET - | vk::VideoCodingControlFlagsKHR::ENCODE_RATE_CONTROL - | vk::VideoCodingControlFlagsKHR::ENCODE_QUALITY_LEVEL, - ) - .push(&mut rc_info) - .push(&mut h264_rc_info) - .push(&mut quality_level_info); - unsafe { - (common.video_queue_fn.fp().cmd_control_video_coding_khr)( - command_buffer, - &control_info, - ); - } - } - - unsafe { - let device = common.device(); - device.cmd_begin_query( - command_buffer, - query_pool, - 0, - vk::QueryControlFlags::empty(), - ); - (common.video_encode_fn.fp().cmd_encode_video_khr)(command_buffer, &encode_info); - device.cmd_end_query(command_buffer, query_pool, 0); - - // Write end timestamp - end_timestamp(common.device(), command_buffer, timestamp_query_pool); - - let end_info = vk::VideoEndCodingInfoKHR::default(); - (common.video_queue_fn.fp().cmd_end_video_coding_khr)(command_buffer, &end_info); - - common.record_source_release(command_buffer); - - device - .end_command_buffer(command_buffer) - .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; - } - - let future = common.submit_frame()?; + let first_frame = plan.is_first_frame(); + let future = common.record_encode(EncodeCommands { + begin_slots: &reference_slots_for_begin, + encode_info: &encode_info, + rate_control: &rc, + codec_rate_control, + codec_layer, + first_frame, + reset: first_frame, + post_encode_barrier: false, + })?; // Clear the unmark queue only after the encode is committed to the GPU. // If any fallible step above had failed, the MMCO ops would still live // in `pending_unmark_frame_nums` so a retry re-emits them — otherwise the diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index 82514949..6d1228df 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -3,13 +3,10 @@ use super::H265; -use crate::encoder::codec::{EncoderCommon, FramePlan, RateControlPlan}; +use crate::encoder::codec::{EncodeCommands, EncoderCommon, FramePlan, RateControlPlan}; use crate::encoder::pipeline::EncodeFuture; -use crate::encoder::resources::{ - end_timestamp, prepare_encode_command_buffer, record_dpb_barriers, - record_post_encode_dpb_barrier, reset_start_timestamp, -}; -use crate::error::{PixelForgeError, Result}; +use crate::encoder::resources::{prepare_encode_command_buffer, record_dpb_barriers}; +use crate::error::Result; use ash::vk; use ash::vk::TaggedStructure; use tracing::debug; @@ -27,7 +24,6 @@ impl H265 { let slot = common.pipeline.current(); let command_buffer = slot.encode_command_buffer; let query_pool = slot.query_pool; - let timestamp_query_pool = slot.timestamp_query_pool; let bitstream_buffer = slot.bitstream_buffer; let bitstream_buffer_size = slot.bitstream_buffer_size; let input_image_view = common.source_view(); @@ -369,73 +365,12 @@ impl H265 { qp_p: qp_max, qp_b: qp_max, }; - let mut h265_rc_layer_info = vk::VideoEncodeH265RateControlLayerInfoKHR::default() + let codec_layer = vk::VideoEncodeH265RateControlLayerInfoKHR::default() .min_qp(min_qp) .max_qp(max_qp); - let rc_layer_info = vk::VideoEncodeRateControlLayerInfoKHR::default() - .average_bitrate(rc.average_bitrate as u64) - .max_bitrate(rc.max_bitrate as u64) - .frame_rate_numerator(common.config.frame_rate.numerator) - .frame_rate_denominator(common.config.frame_rate.denominator) - .push(&mut h265_rc_layer_info); - let rc_layers = [rc_layer_info]; - - let mut h265_rc_info = vk::VideoEncodeH265RateControlInfoKHR::default() + let codec_rate_control = vk::VideoEncodeH265RateControlInfoKHR::default() .gop_frame_count(common.config.gop_size) .idr_period(common.config.gop_size); - let mut rc_info = vk::VideoEncodeRateControlInfoKHR::default().rate_control_mode(rc.mode); - if !rc.is_disabled() { - rc_info = rc_info - .layers(&rc_layers) - .virtual_buffer_size_in_ms(common.config.virtual_buffer_size_ms) - .initial_virtual_buffer_size_in_ms(common.config.initial_virtual_buffer_size_ms); - } - - // Reset and write start timestamp - reset_start_timestamp(common.device(), command_buffer, timestamp_query_pool); - - let is_first_frame = plan.is_first_frame(); - let begin_coding_info = if is_first_frame { - vk::VideoBeginCodingInfoKHR::default() - .video_session(common.session) - .video_session_parameters(common.session_params) - .reference_slots(&reference_slots_for_begin) - .push(&mut h265_rc_info) - } else { - vk::VideoBeginCodingInfoKHR::default() - .video_session(common.session) - .video_session_parameters(common.session_params) - .reference_slots(&reference_slots_for_begin) - .push(&mut h265_rc_info) - .push(&mut rc_info) - }; - - unsafe { - (common.video_queue_fn.fp().cmd_begin_video_coding_khr)( - command_buffer, - &begin_coding_info, - ); - } - - if is_first_frame { - let mut quality_level_info = - vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); - let control_info = vk::VideoCodingControlInfoKHR::default() - .flags( - vk::VideoCodingControlFlagsKHR::RESET - | vk::VideoCodingControlFlagsKHR::ENCODE_RATE_CONTROL - | vk::VideoCodingControlFlagsKHR::ENCODE_QUALITY_LEVEL, - ) - .push(&mut rc_info) - .push(&mut h265_rc_info) - .push(&mut quality_level_info); - unsafe { - (common.video_queue_fn.fp().cmd_control_video_coding_khr)( - command_buffer, - &control_info, - ); - } - } let encode_info = vk::VideoEncodeInfoKHR::default() .flags(vk::VideoEncodeFlagsKHR::empty()) @@ -455,38 +390,16 @@ impl H265 { common.current_dpb_slot ); - unsafe { - let device = common.device(); - device.cmd_begin_query( - command_buffer, - query_pool, - 0, - vk::QueryControlFlags::empty(), - ); - (common.video_encode_fn.fp().cmd_encode_video_khr)(command_buffer, &encode_info); - device.cmd_end_query(command_buffer, query_pool, 0); - - // Write end timestamp - end_timestamp(common.device(), command_buffer, timestamp_query_pool); - - record_post_encode_dpb_barrier( - device, - command_buffer, - &common.dpb_images, - common.use_layered_dpb, - common.current_dpb_slot, - ); - - let end_coding_info = vk::VideoEndCodingInfoKHR::default(); - (common.video_queue_fn.fp().cmd_end_video_coding_khr)(command_buffer, &end_coding_info); - - common.record_source_release(command_buffer); - - device - .end_command_buffer(command_buffer) - .map_err(PixelForgeError::vk("vkEndCommandBuffer"))?; - } - - common.submit_frame() + let first_frame = plan.is_first_frame(); + common.record_encode(EncodeCommands { + begin_slots: &reference_slots_for_begin, + encode_info: &encode_info, + rate_control: &rc, + codec_rate_control, + codec_layer, + first_frame, + reset: first_frame, + post_encode_barrier: true, + }) } } From ffa7fe708ed458f3142a2d58c3be19eaeaf477e8 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 08:21:31 +0100 Subject: [PATCH 7/9] refactor(encoder): remove the DPB tracker that nothing read --- src/encoder/codec.rs | 9 +- src/encoder/dpb/entry.rs | 186 ------------------- src/encoder/dpb/h264.rs | 385 --------------------------------------- src/encoder/dpb/h265.rs | 359 ------------------------------------ src/encoder/dpb/mod.rs | 94 ---------- src/encoder/dpb/types.rs | 78 -------- src/encoder/gop.rs | 15 +- src/encoder/h264/init.rs | 11 -- src/encoder/h264/mod.rs | 32 +--- src/encoder/h265/init.rs | 10 - src/encoder/h265/mod.rs | 26 --- src/encoder/mod.rs | 1 - 12 files changed, 7 insertions(+), 1199 deletions(-) delete mode 100644 src/encoder/dpb/entry.rs delete mode 100644 src/encoder/dpb/h264.rs delete mode 100644 src/encoder/dpb/h265.rs delete mode 100644 src/encoder/dpb/mod.rs delete mode 100644 src/encoder/dpb/types.rs diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 610261d8..81832385 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -16,7 +16,6 @@ use ash::vk; use ash::vk::TaggedStructure; use crate::converter::InputFormat; -use crate::encoder::dpb::MAX_DPB_SLOTS; use crate::encoder::gop::{GopFrameType, GopPosition, GopStructure}; use crate::encoder::input::{FrameInput, FrameSync, SourceOwnership}; use crate::encoder::pipeline::{ @@ -36,6 +35,9 @@ use crate::sync::TimelinePoint; use crate::video::{TimelineChain, allocate_command_buffers}; use crate::vulkan::VideoContext; +/// The most DPB slots an encoder uses, for any codec. +pub(crate) const MAX_DPB_SLOTS: usize = 16; + /// Per-encoder state shared by every codec. /// /// Holds the Vulkan video session, the DPB images, the async [`EncodePipeline`], @@ -1074,10 +1076,9 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result upload_fence: cmd.upload_fence, })?; - // B-frames are not yet supported, so an I-P GOP. Set the SPS-matching - // counters (no-ops for AV1, which keys off order hints). + // B-frames are not yet supported, so an I-P GOP. Match the SPS's POC + // wrap (no-op for AV1, which keys off order hints). let mut gop = GopStructure::new(config.gop_size); - gop.set_max_frame_num(4); gop.set_max_poc_lsb(4); let common = EncoderCommon { diff --git a/src/encoder/dpb/entry.rs b/src/encoder/dpb/entry.rs deleted file mode 100644 index e77548f5..00000000 --- a/src/encoder/dpb/entry.rs +++ /dev/null @@ -1,186 +0,0 @@ -//! DPB entry structures for individual pictures in the buffer. - -use super::{MAX_DPB_SIZE, PictureType}; - -/// Reference marking state for a picture. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum MarkingState { - /// Unused for reference. - #[default] - Unused, - /// Used for short-term reference. - ShortTerm, - /// Used for long-term reference. - LongTerm, -} - -impl MarkingState { - /// Returns true if this entry is used for reference. - pub fn is_reference(&self) -> bool { - !matches!(self, Self::Unused) - } - - /// Returns true if this is a short-term reference. - pub fn is_short_term(&self) -> bool { - matches!(self, Self::ShortTerm) - } - - /// Returns true if this is a long-term reference. - pub fn is_long_term(&self) -> bool { - matches!(self, Self::LongTerm) - } -} - -/// DPB entry state (empty or in use). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum DpbState { - /// Slot is empty. - #[default] - Empty, - /// Slot contains a frame. - InUse, -} - -/// A single entry in the Decoded Picture Buffer. -#[derive(Debug, Clone)] -pub struct DpbEntry { - /// State of this DPB slot. - pub state: DpbState, - /// Reference marking for top field / frame. - pub marking: MarkingState, - /// Reference marking for bottom field (H.264 field coding). - pub bottom_field_marking: MarkingState, - /// Whether this picture is needed for output. - pub output: bool, - /// Whether this frame is corrupted (for error resilience). - pub corrupted: bool, - /// Picture Order Count. - pub pic_order_cnt: i32, - /// Top field order count (H.264). - pub top_foc: i32, - /// Bottom field order count (H.264). - pub bottom_foc: i32, - /// Frame number (H.264 frame_num syntax element). - pub frame_num: u32, - /// Frame number wrapped for POC calculation. - pub frame_num_wrap: i32, - /// Top field picture number (H.264). - pub top_pic_num: i32, - /// Bottom field picture number (H.264). - pub bottom_pic_num: i32, - /// Long-term frame index. - pub long_term_frame_idx: i32, - /// Top field long-term picture number. - pub top_long_term_pic_num: i32, - /// Bottom field long-term picture number. - pub bottom_long_term_pic_num: i32, - /// POC values of reference pictures (for H.265 RPS). - pub ref_pic_order_cnt: [i32; MAX_DPB_SIZE], - /// Bitmask indicating which refs are long-term. - pub long_term_ref_pic: u32, - /// Internal unique frame ID. - pub frame_id: u64, - /// Temporal layer ID. - pub temporal_id: i32, - /// Timestamp of this picture. - pub timestamp: u64, - /// Picture type. - pub pic_type: PictureType, - /// Whether this entry represents a non-existing frame (gap handling). - pub not_existing: bool, -} - -impl Default for DpbEntry { - fn default() -> Self { - Self::new() - } -} - -impl DpbEntry { - /// Create a new empty DPB entry. - pub fn new() -> Self { - Self { - state: DpbState::Empty, - marking: MarkingState::Unused, - bottom_field_marking: MarkingState::Unused, - output: false, - corrupted: false, - pic_order_cnt: 0, - top_foc: 0, - bottom_foc: 0, - frame_num: 0, - frame_num_wrap: 0, - top_pic_num: 0, - bottom_pic_num: 0, - long_term_frame_idx: -1, - top_long_term_pic_num: 0, - bottom_long_term_pic_num: 0, - ref_pic_order_cnt: [0; MAX_DPB_SIZE], - long_term_ref_pic: 0, - frame_id: 0, - temporal_id: 0, - timestamp: 0, - pic_type: PictureType::I, - not_existing: false, - } - } - - /// Reset this entry to empty state. - pub fn reset(&mut self) { - *self = Self::new(); - } - - /// Check if this entry is a reference picture. - pub fn is_reference(&self) -> bool { - self.marking.is_reference() || self.bottom_field_marking.is_reference() - } - - /// Check if this entry is a short-term reference. - pub fn is_short_term_reference(&self) -> bool { - self.marking.is_short_term() || self.bottom_field_marking.is_short_term() - } - - /// Check if this entry is a long-term reference. - pub fn is_long_term_reference(&self) -> bool { - self.marking.is_long_term() || self.bottom_field_marking.is_long_term() - } - - /// Mark as unused for reference. - pub fn mark_unused(&mut self) { - self.marking = MarkingState::Unused; - self.bottom_field_marking = MarkingState::Unused; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_dpb_entry_default() { - let entry = DpbEntry::new(); - assert_eq!(entry.state, DpbState::Empty); - assert_eq!(entry.marking, MarkingState::Unused); - assert!(!entry.is_reference()); - } - - #[test] - fn test_marking_state() { - let mut entry = DpbEntry::new(); - - entry.marking = MarkingState::ShortTerm; - entry.bottom_field_marking = MarkingState::ShortTerm; - assert!(entry.is_short_term_reference()); - assert!(entry.is_reference()); - assert!(!entry.is_long_term_reference()); - - entry.marking = MarkingState::LongTerm; - entry.bottom_field_marking = MarkingState::LongTerm; - assert!(entry.is_long_term_reference()); - assert!(entry.is_reference()); - assert!(!entry.is_short_term_reference()); - - entry.mark_unused(); - assert!(!entry.is_reference()); - } -} diff --git a/src/encoder/dpb/h264.rs b/src/encoder/dpb/h264.rs deleted file mode 100644 index 7dea5aa6..00000000 --- a/src/encoder/dpb/h264.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! H.264/AVC-specific DPB implementation. -//! -//! This module implements the H.264 DPB according to the specification, -//! including: -//! - POC calculation (Type 0) -//! - Picture number (PicNum) derivation -//! - Sliding window memory management - -use super::entry::{DpbEntry, DpbState, MarkingState}; -use super::types::{DpbConfig, PictureStartInfo}; -use super::{DecodedPictureBufferTrait, MAX_DPB_SIZE, PictureType}; - -/// H.264-specific DPB implementation. -#[derive(Debug)] -pub struct DpbH264 { - /// DPB entries. - entries: [DpbEntry; MAX_DPB_SIZE], - /// Current DPB slot index (-1 if none). - current_slot: i8, - /// Maximum DPB size. - max_dpb_size: i8, - /// Maximum long-term frame index (-1 means no LTR allowed). - max_long_term_frame_idx: i32, - /// Previous POC MSB (for POC Type 0). - prev_poc_msb: i32, - /// Previous POC LSB (for POC Type 0). - prev_poc_lsb: i32, - /// Previous frame_num offset (for POC Type 2). - prev_frame_num_offset: i32, - /// Previous frame_num. - prev_frame_num: u32, - /// Previous reference frame_num. - prev_ref_frame_num: u32, - /// Maximum frame_num value. - max_frame_num: u32, - /// Maximum POC LSB value. - max_poc_lsb: i32, - /// Last IDR timestamp. - last_idr_timestamp: u64, - /// Number of temporal layers. - num_temporal_layers: u32, -} - -impl Default for DpbH264 { - fn default() -> Self { - Self::new() - } -} - -impl DpbH264 { - /// Create a new H.264 DPB. - pub fn new() -> Self { - Self { - entries: std::array::from_fn(|_| DpbEntry::new()), - current_slot: -1, - max_dpb_size: 0, - max_long_term_frame_idx: -1, - prev_poc_msb: 0, - prev_poc_lsb: 0, - prev_frame_num_offset: 0, - prev_frame_num: 0, - prev_ref_frame_num: 0, - max_frame_num: 16, - max_poc_lsb: 16, - last_idr_timestamp: 0, - num_temporal_layers: 1, - } - } - - /// Calculate POC for the current picture. - fn calculate_poc(&mut self, info: &PictureStartInfo) { - self.calculate_poc_type0(info); - - // Derive PicOrderCnt (8-1) - let entry = &mut self.entries[self.current_slot as usize]; - entry.pic_order_cnt = std::cmp::min(entry.top_foc, entry.bottom_foc); - } - - /// Calculate POC Type 0 (8.2.1.1). - fn calculate_poc_type0(&mut self, info: &PictureStartInfo) { - if info.pic_type == PictureType::Idr { - self.prev_poc_msb = 0; - self.prev_poc_lsb = 0; - } - - let poc_lsb = info.pic_order_cnt; - - // (8-3) - let poc_msb = if (poc_lsb < self.prev_poc_lsb) - && ((self.prev_poc_lsb - poc_lsb) >= (self.max_poc_lsb / 2)) - { - self.prev_poc_msb + self.max_poc_lsb - } else if (poc_lsb > self.prev_poc_lsb) - && ((poc_lsb - self.prev_poc_lsb) > (self.max_poc_lsb / 2)) - { - self.prev_poc_msb - self.max_poc_lsb - } else { - self.prev_poc_msb - }; - - let entry = &mut self.entries[self.current_slot as usize]; - - // (8-4) and (8-5) - entry.top_foc = poc_msb + poc_lsb; - entry.bottom_foc = poc_msb + poc_lsb; - - if info.is_reference { - self.prev_poc_msb = poc_msb; - self.prev_poc_lsb = poc_lsb; - } - } - - /// Calculate PicNum for all DPB entries (8.2.4.1). - fn calculate_pic_num(&mut self, frame_num: u32) { - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - - // (8-28) FrameNumWrap - if entry.frame_num > frame_num { - entry.frame_num_wrap = entry.frame_num as i32 - self.max_frame_num as i32; - } else { - entry.frame_num_wrap = entry.frame_num as i32; - } - - // For frame coding (not field): - // (8-29) PicNum = FrameNumWrap - entry.top_pic_num = entry.frame_num_wrap; - entry.bottom_pic_num = entry.frame_num_wrap; - - // (8-30) LongTermPicNum = LongTermFrameIdx - entry.top_long_term_pic_num = entry.long_term_frame_idx; - entry.bottom_long_term_pic_num = entry.long_term_frame_idx; - } - } - - /// DPB bumping process (C.4.5.3). - fn dpb_bumping(&mut self) { - // Find picture with smallest POC that needs output. - let mut min_poc = i32::MAX; - let mut min_idx: Option = None; - - for i in 0..self.max_dpb_size as usize { - let entry = &self.entries[i]; - if entry.state == DpbState::InUse && entry.output && entry.pic_order_cnt < min_poc { - min_poc = entry.pic_order_cnt; - min_idx = Some(i); - } - } - - if let Some(idx) = min_idx { - self.entries[idx].output = false; - // If also not a reference, mark slot as empty. - if !self.entries[idx].is_reference() { - self.entries[idx].state = DpbState::Empty; - } - } - } -} - -impl DecodedPictureBufferTrait for DpbH264 { - fn sequence_start(&mut self, config: DpbConfig) { - // Reset all entries. - for entry in &mut self.entries { - entry.reset(); - } - - self.max_dpb_size = std::cmp::min(config.dpb_size as i8, MAX_DPB_SIZE as i8); - self.max_long_term_frame_idx = -1; - self.max_frame_num = config.max_frame_num(); - self.max_poc_lsb = config.max_pic_order_cnt_lsb(); - self.num_temporal_layers = config.num_temporal_layers; - self.current_slot = -1; - self.prev_poc_msb = 0; - self.prev_poc_lsb = 0; - self.prev_frame_num_offset = 0; - self.prev_frame_num = 0; - self.prev_ref_frame_num = 0; - } - - fn picture_start(&mut self, info: PictureStartInfo) -> i8 { - // Handle IDR. - if info.pic_type == PictureType::Idr { - if !info.no_output_of_prior_pics_flag { - while !self.is_empty() { - self.dpb_bumping(); - } - } - // Clear DPB for IDR. - for entry in &mut self.entries { - entry.state = DpbState::Empty; - entry.mark_unused(); - } - self.last_idr_timestamp = info.timestamp; - } - - // Ensure space. - while self.is_full() { - self.dpb_bumping(); - } - - // Find empty slot. - self.current_slot = -1; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::Empty { - self.current_slot = i as i8; - break; - } - } - - if self.current_slot < 0 { - // Force bumping if no slot found. - self.dpb_bumping(); - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::Empty { - self.current_slot = i as i8; - break; - } - } - } - - if self.current_slot >= 0 { - let entry = &mut self.entries[self.current_slot as usize]; - entry.state = DpbState::InUse; - entry.frame_id = info.frame_id; - entry.frame_num = info.frame_num; - entry.pic_type = info.pic_type; - entry.temporal_id = info.temporal_id as i32; - entry.timestamp = info.timestamp; - entry.output = info.pic_output_flag; - entry.corrupted = false; - entry.not_existing = false; - - // Calculate POC. - self.calculate_poc(&info); - - // Calculate PicNum. - self.calculate_pic_num(info.frame_num); - } - - self.current_slot - } - - fn picture_end(&mut self, is_reference: bool) { - if self.current_slot < 0 { - return; - } - - let entry = &mut self.entries[self.current_slot as usize]; - - // Mark as reference if needed. - if is_reference { - if entry.marking == MarkingState::Unused { - entry.marking = MarkingState::ShortTerm; - } - if entry.bottom_field_marking == MarkingState::Unused { - entry.bottom_field_marking = MarkingState::ShortTerm; - } - - // Apply sliding window memory management to remove oldest reference. - // if we exceed the max number of reference frames - let num_short_term = self.num_short_term_refs(); - let num_long_term = self.num_long_term_refs(); - let max_refs = (self.max_dpb_size - 1) as u32; // Reserve one slot for current - - if (num_short_term + num_long_term) > max_refs && num_short_term > 0 { - // Find short-term ref with lowest FrameNumWrap (oldest) - let mut oldest_idx: Option = None; - let mut lowest_frame_num_wrap = i32::MAX; - - for i in 0..self.max_dpb_size as usize { - let e = &self.entries[i]; - if e.is_short_term_reference() && i as i8 != self.current_slot { - // frame_num_wrap is a rough approximation based on frame_num - let frame_num_wrap = e.frame_num as i32; - if frame_num_wrap < lowest_frame_num_wrap { - lowest_frame_num_wrap = frame_num_wrap; - oldest_idx = Some(i); - } - } - } - - if let Some(idx) = oldest_idx { - self.entries[idx].mark_unused(); - if !self.entries[idx].output { - self.entries[idx].state = DpbState::Empty; - } - } - } - } else { - entry.mark_unused(); - } - } - - fn is_full(&self) -> bool { - let mut count = 0; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::InUse { - count += 1; - } - } - count >= self.max_dpb_size as usize - } - - fn is_empty(&self) -> bool { - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::InUse { - return false; - } - } - true - } - - fn flush(&mut self) { - // Mark all as unused for reference. - for entry in &mut self.entries { - entry.mark_unused(); - } - // Bump until empty. - while !self.is_empty() { - self.dpb_bumping(); - } - } - - fn num_short_term_refs(&self) -> u32 { - let mut count = 0; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].is_short_term_reference() { - count += 1; - } - } - count - } - - fn num_long_term_refs(&self) -> u32 { - let mut count = 0; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].is_long_term_reference() { - count += 1; - } - } - count - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_h264_dpb_sequence_start() { - let mut dpb = DpbH264::new(); - dpb.sequence_start(DpbConfig { - dpb_size: 4, - ..Default::default() - }); - - assert_eq!(dpb.max_dpb_size, 4); - assert!(dpb.is_empty()); - assert!(!dpb.is_full()); - } - - #[test] - fn test_h264_dpb_picture_start_end() { - let mut dpb = DpbH264::new(); - dpb.sequence_start(DpbConfig { - dpb_size: 4, - ..Default::default() - }); - - // Add IDR. - let slot = dpb.picture_start(PictureStartInfo { - frame_id: 0, - pic_order_cnt: 0, - frame_num: 0, - pic_type: PictureType::Idr, - is_reference: true, - ..Default::default() - }); - assert_eq!(slot, 0); - dpb.picture_end(true); - - assert!(!dpb.is_empty()); - assert_eq!(dpb.num_short_term_refs(), 1); - } -} diff --git a/src/encoder/dpb/h265.rs b/src/encoder/dpb/h265.rs deleted file mode 100644 index 81709f97..00000000 --- a/src/encoder/dpb/h265.rs +++ /dev/null @@ -1,359 +0,0 @@ -//! H.265/HEVC-specific DPB implementation. -//! -//! This module keeps track of the H.265 reference pictures: which DPB slot -//! holds each picture, its POC, and whether it is still a reference. -use super::entry::{DpbEntry, DpbState, MarkingState}; -use super::types::{DpbConfig, PictureStartInfo}; -use super::{DecodedPictureBufferTrait, MAX_DPB_SIZE, PictureType}; - -/// H.265-specific DPB implementation. -#[derive(Debug)] -pub struct DpbH265 { - /// DPB entries. - entries: [DpbEntry; MAX_DPB_SIZE], - /// Current DPB slot index (-1 if none). - current_slot: i8, - /// Maximum DPB size. - max_dpb_size: i8, - /// Number of POC values in stCurrBefore. - num_poc_st_curr_before: i8, - /// Number of POC values in stCurrAfter. - num_poc_st_curr_after: i8, - /// Number of POC values in stFoll. - num_poc_st_foll: i8, - /// Number of POC values in ltCurr. - num_poc_lt_curr: i8, - /// Number of POC values in ltFoll. - num_poc_lt_foll: i8, - /// Last IDR timestamp. - last_idr_timestamp: u64, - /// Long-term flags bitmask. - long_term_flags: u32, - /// Maximum POC LSB value. - max_poc_lsb: i32, - /// Number of temporal layers. - num_temporal_layers: u32, -} - -impl Default for DpbH265 { - fn default() -> Self { - Self::new() - } -} - -impl DpbH265 { - /// Create a new H.265 DPB. - pub fn new() -> Self { - Self { - entries: std::array::from_fn(|_| DpbEntry::new()), - current_slot: -1, - max_dpb_size: 0, - num_poc_st_curr_before: 0, - num_poc_st_curr_after: 0, - num_poc_st_foll: 0, - num_poc_lt_curr: 0, - num_poc_lt_foll: 0, - last_idr_timestamp: 0, - long_term_flags: 0, - max_poc_lsb: 16, - num_temporal_layers: 1, - } - } - - /// DPB bumping process. - fn dpb_bumping(&mut self) { - // Find picture with smallest POC that needs output. - let mut min_poc = i32::MAX; - let mut min_idx: Option = None; - - for i in 0..self.max_dpb_size as usize { - let entry = &self.entries[i]; - if entry.state == DpbState::InUse && entry.output && entry.pic_order_cnt < min_poc { - min_poc = entry.pic_order_cnt; - min_idx = Some(i); - } - } - - if let Some(idx) = min_idx { - self.entries[idx].output = false; - if self.entries[idx].marking == MarkingState::Unused { - self.entries[idx].state = DpbState::Empty; - } - } - } -} - -impl DecodedPictureBufferTrait for DpbH265 { - fn sequence_start(&mut self, config: DpbConfig) { - // Reset all entries. - for entry in &mut self.entries { - entry.reset(); - } - - self.max_dpb_size = std::cmp::min(config.dpb_size as i8, MAX_DPB_SIZE as i8); - self.max_poc_lsb = config.max_pic_order_cnt_lsb(); - self.num_temporal_layers = config.num_temporal_layers; - self.current_slot = -1; - self.num_poc_st_curr_before = 0; - self.num_poc_st_curr_after = 0; - self.num_poc_st_foll = 0; - self.num_poc_lt_curr = 0; - self.num_poc_lt_foll = 0; - self.long_term_flags = 0; - } - - fn picture_start(&mut self, info: PictureStartInfo) -> i8 { - let is_irap = info.pic_type == PictureType::Idr || info.pic_type == PictureType::I; - let no_rasl_output_flag = info.pic_type == PictureType::Idr; - - if is_irap && no_rasl_output_flag { - if info.no_output_of_prior_pics_flag { - // Clear all DPB entries. - for entry in &mut self.entries[..self.max_dpb_size as usize] { - entry.state = DpbState::Empty; - entry.mark_unused(); - } - } else { - // Flush DPB. - self.flush(); - } - } else { - // Remove entries that are unused and not needed for output. - for entry in &mut self.entries[..self.max_dpb_size as usize] { - if entry.marking == MarkingState::Unused && !entry.output { - entry.state = DpbState::Empty; - } - } - while self.is_full() { - self.dpb_bumping(); - } - } - - // Find empty slot. - self.current_slot = -1; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::Empty { - self.current_slot = i as i8; - break; - } - } - - if self.current_slot < 0 { - // Force bumping. - self.dpb_bumping(); - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::Empty { - self.current_slot = i as i8; - break; - } - } - } - - if self.current_slot >= 0 { - // Collect reference POCs and long-term flags first. - let mut ref_pocs = [0i32; MAX_DPB_SIZE]; - let mut long_term_mask = 0u32; - for (i, entry) in self - .entries - .iter() - .enumerate() - .take(self.max_dpb_size as usize) - { - ref_pocs[i] = entry.pic_order_cnt; - if entry.marking == MarkingState::LongTerm { - long_term_mask |= 1 << i; - } - } - - let entry = &mut self.entries[self.current_slot as usize]; - entry.state = DpbState::InUse; - entry.frame_id = info.frame_id; - entry.pic_order_cnt = info.pic_order_cnt; - entry.output = info.pic_output_flag; - entry.corrupted = false; - entry.temporal_id = info.temporal_id as i32; - entry.pic_type = info.pic_type; - - // Store reference POCs. - entry.ref_pic_order_cnt = ref_pocs; - entry.long_term_ref_pic = long_term_mask; - - if is_irap && no_rasl_output_flag { - self.last_idr_timestamp = info.timestamp; - } - } - - self.current_slot - } - - fn picture_end(&mut self, is_reference: bool) { - if self.current_slot < 0 { - return; - } - - // For temporal SVC, unmark refs with same temporal ID. - if self.num_temporal_layers > 1 { - let current_temporal_id = self.entries[self.current_slot as usize].temporal_id; - for i in 0..self.max_dpb_size as usize { - let entry = &mut self.entries[i]; - if entry.state == DpbState::InUse - && entry.marking != MarkingState::Unused - && entry.temporal_id == current_temporal_id - && i != self.current_slot as usize - { - entry.mark_unused(); - } - } - } - - let entry = &mut self.entries[self.current_slot as usize]; - entry.marking = if is_reference { - MarkingState::ShortTerm - } else { - MarkingState::Unused - }; - - // Apply sliding window to remove oldest reference if we exceed max refs. - if is_reference { - let num_short_term = self.num_short_term_refs(); - let num_long_term = self.num_long_term_refs(); - let max_refs = (self.max_dpb_size - 1) as u32; // Reserve one slot for current - - if (num_short_term + num_long_term) > max_refs && num_short_term > 0 { - // Find short-term ref with lowest POC (oldest in display order) - let mut oldest_idx: Option = None; - let mut lowest_poc = i32::MAX; - - for i in 0..self.max_dpb_size as usize { - let e = &self.entries[i]; - if e.is_short_term_reference() - && i as i8 != self.current_slot - && e.pic_order_cnt < lowest_poc - { - lowest_poc = e.pic_order_cnt; - oldest_idx = Some(i); - } - } - - if let Some(idx) = oldest_idx { - self.entries[idx].mark_unused(); - if !self.entries[idx].output { - self.entries[idx].state = DpbState::Empty; - } - } - } - } - } - - fn is_full(&self) -> bool { - let mut count = 0; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::InUse { - count += 1; - } - } - count >= self.max_dpb_size as usize - } - - fn is_empty(&self) -> bool { - for i in 0..self.max_dpb_size as usize { - if self.entries[i].state == DpbState::InUse { - return false; - } - } - true - } - - fn flush(&mut self) { - // Mark all as unused. - for entry in &mut self.entries[..self.max_dpb_size as usize] { - entry.mark_unused(); - } - // Empty slots not needed for output. - for entry in &mut self.entries[..self.max_dpb_size as usize] { - if entry.state == DpbState::InUse - && !entry.output - && entry.marking == MarkingState::Unused - { - entry.state = DpbState::Empty; - } - } - // Bump until empty. - while !self.is_empty() { - self.dpb_bumping(); - } - } - - fn num_short_term_refs(&self) -> u32 { - let mut count = 0; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].is_short_term_reference() { - count += 1; - } - } - count - } - - fn num_long_term_refs(&self) -> u32 { - let mut count = 0; - for i in 0..self.max_dpb_size as usize { - if self.entries[i].is_long_term_reference() { - count += 1; - } - } - count - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_h265_dpb_sequence_start() { - let mut dpb = DpbH265::new(); - dpb.sequence_start(DpbConfig { - dpb_size: 4, - ..Default::default() - }); - - assert_eq!(dpb.max_dpb_size, 4); - assert!(dpb.is_empty()); - } - - #[test] - fn test_h265_dpb_picture_flow() { - let mut dpb = DpbH265::new(); - dpb.sequence_start(DpbConfig { - dpb_size: 4, - ..Default::default() - }); - - // Add IDR. - let slot = dpb.picture_start(PictureStartInfo { - frame_id: 0, - pic_order_cnt: 0, - pic_type: PictureType::Idr, - is_reference: true, - ..Default::default() - }); - assert_eq!(slot, 0); - dpb.picture_end(true); - - assert!(!dpb.is_empty()); - assert_eq!(dpb.num_short_term_refs(), 1); - - // Add P frame. - let slot = dpb.picture_start(PictureStartInfo { - frame_id: 1, - pic_order_cnt: 2, - pic_type: PictureType::P, - is_reference: true, - ..Default::default() - }); - assert_eq!(slot, 1); - dpb.picture_end(true); - - assert_eq!(dpb.num_short_term_refs(), 2); - } -} diff --git a/src/encoder/dpb/mod.rs b/src/encoder/dpb/mod.rs deleted file mode 100644 index 4f41962b..00000000 --- a/src/encoder/dpb/mod.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! Decoded Picture Buffer (DPB) management for video encoding. -//! -//! This module tracks the reference pictures of H.264 and H.265 encoding: -//! picture order counts, frame numbers and reference marking. -//! -//! The DPB is codec-agnostic at the interface level, with codec-specific -//! implementations for H.264 and H.265 in separate modules. - -mod entry; -mod h264; -mod h265; -mod types; - -pub use h264::DpbH264; -pub use h265::DpbH265; -pub use types::*; - -/// Maximum DPB size (16 slots for H.264, 15 for H.265, use 16 for both). -pub const MAX_DPB_SIZE: usize = 16; - -/// Alias for MAX_DPB_SIZE for backwards compatibility. -pub const MAX_DPB_SLOTS: usize = MAX_DPB_SIZE; - -/// Default DPB size for encoding. -pub const DEFAULT_DPB_SIZE: u32 = 4; - -/// Picture type for encoding. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PictureType { - /// IDR picture (Instantaneous Decoder Refresh). - Idr, - /// I picture (Intra). - I, - /// P picture (Predicted, uses L0 references). - P, -} - -/// Common DPB interface for both H.264 and H.265. -/// -/// This trait provides a codec-agnostic interface for DPB operations. -pub trait DecodedPictureBufferTrait { - /// Initialize the DPB for a new sequence. - fn sequence_start(&mut self, config: DpbConfig); - - /// Start encoding a new picture. - /// - /// Returns the DPB slot index allocated for this picture. - fn picture_start(&mut self, info: PictureStartInfo) -> i8; - - /// End encoding a picture and update the DPB. - fn picture_end(&mut self, is_reference: bool); - - /// Check if the DPB is full. - fn is_full(&self) -> bool; - - /// Check if the DPB is empty. - fn is_empty(&self) -> bool; - - /// Flush all frames from the DPB. - fn flush(&mut self); - - /// Get the number of short-term reference frames. - fn num_short_term_refs(&self) -> u32; - - /// Get the number of long-term reference frames. - fn num_long_term_refs(&self) -> u32; -} - -/// Unified DPB that works for both H.264 and H.265. -/// -/// This provides a simple interface for encoders while delegating. -/// to codec-specific implementations internally. -pub struct DecodedPictureBuffer { - /// H.264-specific DPB (used when codec is H.264). - pub h264: DpbH264, - /// H.265-specific DPB (used when codec is H.265). - pub h265: DpbH265, -} - -impl DecodedPictureBuffer { - /// Create a new DPB. - pub fn new() -> Self { - Self { - h264: DpbH264::new(), - h265: DpbH265::new(), - } - } -} - -impl Default for DecodedPictureBuffer { - fn default() -> Self { - Self::new() - } -} diff --git a/src/encoder/dpb/types.rs b/src/encoder/dpb/types.rs deleted file mode 100644 index e10c7c6a..00000000 --- a/src/encoder/dpb/types.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! DPB types and configuration structures. - -use super::DEFAULT_DPB_SIZE; - -/// Configuration for DPB initialization. -#[derive(Debug, Clone)] -pub struct DpbConfig { - /// Maximum DPB size in frames. - pub dpb_size: u32, - /// Log2 of max_frame_num (for H.264 frame_num calculation). - pub log2_max_frame_num_minus4: u8, - /// Log2 of max_pic_order_cnt_lsb (for POC calculation). - pub log2_max_pic_order_cnt_lsb_minus4: u8, - /// Number of temporal layers (for temporal SVC support). - pub num_temporal_layers: u32, -} - -impl Default for DpbConfig { - fn default() -> Self { - Self { - dpb_size: DEFAULT_DPB_SIZE, - log2_max_frame_num_minus4: 0, // max_frame_num = 16 - log2_max_pic_order_cnt_lsb_minus4: 0, // max_poc_lsb = 16 - num_temporal_layers: 1, - } - } -} - -impl DpbConfig { - /// Get the maximum frame_num value. - pub fn max_frame_num(&self) -> u32 { - 1 << (self.log2_max_frame_num_minus4 + 4) - } - - /// Get the maximum POC LSB value. - pub fn max_pic_order_cnt_lsb(&self) -> i32 { - 1 << (self.log2_max_pic_order_cnt_lsb_minus4 + 4) - } -} - -/// Information for starting a new picture. -#[derive(Debug, Clone)] -pub struct PictureStartInfo { - /// Unique frame ID (internal tracking). - pub frame_id: u64, - /// Picture Order Count. - pub pic_order_cnt: i32, - /// Frame number (for H.264). - pub frame_num: u32, - /// Picture type (IDR, I, P, B). - pub pic_type: super::PictureType, - /// Temporal layer ID. - pub temporal_id: u8, - /// Timestamp for this picture. - pub timestamp: u64, - /// Whether this picture should be output. - pub pic_output_flag: bool, - /// Whether this is a reference picture. - pub is_reference: bool, - /// For H.264: no_output_of_prior_pics_flag. - pub no_output_of_prior_pics_flag: bool, -} - -impl Default for PictureStartInfo { - fn default() -> Self { - Self { - frame_id: 0, - pic_order_cnt: 0, - frame_num: 0, - pic_type: super::PictureType::I, - temporal_id: 0, - timestamp: 0, - pic_output_flag: true, - is_reference: true, - no_output_of_prior_pics_flag: false, - } - } -} diff --git a/src/encoder/gop.rs b/src/encoder/gop.rs index 8a1bae65..c00b1ed8 100644 --- a/src/encoder/gop.rs +++ b/src/encoder/gop.rs @@ -38,14 +38,10 @@ pub struct GopPosition { pub struct GopStructure { /// Current frame index. frame_index: u64, - /// Current frame number. - frame_num: u32, /// Current POC. poc: i32, /// IDR period. idr_period: u32, - /// Max frame_num value (log2_max_frame_num_minus4 + 4). - max_frame_num: u32, /// Max POC LSB value (log2_max_pic_order_cnt_lsb_minus4 + 4). max_poc_lsb: u32, /// Flag to force next frame to be an IDR. @@ -58,19 +54,12 @@ impl GopStructure { Self { idr_period: gop_size, frame_index: 0, - frame_num: 0, poc: 0, - max_frame_num: 16, // log2_max_frame_num_minus4=0 -> 2^4 = 16 - max_poc_lsb: 16, // log2_max_pic_order_cnt_lsb_minus4=0 -> 2^4 = 16 + max_poc_lsb: 16, // log2_max_pic_order_cnt_lsb_minus4=0 -> 2^4 = 16 force_idr: false, } } - /// Set max frame_num (2^(log2_max_frame_num_minus4+4)). - pub fn set_max_frame_num(&mut self, log2_max_frame_num_minus4: u8) { - self.max_frame_num = 1 << (log2_max_frame_num_minus4 + 4); - } - /// Set max POC LSB (2^(log2_max_pic_order_cnt_lsb_minus4+4)). pub fn set_max_poc_lsb(&mut self, log2_max_pic_order_cnt_lsb_minus4: u8) { self.max_poc_lsb = 1 << (log2_max_pic_order_cnt_lsb_minus4 + 4); @@ -87,7 +76,6 @@ impl GopStructure { // If IDR, reset counters. if is_idr { - self.frame_num = 0; self.poc = 0; } @@ -104,7 +92,6 @@ impl GopStructure { }; // Update counters for next frame. - self.frame_num = (self.frame_num + 1) % self.max_frame_num; self.poc = (self.poc + 2) % (self.max_poc_lsb as i32 * 2); self.frame_index += 1; diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index 2cc32a8b..07249f94 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -5,7 +5,6 @@ use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; -use crate::encoder::dpb::{DecodedPictureBuffer, DecodedPictureBufferTrait, DpbConfig}; use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h264_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; @@ -75,17 +74,7 @@ impl H264 { let active_reference_count = init.active_reference_count; let common = init.common; - // H.264 reference marking bookkeeping. - let mut dpb = DecodedPictureBuffer::new(); - dpb.h264.sequence_start(DpbConfig { - dpb_size: common.dpb_slot_count as u32, - log2_max_frame_num_minus4: 4, - log2_max_pic_order_cnt_lsb_minus4: 4, - num_temporal_layers: 1, - }); - let codec = H264 { - dpb, frame_num_syntax: 0, idr_pic_id: 0, l0_references: Vec::new(), diff --git a/src/encoder/h264/mod.rs b/src/encoder/h264/mod.rs index 078a7cfa..8ed9cfe3 100644 --- a/src/encoder/h264/mod.rs +++ b/src/encoder/h264/mod.rs @@ -11,9 +11,6 @@ mod session_params; use crate::encoder::ColorDescription; use crate::encoder::codec::{EncoderCommon, FramePlan, PictureSetup, VideoCodec}; -use crate::encoder::dpb::{ - DecodedPictureBuffer, DecodedPictureBufferTrait, DpbConfig, PictureStartInfo, PictureType, -}; use crate::encoder::pipeline::EncodeFuture; use crate::error::Result; use ash::vk; @@ -32,8 +29,6 @@ pub(crate) struct ReferenceInfo { /// H.264-specific encoder state (everything the generic encoder doesn't own). pub(crate) struct H264 { - /// Decoded-picture-buffer bookkeeping (reference marking). - dpb: DecodedPictureBuffer, /// `frame_num` syntax element (mod `max_frame_num`). frame_num_syntax: u32, /// IDR picture id, toggled each IDR. @@ -65,14 +60,7 @@ impl VideoCodec for H264 { if plan.is_idr() { self.frame_num_syntax = 0; self.idr_pic_id = (self.idr_pic_id + 1) & 1; - // Reset the DPB for the new coded video sequence. - let dpb_config = DpbConfig { - dpb_size: common.dpb_slot_count as u32, - log2_max_frame_num_minus4: 4, - log2_max_pic_order_cnt_lsb_minus4: 4, - ..Default::default() - }; - self.dpb.h264.sequence_start(dpb_config); + // The IDR empties the DPB for the new coded video sequence. self.l0_references.clear(); // An IDR resets the decoder's whole DPB, so any deferred // unused-reference markings are moot. @@ -104,24 +92,6 @@ impl VideoCodec for H264 { if plan.is_reference() { self.frame_num_syntax = (frame_num + 1) % 256; - } - - if plan.is_reference() { - let pic_type = if plan.is_idr() { - PictureType::Idr - } else { - PictureType::P - }; - let pic_info = PictureStartInfo { - frame_id: plan.display_order, - pic_order_cnt, - frame_num, - pic_type, - is_reference: true, - ..Default::default() - }; - self.dpb.h264.picture_start(pic_info); - self.dpb.h264.picture_end(true); // The current frame becomes a reference for subsequent frames. self.l0_references.insert( diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index 8fb8ddfd..a147b8e5 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -5,7 +5,6 @@ use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; -use crate::encoder::dpb::{DecodedPictureBuffer, DecodedPictureBufferTrait, DpbConfig}; use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h265_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; @@ -74,16 +73,7 @@ impl H265 { let active_reference_count = init.active_reference_count; let common = init.common; - let mut dpb = DecodedPictureBuffer::new(); - dpb.h265.sequence_start(DpbConfig { - dpb_size: common.dpb_slot_count as u32, - log2_max_frame_num_minus4: 0, - log2_max_pic_order_cnt_lsb_minus4: 4, - num_temporal_layers: 1, - }); - let codec = H265 { - dpb, header_data: None, l0_references: Vec::new(), reference_window, diff --git a/src/encoder/h265/mod.rs b/src/encoder/h265/mod.rs index 21d5f4f3..0c32ca8b 100644 --- a/src/encoder/h265/mod.rs +++ b/src/encoder/h265/mod.rs @@ -10,9 +10,6 @@ mod session_params; use crate::encoder::ColorDescription; use crate::encoder::codec::{EncoderCommon, FramePlan, PictureSetup, VideoCodec}; -use crate::encoder::dpb::{ - DecodedPictureBuffer, DecodedPictureBufferTrait, DpbConfig, PictureStartInfo, PictureType, -}; use crate::encoder::pipeline::EncodeFuture; use crate::error::Result; use ash::vk; @@ -30,7 +27,6 @@ pub(crate) struct ReferenceInfo { /// H.265-specific encoder state. pub(crate) struct H265 { - dpb: DecodedPictureBuffer, /// Cached VPS/SPS/PPS header (invalidated when session params change). header_data: Option>, /// References held in the DPB, most-recent first. Each frame predicts @@ -63,13 +59,6 @@ impl VideoCodec for H265 { plan: &FramePlan, ) -> Result { if plan.is_idr() { - let dpb_config = DpbConfig { - dpb_size: common.dpb_slot_count as u32, - log2_max_frame_num_minus4: 0, - log2_max_pic_order_cnt_lsb_minus4: 4, - ..Default::default() - }; - self.dpb.h265.sequence_start(dpb_config); self.l0_references.clear(); // All DPB slots become inactive at the start of a coded video sequence. common.dpb_slot_active.fill(false); @@ -100,21 +89,6 @@ impl VideoCodec for H265 { return; } let pic_order_cnt = plan.pic_order_cnt(); - let pic_type = if plan.is_idr() { - PictureType::Idr - } else { - PictureType::P - }; - let pic_info = PictureStartInfo { - frame_id: plan.display_order, - pic_order_cnt, - frame_num: 0, - pic_type, - is_reference: true, - ..Default::default() - }; - self.dpb.h265.picture_start(pic_info); - self.dpb.h265.picture_end(true); self.l0_references.insert( 0, diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index d1ed2b7c..a0aab588 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -7,7 +7,6 @@ mod api; pub(crate) mod av1; pub(crate) mod codec; -pub(crate) mod dpb; #[cfg(feature = "dmabuf")] pub mod external; mod frame; From 9a6a9a789ec51803419e1d5024346f03decb9803 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 08:26:19 +0100 Subject: [PATCH 8/9] feat(vulkan)!: adopt a caller device for encode and decode in one context --- CHANGELOG.md | 7 + README.md | 7 +- src/decoder/mod.rs | 5 +- src/lib.rs | 7 +- src/vulkan.rs | 193 +------------ src/vulkan/{encode_device.rs => adopt.rs} | 328 ++++++++++++++-------- tests/adopted.rs | 63 ++++- 7 files changed, 292 insertions(+), 318 deletions(-) rename src/vulkan/{encode_device.rs => adopt.rs} (52%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b43fbd49..0c6219c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `EncodedPacket::is_key_frame` is a method, not a field. An IDR's `frame_type` is `FrameType::Idr`; it was `FrameType::I`. `FrameType::Unknown`, which no packet had, is removed. +- **Breaking:** `VideoContextBuilder::device_requirements` and + `build_from_existing` adopt a caller's device for the codecs asked for with + `require_encode` and `require_decode`, so one adopted context can encode + and decode. They replace `encode_device_requirements`, + `decode_device_requirements`, `build_from_existing_encode` and + `build_from_existing_decode`. At least one codec must be asked for: an + encode adoption with none no longer takes every codec the device has. - **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the encode API: `EncodeConfig::with_color` replaces `with_color_description`, `Encoder::reconfigure` replaces `set_color_description`, and diff --git a/README.md b/README.md index 2c78a62c..aaff2743 100644 --- a/README.md +++ b/README.md @@ -271,9 +271,10 @@ device instead of letting it create a second one. Frames then never leave the device: no external memory, no import, and no CPU wait between the application's work and pixelforge's. -[`VideoContextBuilder::encode_device_requirements`] lists the queue -families, extensions and features to create the device with, and -[`VideoContextBuilder::build_from_existing_encode`] adopts it. A `VkQueue` +[`VideoContextBuilder::device_requirements`] lists the queue +families, extensions and features to create the device with, for the +codecs asked for with `require_encode` and `require_decode`, and +[`VideoContextBuilder::build_from_existing`] adopts it. A `VkQueue` may not be submitted to from two threads at once, so give pixelforge queues the application does not use, with [`VideoContextBuilder::with_encode_queue`] and friends. diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index b6462202..965616eb 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -162,9 +162,8 @@ impl DecodeConfig { /// Passing that family here adds it to the sharing set, so frames can be /// sampled directly with no ownership transfer and no copy. Set it whenever /// frames are consumed by anything outside the decoder; for a context - /// adopted from an existing device - /// (see - /// [`build_from_existing_decode`](crate::vulkan::VideoContextBuilder::build_from_existing_decode)) + /// adopted from an existing device (see + /// [`build_from_existing`](crate::vulkan::VideoContextBuilder::build_from_existing)) /// this is the family the caller already renders on. pub fn with_consumer_queue_family(mut self, family: u32) -> Self { self.consumer_queue_family = Some(family); diff --git a/src/lib.rs b/src/lib.rs index 45bf780d..680d8393 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -268,9 +268,10 @@ //! the device: no external memory, no import, and no CPU wait between the //! application's work and pixelforge's. //! -//! [`VideoContextBuilder::encode_device_requirements`] lists the queue -//! families, extensions and features to create the device with, and -//! [`VideoContextBuilder::build_from_existing_encode`] adopts it. A `VkQueue` +//! [`VideoContextBuilder::device_requirements`] lists the queue +//! families, extensions and features to create the device with, for the +//! codecs asked for with `require_encode` and `require_decode`, and +//! [`VideoContextBuilder::build_from_existing`] adopts it. A `VkQueue` //! may not be submitted to from two threads at once, so give pixelforge queues //! the application does not use, with //! [`VideoContextBuilder::with_encode_queue`] and friends. diff --git a/src/vulkan.rs b/src/vulkan.rs index 2014b9c8..66e2709c 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -7,7 +7,7 @@ use std::ffi::{CStr, CString}; use std::os::raw::c_char; use tracing::{debug, info, warn}; -mod encode_device; +mod adopt; mod queues; use queues::QueueOverrides; pub use queues::{DeviceQueue, QueueRoles}; @@ -169,59 +169,13 @@ impl VideoContextBuilder { VideoContext::new(self) } - /// What a caller-created device must provide for pixelforge to decode on it. - /// - /// Use this when you already have your own Vulkan device (e.g. a renderer's) - /// and want to decode into images that device can use directly, without the - /// cross-device copy a separate context would require. - pub fn decode_device_requirements( - &self, - entry: &ash::Entry, - instance: &ash::Instance, - physical_device: vk::PhysicalDevice, - ) -> Result { - let families = find_decode_queue_families( - entry, - instance, - physical_device, - &self.required_decode_codecs, - )?; - let unified_image_layouts = supports_unified_image_layouts(instance, physical_device); - let mut extensions = decode_extension_names(&self.required_decode_codecs); - if unified_image_layouts { - extensions.push(ash::khr::unified_image_layouts::NAME); - } - let queues = QueueRoles { - encode: None, - decode: Some(families.decode), - transfer: families.transfer, - compute: families.compute, - }; - Ok(DeviceRequirements { - queue_families: queues.unique_families(), - queues, - extensions, - features: DeviceFeatures { - synchronization2: true, - timeline_semaphore: true, - ..DeviceFeatures::default() - }, - unified_image_layouts, - internally_synchronized_queues: supports_internally_synchronized_queues( - instance, - physical_device, - &DeviceExtensions::of(instance, physical_device), - ), - }) - } - /// Declare that the adopted device was created with /// `VK_KHR_unified_image_layouts` enabled, including both /// `unifiedImageLayouts` and `unifiedImageLayoutsVideo`. /// - /// Only meaningful for [`Self::build_from_existing_decode`]: a context that - /// creates its own device enables the extension itself when the hardware - /// has it. Vulkan offers no way to ask a device which features were enabled + /// Only meaningful for [`Self::build_from_existing`] with a decode codec: + /// a context that creates its own device enables the extension itself when + /// the hardware has it. Vulkan offers no way to ask a device which features were enabled /// on it, so this is a promise pixelforge cannot verify. What it *can* /// check, and does, is that the physical device supports both bits; a /// declaration on hardware that cannot support them is a clean error rather @@ -249,41 +203,12 @@ impl VideoContextBuilder { self.unified_image_layouts = Some(false); self } - - /// Adopt a caller-created device for decoding, rather than creating one. - /// - /// The device must have been created with the queue families and extensions - /// reported by [`Self::decode_device_requirements`] for the same - /// `physical_device`, and with the `synchronization2` and - /// `timelineSemaphore` features enabled (the decoder orders its pipelined - /// submissions with timeline semaphores). The - /// resulting context **borrows** `instance` and `device`: dropping it frees - /// neither, so the caller must keep both alive for at least as long as the - /// context and anything decoded with it. - pub fn build_from_existing_decode( - self, - entry: ash::Entry, - instance: ash::Instance, - physical_device: vk::PhysicalDevice, - device: ash::Device, - ) -> Result { - VideoContext::from_existing_decode( - self.required_decode_codecs, - self.unified_image_layouts.unwrap_or(false), - self.queues, - entry, - instance, - physical_device, - device, - ) - } } /// Queue families, extensions and features a caller's device must provide for /// pixelforge to work on it. /// -/// Returned by [`VideoContextBuilder::decode_device_requirements`] and -/// [`VideoContextBuilder::encode_device_requirements`]. Enable every extension +/// Returned by [`VideoContextBuilder::device_requirements`]. Enable every extension /// in [`Self::extensions`] and every feature set in [`Self::features`]: /// pixelforge assumes all of them are on, since Vulkan cannot be asked /// afterwards which were. @@ -1493,114 +1418,6 @@ impl VideoContext { }) } - /// Adopt a caller-created device for decoding. - /// - /// See [`VideoContextBuilder::build_from_existing_decode`], the public entry - /// point. The returned context borrows `instance` and `device` and destroys - /// neither on drop. - fn from_existing_decode( - required_decode_codecs: Vec, - declared_unified_image_layouts: bool, - queue_overrides: QueueOverrides, - entry: ash::Entry, - instance: ash::Instance, - physical_device: vk::PhysicalDevice, - device: ash::Device, - ) -> Result { - let families = find_decode_queue_families( - &entry, - &instance, - physical_device, - &required_decode_codecs, - )?; - - let device_properties = unsafe { instance.get_physical_device_properties(physical_device) }; - let memory_properties = - unsafe { instance.get_physical_device_memory_properties(physical_device) }; - - let decode = queues::resolve( - &instance, - physical_device, - "decode", - queue_overrides.decode, - families.decode, - vk::QueueFlags::VIDEO_DECODE_KHR, - )?; - let transfer = queues::resolve( - &instance, - physical_device, - "transfer", - queue_overrides.transfer, - families.transfer, - vk::QueueFlags::TRANSFER, - )?; - let compute = queues::resolve( - &instance, - physical_device, - "compute", - queue_overrides.compute, - families.compute, - vk::QueueFlags::COMPUTE, - )?; - - let video_decode_queue = unsafe { device.get_device_queue(decode.family, decode.index) }; - let transfer_queue = unsafe { device.get_device_queue(transfer.family, transfer.index) }; - let compute_queue = unsafe { device.get_device_queue(compute.family, compute.index) }; - - let supported_decode_codecs = query_decode_codecs(&entry, &instance, physical_device); - - // The declaration is a promise about what the caller enabled, which - // cannot be verified. That the hardware could support it can be, so a - // declaration that is impossible fails here rather than in the driver. - if declared_unified_image_layouts - && !supports_unified_image_layouts(&instance, physical_device) - { - return Err(PixelForgeError::NoSuitableDevice( - "unified image layouts were declared, but this device does not support \ - VK_KHR_unified_image_layouts with both unifiedImageLayouts and \ - unifiedImageLayoutsVideo" - .to_string(), - )); - } - - info!( - "Adopted caller device for decode: decode queue {:?}, transfer queue {:?}, compute queue {:?}", - decode, transfer, compute - ); - - Ok(VideoContext { - inner: std::sync::Arc::new(VideoContextInner { - entry, - sync2: ash::khr::synchronization2::Device::load(&instance, &device), - timeline: ash::khr::timeline_semaphore::Device::load(&instance, &device), - instance, - physical_device, - device, - video_encode_queue_family: None, - video_encode_timestamp_valid_bits: 0u32, - video_encode_queue: None, - video_decode_queue_family: Some(decode.family), - video_decode_queue: Some(video_decode_queue), - transfer_queue_family: transfer.family, - transfer_queue, - compute_queue_family: compute.family, - compute_queue, - memory_properties, - device_properties, - supported_encode_codecs: Vec::new(), - supported_decode_codecs, - has_push_descriptor: false, - has_video_encode_rgb_conversion: false, - has_unified_image_layouts: declared_unified_image_layouts, - external: ExternalSupport::default(), - owns_device: false, - retired_semaphores: Default::default(), - // The caller owns the instance; reporting is theirs to set up. - debug_messenger: None, - }), - }) - } - fn check_h264_encode_support( entry: &ash::Entry, instance: &ash::Instance, diff --git a/src/vulkan/encode_device.rs b/src/vulkan/adopt.rs similarity index 52% rename from src/vulkan/encode_device.rs rename to src/vulkan/adopt.rs index f6f4f9c7..72683e53 100644 --- a/src/vulkan/encode_device.rs +++ b/src/vulkan/adopt.rs @@ -1,18 +1,19 @@ -//! Encoding on a device the caller created. +//! Pixelforge on a device the caller created. //! -//! The encode counterpart of [`VideoContextBuilder::build_from_existing_decode`]. -//! An application that already renders on its own device, and wants to encode -//! what it renders, can hand that device to pixelforge. The encoder and the -//! colour converter then work on the application's images directly: no -//! external memory, no export and import, and a semaphore instead of a CPU -//! wait where one side hands a frame to the other. +//! An application that already renders on its own device can hand that device +//! to pixelforge, to encode what it renders or to decode into images it +//! samples. The encoder, the decoder and the colour converter then work on the +//! application's images directly: no external memory, no export and import, +//! and a semaphore instead of a CPU wait where one side hands a frame to the +//! other. use super::ExternalSupport; -use super::queues::{self, QueueOverrides}; +use super::queues; use super::{ - DeviceExtensions, DeviceFeatures, DeviceQueue, DeviceRequirements, QueueRoles, VideoContext, - VideoContextBuilder, VideoContextInner, scan_queue_families, - supports_internally_synchronized_queues, + DecodeQueueFamilies, DeviceExtensions, DeviceFeatures, DeviceQueue, DeviceRequirements, + QueueRoles, VideoContext, VideoContextBuilder, VideoContextInner, decode_extension_names, + find_decode_queue_families, query_decode_codecs, scan_queue_families, + supports_internally_synchronized_queues, supports_unified_image_layouts, }; use crate::encoder::Codec; use crate::error::{PixelForgeError, Result}; @@ -32,16 +33,12 @@ struct EncodeQueueFamilies { } /// What pixelforge will use on `physical_device` to encode `required`. -/// -/// When `required` is empty every codec the device encodes is included, so a -/// caller that has not decided yet gets a device that can do any of them. struct EncodePlan { families: EncodeQueueFamilies, codecs: Vec, push_descriptor: bool, ycbcr_2plane_444: bool, sampler_ycbcr_conversion: bool, - internally_synchronized_queues: bool, rgb_conversion: bool, external: ExternalSupport, } @@ -122,24 +119,15 @@ impl EncodePlan { { available.push(Codec::AV1); } - let codecs = if required.is_empty() { - available - } else { - for codec in required { - if !available.contains(codec) { - return Err(PixelForgeError::NoSuitableDevice(format!( - "Queue family {} does not support encoding {:?}", - family, codec - ))); - } + for codec in required { + if !available.contains(codec) { + return Err(PixelForgeError::NoSuitableDevice(format!( + "Queue family {} does not support encoding {:?}", + family, codec + ))); } - required.to_vec() - }; - if codecs.is_empty() { - return Err(PixelForgeError::NoSuitableDevice( - "Physical device encodes none of H.264, H.265 or AV1".to_string(), - )); } + let codecs = required.to_vec(); Ok(Self { families: EncodeQueueFamilies { @@ -153,11 +141,6 @@ impl EncodePlan { ycbcr_2plane_444: has(ash::ext::ycbcr_2plane_444_formats::NAME) && ycbcr_444.ycbcr2plane444_formats != 0, sampler_ycbcr_conversion: ycbcr.sampler_ycbcr_conversion != 0, - internally_synchronized_queues: supports_internally_synchronized_queues( - instance, - physical_device, - &extensions, - ), rgb_conversion: has(ash::valve::video_encode_rgb_conversion::NAME) && rgb.video_encode_rgb_conversion != 0, external: ExternalSupport::detect(has), @@ -204,138 +187,248 @@ impl EncodePlan { video_encode_rgb_conversion: self.rgb_conversion, } } +} + +/// What pixelforge will use on an adopted device: the encode role for the +/// codecs asked for with [`VideoContextBuilder::require_encode`], the decode +/// role for those asked for with [`VideoContextBuilder::require_decode`], or +/// both, and a transfer and a compute queue. +struct AdoptPlan { + encode: Option, + decode: Option, + transfer: u32, + compute: u32, + decode_codecs: Vec, + /// Whether the device supports unified image layouts, which only the + /// decoder uses. + unified_image_layouts: bool, + internally_synchronized_queues: bool, +} + +impl AdoptPlan { + fn new( + builder: &VideoContextBuilder, + entry: &ash::Entry, + instance: &ash::Instance, + physical_device: vk::PhysicalDevice, + ) -> Result { + let encode_codecs = &builder.required_encode_codecs; + let decode_codecs = &builder.required_decode_codecs; + if encode_codecs.is_empty() && decode_codecs.is_empty() { + return Err(PixelForgeError::InvalidInput( + "an adopted device needs at least one codec, asked for with require_encode or \ + require_decode" + .to_string(), + )); + } + let encode = (!encode_codecs.is_empty()) + .then(|| { + EncodePlan::new( + entry, + instance, + physical_device, + encode_codecs, + builder.queues.encode, + ) + }) + .transpose()?; + let decode = (!decode_codecs.is_empty()) + .then(|| find_decode_queue_families(entry, instance, physical_device, decode_codecs)) + .transpose()?; + // Both scans pick the transfer and compute families the same way. + let (transfer, compute) = match (&encode, &decode) { + (Some(e), _) => (e.families.transfer, e.families.compute), + (None, Some(d)) => (d.transfer, d.compute), + (None, None) => unreachable!("a codec was asked for"), + }; + Ok(Self { + unified_image_layouts: decode.is_some() + && supports_unified_image_layouts(instance, physical_device), + internally_synchronized_queues: supports_internally_synchronized_queues( + instance, + physical_device, + &DeviceExtensions::of(instance, physical_device), + ), + encode, + decode, + transfer, + compute, + decode_codecs: decode_codecs.clone(), + }) + } fn roles(&self) -> QueueRoles { QueueRoles { - encode: Some(self.families.encode.family), - decode: None, - transfer: self.families.transfer, - compute: self.families.compute, + encode: self.encode.as_ref().map(|e| e.families.encode.family), + decode: self.decode.as_ref().map(|d| d.decode), + transfer: self.transfer, + compute: self.compute, } } + + fn extensions(&self) -> Vec<&'static std::ffi::CStr> { + let mut names = self + .encode + .as_ref() + .map(EncodePlan::extensions) + .unwrap_or_default(); + if self.decode.is_some() { + names.extend(decode_extension_names(&self.decode_codecs)); + } + if self.unified_image_layouts { + names.push(ash::khr::unified_image_layouts::NAME); + } + let mut unique = Vec::with_capacity(names.len()); + for name in names { + if !unique.contains(&name) { + unique.push(name); + } + } + unique + } + + fn features(&self) -> DeviceFeatures { + self.encode + .as_ref() + .map(EncodePlan::features) + .unwrap_or(DeviceFeatures { + synchronization2: true, + timeline_semaphore: true, + ..DeviceFeatures::default() + }) + } } impl VideoContextBuilder { - /// What a caller-created device must provide for pixelforge to encode on - /// it, and to run the colour converter. + /// What a caller-created device must provide for pixelforge to work on + /// it, for the codecs asked for with [`Self::require_encode`] and + /// [`Self::require_decode`]. At least one codec must be asked for, and + /// the device must support every one. /// /// Use this when you already have your own Vulkan device (a renderer's, or - /// one a game created) and want to encode images that live on it, without - /// exporting them to a second device. The codecs asked for with - /// [`Self::require_encode`] must all be supported; with none asked for, - /// every codec the device encodes is included. + /// one a game created) and want to encode images that live on it, or + /// decode into images it uses, without a second device and the copies + /// between the two. /// - /// The colour converter needs `VK_KHR_push_descriptor`, which is included - /// whenever the device has it. On a device without it, encoding from YUV - /// input still works and only the converter is unavailable. - /// `VK_VALVE_video_encode_rgb_conversion` is included the same way, for - /// [`Conversion::Encoder`](crate::Conversion::Encoder). - pub fn encode_device_requirements( + /// For encoding, the colour converter needs `VK_KHR_push_descriptor`, + /// which is included whenever the device has it. On a device without it, + /// encoding from YUV input still works and only the converter is + /// unavailable. `VK_VALVE_video_encode_rgb_conversion` is included the + /// same way, for [`Conversion::Encoder`](crate::Conversion::Encoder). + pub fn device_requirements( &self, entry: &ash::Entry, instance: &ash::Instance, physical_device: vk::PhysicalDevice, ) -> Result { - let plan = EncodePlan::new( - entry, - instance, - physical_device, - &self.required_encode_codecs, - self.queues.encode, - )?; + let plan = AdoptPlan::new(self, entry, instance, physical_device)?; let queues = plan.roles(); Ok(DeviceRequirements { queue_families: queues.unique_families(), queues, extensions: plan.extensions(), features: plan.features(), - unified_image_layouts: false, + unified_image_layouts: plan.unified_image_layouts, internally_synchronized_queues: plan.internally_synchronized_queues, }) } - /// Adopt a caller-created device for encoding, rather than creating one. + /// Adopt a caller-created device, rather than creating one. /// /// The device must have been created with the queue families, extensions - /// and features reported by [`Self::encode_device_requirements`], called - /// with the same codecs for the same `physical_device`. The instance may - /// ask for any Vulkan version from 1.1 up. + /// and features reported by [`Self::device_requirements`], called with + /// the same codecs for the same `physical_device`. The instance may ask + /// for any Vulkan version from 1.1 up. /// /// The resulting context **borrows** `instance` and `device`: dropping it /// frees neither, so the caller must keep both alive for at least as long - /// as the context and every encoder and converter made from it. + /// as the context and every encoder, decoder and converter made from it. /// /// Pixelforge submits to one queue per role, queue 0 of each family unless /// told otherwise. A caller that keeps submitting to its own queues while /// pixelforge works on another thread should give it spare ones with - /// [`Self::with_encode_queue`], [`Self::with_transfer_queue`] and - /// [`Self::with_compute_queue`]. - pub fn build_from_existing_encode( + /// [`Self::with_encode_queue`], [`Self::with_decode_queue`], + /// [`Self::with_transfer_queue`] and [`Self::with_compute_queue`]. + pub fn build_from_existing( self, entry: ash::Entry, instance: ash::Instance, physical_device: vk::PhysicalDevice, device: ash::Device, ) -> Result { - VideoContext::from_existing_encode( - &self.required_encode_codecs, - self.queues, - entry, - instance, - physical_device, - device, - ) - } -} + let plan = AdoptPlan::new(&self, &entry, &instance, physical_device)?; + let overrides = self.queues; + let declared_unified_image_layouts = + plan.decode.is_some() && self.unified_image_layouts.unwrap_or(false); + // The declaration is a promise about what the caller enabled, which + // cannot be verified. That the hardware could support it can be, so a + // declaration that is impossible fails here rather than in the driver. + if declared_unified_image_layouts && !plan.unified_image_layouts { + return Err(PixelForgeError::NoSuitableDevice( + "unified image layouts were declared, but this device does not support \ + VK_KHR_unified_image_layouts with both unifiedImageLayouts and \ + unifiedImageLayoutsVideo" + .to_string(), + )); + } -impl VideoContext { - fn from_existing_encode( - required_encode_codecs: &[Codec], - queue_overrides: QueueOverrides, - entry: ash::Entry, - instance: ash::Instance, - physical_device: vk::PhysicalDevice, - device: ash::Device, - ) -> Result { - let plan = EncodePlan::new( - &entry, - &instance, - physical_device, - required_encode_codecs, - queue_overrides.encode, - )?; - let families = &plan.families; - let encode = families.encode; + let encode = plan.encode.as_ref().map(|e| e.families.encode); + let decode = plan + .decode + .as_ref() + .map(|d| { + queues::resolve( + &instance, + physical_device, + "decode", + overrides.decode, + d.decode, + vk::QueueFlags::VIDEO_DECODE_KHR, + ) + }) + .transpose()?; let transfer = queues::resolve( &instance, physical_device, "transfer", - queue_overrides.transfer, - families.transfer, + overrides.transfer, + plan.transfer, vk::QueueFlags::TRANSFER, )?; let compute = queues::resolve( &instance, physical_device, "compute", - queue_overrides.compute, - families.compute, + overrides.compute, + plan.compute, vk::QueueFlags::COMPUTE, )?; let device_properties = unsafe { instance.get_physical_device_properties(physical_device) }; let memory_properties = unsafe { instance.get_physical_device_memory_properties(physical_device) }; - - let video_encode_queue = unsafe { device.get_device_queue(encode.family, encode.index) }; - let transfer_queue = unsafe { device.get_device_queue(transfer.family, transfer.index) }; - let compute_queue = unsafe { device.get_device_queue(compute.family, compute.index) }; + let queue = |q: DeviceQueue| unsafe { device.get_device_queue(q.family, q.index) }; + let video_encode_queue = encode.map(queue); + let video_decode_queue = decode.map(queue); + let transfer_queue = queue(transfer); + let compute_queue = queue(compute); + let supported_decode_codecs = if plan.decode.is_some() { + query_decode_codecs(&entry, &instance, physical_device) + } else { + Vec::new() + }; info!( - "Adopted caller device for encode: {:?}, encode queue {:?}, transfer queue {:?}, compute queue {:?}", - plan.codecs, encode, transfer, compute + "Adopted caller device: encode {:?} on {:?}, decode {:?} on {:?}, transfer queue {:?}, compute queue {:?}", + plan.encode.as_ref().map(|e| &e.codecs), + encode, + plan.decode.as_ref().map(|_| &plan.decode_codecs), + decode, + transfer, + compute ); + let encode_plan = plan.encode.as_ref(); Ok(VideoContext { inner: std::sync::Arc::new(VideoContextInner { sync2: ash::khr::synchronization2::Device::load(&instance, &device), @@ -344,23 +437,24 @@ impl VideoContext { instance, physical_device, device, - video_encode_queue_family: Some(encode.family), - video_encode_timestamp_valid_bits: families.encode_timestamp_valid_bits, - video_encode_queue: Some(video_encode_queue), - video_decode_queue_family: None, - video_decode_queue: None, + video_encode_queue_family: encode.map(|q| q.family), + video_encode_timestamp_valid_bits: encode_plan + .map_or(0, |e| e.families.encode_timestamp_valid_bits), + video_encode_queue, + video_decode_queue_family: decode.map(|q| q.family), + video_decode_queue, transfer_queue_family: transfer.family, transfer_queue, compute_queue_family: compute.family, compute_queue, memory_properties, device_properties, - supported_encode_codecs: plan.codecs.clone(), - supported_decode_codecs: Vec::new(), - has_push_descriptor: plan.push_descriptor, - has_video_encode_rgb_conversion: plan.rgb_conversion, - has_unified_image_layouts: false, - external: plan.external, + supported_encode_codecs: encode_plan.map(|e| e.codecs.clone()).unwrap_or_default(), + supported_decode_codecs, + has_push_descriptor: encode_plan.is_some_and(|e| e.push_descriptor), + has_video_encode_rgb_conversion: encode_plan.is_some_and(|e| e.rgb_conversion), + has_unified_image_layouts: declared_unified_image_layouts, + external: encode_plan.map(|e| e.external).unwrap_or_default(), owns_device: false, retired_semaphores: Default::default(), // The caller owns the instance; reporting is theirs to set up. diff --git a/tests/adopted.rs b/tests/adopted.rs index f79cf274..8b721fb8 100644 --- a/tests/adopted.rs +++ b/tests/adopted.rs @@ -183,7 +183,7 @@ fn decode_on_a_vulkan_1_1_device() -> Result<(), Box> { let builder = VideoContextBuilder::new().require_decode(Codec::H264); let physical_device = same_gpu(&instance, &own); drop(own); - let reqs = builder.decode_device_requirements(&entry, &instance, physical_device)?; + let reqs = builder.device_requirements(&entry, &instance, physical_device)?; let app = create_app_device(entry, instance, physical_device, &reqs)?; let mut builder = builder @@ -193,7 +193,7 @@ fn decode_on_a_vulkan_1_1_device() -> Result<(), Box> { if reqs.unified_image_layouts { builder = builder.declare_unified_image_layouts(); } - let context = builder.build_from_existing_decode( + let context = builder.build_from_existing( app.entry.clone(), app.instance.clone(), app.physical_device, @@ -352,7 +352,7 @@ fn encode_on_a_vulkan_1_1_device() -> Result<(), Box> { let (entry, instance) = instance_1_1()?; let builder = VideoContextBuilder::new().require_encode(Codec::H264); let physical_device = same_gpu(&instance, &own); - let reqs = builder.encode_device_requirements(&entry, &instance, physical_device)?; + let reqs = builder.device_requirements(&entry, &instance, physical_device)?; assert!( reqs.extensions.contains(&ash::khr::push_descriptor::NAME), "the converter needs push descriptors, which every driver this runs on has" @@ -362,7 +362,7 @@ fn encode_on_a_vulkan_1_1_device() -> Result<(), Box> { .with_encode_queue(spare(&app, reqs.queues.encode.unwrap())) .with_transfer_queue(spare(&app, reqs.queues.transfer)) .with_compute_queue(spare(&app, reqs.queues.compute)) - .build_from_existing_encode( + .build_from_existing( app.entry.clone(), app.instance.clone(), app.physical_device, @@ -399,3 +399,58 @@ fn encode_on_a_vulkan_1_1_device() -> Result<(), Box> { } Ok(()) } + +/// One adopted device that both encodes and decodes: what it encodes decodes +/// on it to what a context pixelforge created decodes from the same stream. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn encode_and_decode_on_one_adopted_device() -> Result<(), Box> { + common::init_logging(); + let own = VideoContextBuilder::new() + .enable_validation(validation_requested()) + .require_encode(Codec::H264) + .require_decode(Codec::H264) + .build()?; + + let (entry, instance) = instance_1_1()?; + let physical_device = same_gpu(&instance, &own); + assert!( + VideoContextBuilder::new() + .device_requirements(&entry, &instance, physical_device) + .is_err(), + "requirements without a codec" + ); + let builder = VideoContextBuilder::new() + .require_encode(Codec::H264) + .require_decode(Codec::H264); + let reqs = builder.device_requirements(&entry, &instance, physical_device)?; + for name in [ + ash::khr::video_encode_h264::NAME, + ash::khr::video_decode_h264::NAME, + ] { + assert!(reqs.extensions.contains(&name), "{name:?} is missing"); + } + let app = create_app_device(entry, instance, physical_device, &reqs)?; + let mut builder = builder + .with_encode_queue(spare(&app, reqs.queues.encode.unwrap())) + .with_decode_queue(spare(&app, reqs.queues.decode.unwrap())) + .with_transfer_queue(spare(&app, reqs.queues.transfer)) + .with_compute_queue(spare(&app, reqs.queues.compute)); + if reqs.unified_image_layouts { + builder = builder.declare_unified_image_layouts(); + } + let context = builder.build_from_existing( + app.entry.clone(), + app.instance.clone(), + app.physical_device, + app.device.clone(), + )?; + let stream = encode_all(&context, Wait::Gpu)?; + let actual = decode_all(&context, &stream)?; + drop(context); + + let expected = decode_all(&own, &stream)?; + assert_eq!(actual.len(), ENCODE_FRAMES as usize, "frame count"); + assert!(actual == expected, "the adopted decode differs"); + Ok(()) +} From 9ba810a68e0e9d815874f00350e569f001ed8080 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Thu, 5 Nov 2026 08:32:20 +0100 Subject: [PATCH 9/9] refactor!: rename types for what they are --- CHANGELOG.md | 19 ++++-- README.md | 20 +++---- benches/encode.rs | 6 +- examples/encode.rs | 8 +-- examples/sample_frame.rs | 2 +- src/converter/device_tests.rs | 16 +++--- src/converter/mod.rs | 88 ++++++++++++++-------------- src/decoder/common.rs | 8 +-- src/decoder/frames.rs | 10 ++-- src/decoder/h264/session.rs | 34 +++++------ src/decoder/mod.rs | 6 +- src/decoder/transfer.rs | 2 +- src/encoder/api.rs | 12 ++-- src/encoder/av1/init.rs | 8 +-- src/encoder/av1/session_params.rs | 12 ++-- src/encoder/codec.rs | 12 ++-- src/encoder/external.rs | 60 +++++++++---------- src/encoder/frame.rs | 4 +- src/encoder/h264/init.rs | 10 ++-- src/encoder/h264/session_params.rs | 22 +++---- src/encoder/h265/init.rs | 12 ++-- src/encoder/h265/session_params.rs | 20 +++---- src/encoder/mod.rs | 76 ++++++++++++------------ src/encoder/pipeline.rs | 6 +- src/encoder/profile.rs | 34 +++++------ src/encoder/resources.rs | 32 +++++------ src/encoder/rgb.rs | 24 ++++---- src/encoder/source.rs | 37 ++++++------ src/encoder/support.rs | 22 +++---- src/image.rs | 92 +++++++++++++++--------------- src/lib.rs | 32 +++++------ src/video.rs | 20 +++---- tests/adopted.rs | 6 +- tests/dmabuf.rs | 16 +++--- tests/encode_format_support.rs | 22 +++---- tests/rfi.rs | 8 +-- tests/rgb_encode.rs | 38 ++++++------ tests/roundtrip.rs | 4 +- tests/submit.rs | 16 +++--- tests/upload.rs | 10 ++-- tests/verify_all.rs | 56 +++++++++--------- 41 files changed, 476 insertions(+), 466 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c6219c4..9a10d420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,12 +23,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 waits for them. - Add `Encoder::input_slot` and `InputSlot`: the encoder's own input image, to write a frame in place and submit with `Frame::input`. -- Add `InputImage::source`. +- Add `YuvUploader::source`. - Add `Encoder::repeat`: encode the last frame again from the encoder's own input image, for a key frame while no new frame arrives. The caller's source does not have to stay alive for it. -- Add DMA-BUF input (feature `dmabuf`): `Encoder::input_formats`, - `Encoder::import`, `Encoder::allocate_input`, `ExternalImage` and +- Add DMA-BUF input (feature `dmabuf`): `Encoder::dmabuf_format`, + `Encoder::import`, `Encoder::allocate_input`, `DmaBufImage` and `Frame::external`. The encoder reads a buffer directly where the device can, and copies it otherwise. - Add sync_file fences (feature `dmabuf`): `Frame::wait_sync_file`, @@ -64,15 +64,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `EncodeConfig::with_b_frames` and the public `b_frame_count` field are removed. No codec encodes B-frames yet, and every encoder refused a count above 0 at runtime. -- **Breaking:** `InputImage` has two uploads, `upload` to its own image and +- **Breaking:** `YuvUploader` has two uploads, `upload` to its own image and `upload_to` an encoder's `InputSlot`, which take a `YuvLayout`: `I420`, `Nv12`, `I420P10` or `I444`. They replace `upload_yuv420`, `upload_nv12`, `upload_yuv420_10bit`, `upload_yuv420_to` and `upload_yuv444_to`, which covered only some of the combinations. A layout the image cannot hold is an error; `upload_nv12` panicked for a 10-bit or 4:4:4 image. The uploads wait until the frames that read the image are done with it. -- **Breaking:** `InputImage::new` no longer takes a `Codec`, which it did not - use. `InputImage::for_encoder` makes one for an encoder's frames. +- **Breaking:** `YuvUploader::new` no longer takes a `Codec`, which it did not + use. `YuvUploader::for_encoder` makes one for an encoder's frames. - **Breaking:** `EncodeConfig::frame_rate` (a `FrameRate`) replaces `frame_rate_numerator` and `frame_rate_denominator`. `with_frame_rate` is unchanged. @@ -96,6 +96,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `EncodedPacket::is_key_frame` is a method, not a field. An IDR's `frame_type` is `FrameType::Idr`; it was `FrameType::I`. `FrameType::Unknown`, which no packet had, is removed. +- **Breaking:** types are renamed for what they are: `InputImage` is + `YuvUploader`, `InputFormat` is `RgbFormat`, `PixelFormat` is + `ChromaFormat` (with `EncodeConfig::chroma_format`, `with_chroma_format` + and `DecodedFrame::chroma_format`), and the DMA-BUF types are + `DmaBufImage`, `DmaBufFormat` and `DmaBufModifier`, from + `Encoder::dmabuf_format`. They were `ExternalImage`, `InputFormats`, + `InputModifier` and `Encoder::input_formats`. - **Breaking:** `VideoContextBuilder::device_requirements` and `build_from_existing` adopt a caller's device for the codecs asked for with `require_encode` and `require_decode`, so one adopted context can encode diff --git a/README.md b/README.md index aaff2743..564deb2c 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ H.265 and AV1 encode, and H.264 decode. - **GPU-native API**: Encode directly from Vulkan images (`vk::Image`). - **Flexible configuration**: Rate control (CBR, VBR, CQP), quality levels, GOP settings. - **Multiple input formats**: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). -- **Utility helpers**: [`InputImage`] for easy YUV data upload to GPU. +- **Utility helpers**: [`YuvUploader`] for easy YUV data upload to GPU. - **Optional DMA-BUF input**: import a producer's buffers, or allocate buffers for it, read with no copy where the driver allows, with sync_file fences (Linux only). @@ -96,7 +96,7 @@ fn main() -> Result<(), Box> { ### Encoding Video ```rust -use pixelforge::{Codec, EncodeConfig, Encoder, InputImage, RateControl, VideoContextBuilder}; +use pixelforge::{Codec, EncodeConfig, Encoder, YuvUploader, RateControl, VideoContextBuilder}; fn main() -> Result<(), Box> { let context = VideoContextBuilder::new() @@ -113,8 +113,8 @@ fn main() -> Result<(), Box> { .with_gop_size(60); let mut encoder = Encoder::new(context, config)?; - // Create an InputImage helper for uploading YUV data to the GPU. - let mut input_image = InputImage::for_encoder(&encoder)?; + // Create a YuvUploader helper for uploading YUV data to the GPU. + let mut input_image = YuvUploader::for_encoder(&encoder)?; // For each frame: upload YUV data and encode. // let yuv_data: &[u8] = ...; // YUV420 frame data @@ -195,7 +195,7 @@ stream's label cannot disagree, and a colour change through ```rust use pixelforge::{ - Codec, ColorRange, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, Source, + Codec, ColorRange, ColorSpec, EncodeConfig, Encoder, RgbFormat, RgbSource, Source, StreamColor, VideoContextBuilder, }; @@ -206,7 +206,7 @@ let context = VideoContextBuilder::new() // SDR desktop content, encoded as HDR10 for an HDR streaming session. let config = EncodeConfig::h265(1920, 1080) .with_bit_depth(pixelforge::BitDepth::Ten) - .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) + .with_source(Source::Rgb(RgbSource::new(RgbFormat::BGRx, ColorSpec::Srgb))) .with_color(StreamColor::Hdr10 { range: ColorRange::Full, }); @@ -260,9 +260,9 @@ flight. The pixels are in one of: - a [`SourceImage`], an image of the caller's. Pixelforge copies or converts it into the input image after the call returns, and the `SourceImage` keeps track of that: write the image again only after - [`SourceImage::released`]. [`InputImage`], for frames uploaded from the + [`SourceImage::released`]. [`YuvUploader`], for frames uploaded from the CPU, waits for that itself. -- an `ExternalImage`, a DMA-BUF (see below). +- a `DmaBufImage`, a DMA-BUF (see below). ### Encoding on your own device @@ -296,12 +296,12 @@ let reusable = source.released(); ### DMA-BUF input With the `dmabuf` feature, an encoder reads DMA-BUFs from another process. -`Encoder::input_formats` gives the DRM format it wants and, per DRM +`Encoder::dmabuf_format` gives the DRM format it wants and, per DRM modifier, whether it reads such a buffer with no copy. Offer those modifiers to the producer first. `Encoder::import` takes a producer's buffer; `Encoder::allocate_input` makes one for a producer that writes into buffers the consumer supplies, such as `ext-image-copy-capture`. -Either is checked once and gives an `ExternalImage`, which +Either is checked once and gives a `DmaBufImage`, which `Frame::external` submits. A buffer the encoder cannot read directly is copied on the GPU instead. diff --git a/benches/encode.rs b/benches/encode.rs index 7995d6b5..771dc8f4 100644 --- a/benches/encode.rs +++ b/benches/encode.rs @@ -9,8 +9,8 @@ //! cargo bench --bench encode use pixelforge::{ - Codec, EncodeConfig, Encoder, EncoderTuningMode, Frame, InputImage, RateControl, - VideoContextBuilder, YuvLayout, + Codec, EncodeConfig, Encoder, EncoderTuningMode, Frame, RateControl, VideoContextBuilder, + YuvLayout, YuvUploader, }; use std::fs::File; use std::io::Read; @@ -136,7 +136,7 @@ fn run_encode_test( let mut encoder = Encoder::new(context.clone(), config)?; // Create input image for uploading frames. - let mut input_image = InputImage::for_encoder(&encoder)?; + let mut input_image = YuvUploader::for_encoder(&encoder)?; let mut total_bytes = 0; let mut gpu_durations: Vec = Vec::new(); diff --git a/examples/encode.rs b/examples/encode.rs index 3212bd7b..8651454f 100644 --- a/examples/encode.rs +++ b/examples/encode.rs @@ -10,8 +10,8 @@ //! The codec defaults to `h264`. use pixelforge::{ - Codec, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, - YuvLayout, + ChromaFormat, Codec, EncodeConfig, Encoder, Frame, RateControl, VideoContextBuilder, YuvLayout, + YuvUploader, }; use std::fs::File; use std::io::{Read, Write}; @@ -46,7 +46,7 @@ fn main() -> Result<(), Box> { ), "h265" => ( Codec::H265, - EncodeConfig::h265(WIDTH, HEIGHT).with_pixel_format(PixelFormat::Yuv420), + EncodeConfig::h265(WIDTH, HEIGHT).with_chroma_format(ChromaFormat::Yuv420), "output.h265", ), "av1" => (Codec::AV1, EncodeConfig::av1(WIDTH, HEIGHT), "output.av1"), @@ -103,7 +103,7 @@ fn main() -> Result<(), Box> { let mut encoder = Encoder::new(context, config)?; // Create input image for uploading frames. - let mut input_image = InputImage::for_encoder(&encoder)?; + let mut input_image = YuvUploader::for_encoder(&encoder)?; let mut output = File::create(output_name)?; let mut total_bytes = 0; diff --git a/examples/sample_frame.rs b/examples/sample_frame.rs index 29e2cb0e..6ca9506c 100644 --- a/examples/sample_frame.rs +++ b/examples/sample_frame.rs @@ -175,7 +175,7 @@ impl Sampler { frame: &DecodedFrame, ) -> Result> { let device = context.device().clone(); - let format = frame.pixel_format; + let format = frame.chroma_format; let _ = format; let picture_format = vk::Format::G8_B8R8_2PLANE_420_UNORM; diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index d97cfd48..0579784b 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -31,9 +31,7 @@ mod source; use source::{SrcImage, create_src_image, host_buffer, one_shot}; -use super::{ - ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, InputFormat, OutputFormat, -}; +use super::{ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, OutputFormat, RgbFormat}; use crate::encoder::input::SourceOwnership; use crate::{ BitDepth, Codec, ColorDescription, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, @@ -255,7 +253,7 @@ fn convert( let config = ColorConverterConfig::new( WIDTH, HEIGHT, - InputFormat::BGRA, + RgbFormat::BGRA, output_format, source, target, @@ -427,7 +425,7 @@ fn unsupported_conversions_are_refused() -> Result<(), Box Result<(), Box Result<(), Box> { ColorConverterConfig::new( ENCODE_WIDTH, ENCODE_HEIGHT, - InputFormat::BGRA, + RgbFormat::BGRA, OutputFormat::NV12, ColorSpec::Srgb, ColorSpec::Srgb, diff --git a/src/converter/mod.rs b/src/converter/mod.rs index aaac0ec1..76d7d274 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -107,7 +107,7 @@ impl ColorRange { /// Supported input pixel formats for color conversion. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[allow(clippy::upper_case_acronyms)] -pub enum InputFormat { +pub enum RgbFormat { /// BGRx (32-bit, blue first, x = unused). BGRx, /// RGBx (32-bit, red first, x = unused). @@ -130,26 +130,26 @@ pub enum InputFormat { RGBA16F, } -impl InputFormat { +impl RgbFormat { /// Bytes per pixel for this format. pub fn bytes_per_pixel(&self) -> usize { match self { - InputFormat::BGRx - | InputFormat::RGBx - | InputFormat::BGRA - | InputFormat::RGBA - | InputFormat::ABGR2101010 => 4, - InputFormat::RGBA16F => 8, + RgbFormat::BGRx + | RgbFormat::RGBx + | RgbFormat::BGRA + | RgbFormat::RGBA + | RgbFormat::ABGR2101010 => 4, + RgbFormat::RGBA16F => 8, } } /// Vulkan format for creating image views of this input format. pub fn vk_format(&self) -> vk::Format { match self { - InputFormat::BGRx | InputFormat::BGRA => vk::Format::B8G8R8A8_UNORM, - InputFormat::RGBx | InputFormat::RGBA => vk::Format::R8G8B8A8_UNORM, - InputFormat::ABGR2101010 => vk::Format::A2B10G10R10_UNORM_PACK32, - InputFormat::RGBA16F => vk::Format::R16G16B16A16_SFLOAT, + RgbFormat::BGRx | RgbFormat::BGRA => vk::Format::B8G8R8A8_UNORM, + RgbFormat::RGBx | RgbFormat::RGBA => vk::Format::R8G8B8A8_UNORM, + RgbFormat::ABGR2101010 => vk::Format::A2B10G10R10_UNORM_PACK32, + RgbFormat::RGBA16F => vk::Format::R16G16B16A16_SFLOAT, } } } @@ -219,7 +219,7 @@ pub(crate) struct ColorConverterConfig { /// Input frame height. pub height: u32, /// Input pixel format. - pub input_format: InputFormat, + pub input_format: RgbFormat, /// Output YUV format. pub output_format: OutputFormat, /// What the input pixels already are. @@ -244,7 +244,7 @@ impl ColorConverterConfig { pub fn new( width: u32, height: u32, - input_format: InputFormat, + input_format: RgbFormat, output_format: OutputFormat, source_color: ColorSpec, target_color: ColorSpec, @@ -1017,42 +1017,42 @@ mod tests { use super::*; // ======================== - // InputFormat tests. + // RgbFormat tests. // ======================== #[test] fn test_input_format_bytes_per_pixel() { - assert_eq!(InputFormat::BGRx.bytes_per_pixel(), 4); - assert_eq!(InputFormat::RGBx.bytes_per_pixel(), 4); - assert_eq!(InputFormat::BGRA.bytes_per_pixel(), 4); - assert_eq!(InputFormat::RGBA.bytes_per_pixel(), 4); - assert_eq!(InputFormat::ABGR2101010.bytes_per_pixel(), 4); - assert_eq!(InputFormat::RGBA16F.bytes_per_pixel(), 8); + assert_eq!(RgbFormat::BGRx.bytes_per_pixel(), 4); + assert_eq!(RgbFormat::RGBx.bytes_per_pixel(), 4); + assert_eq!(RgbFormat::BGRA.bytes_per_pixel(), 4); + assert_eq!(RgbFormat::RGBA.bytes_per_pixel(), 4); + assert_eq!(RgbFormat::ABGR2101010.bytes_per_pixel(), 4); + assert_eq!(RgbFormat::RGBA16F.bytes_per_pixel(), 8); } #[test] fn test_input_format_enum_values() { // Verify enum values match shader expectations. - assert_eq!(InputFormat::BGRx as u32, 0); - assert_eq!(InputFormat::RGBx as u32, 1); - assert_eq!(InputFormat::BGRA as u32, 2); - assert_eq!(InputFormat::RGBA as u32, 3); - assert_eq!(InputFormat::ABGR2101010 as u32, 4); - assert_eq!(InputFormat::RGBA16F as u32, 5); + assert_eq!(RgbFormat::BGRx as u32, 0); + assert_eq!(RgbFormat::RGBx as u32, 1); + assert_eq!(RgbFormat::BGRA as u32, 2); + assert_eq!(RgbFormat::RGBA as u32, 3); + assert_eq!(RgbFormat::ABGR2101010 as u32, 4); + assert_eq!(RgbFormat::RGBA16F as u32, 5); } #[test] fn test_input_format_vk_format() { - assert_eq!(InputFormat::BGRx.vk_format(), vk::Format::B8G8R8A8_UNORM); - assert_eq!(InputFormat::BGRA.vk_format(), vk::Format::B8G8R8A8_UNORM); - assert_eq!(InputFormat::RGBx.vk_format(), vk::Format::R8G8B8A8_UNORM); - assert_eq!(InputFormat::RGBA.vk_format(), vk::Format::R8G8B8A8_UNORM); + assert_eq!(RgbFormat::BGRx.vk_format(), vk::Format::B8G8R8A8_UNORM); + assert_eq!(RgbFormat::BGRA.vk_format(), vk::Format::B8G8R8A8_UNORM); + assert_eq!(RgbFormat::RGBx.vk_format(), vk::Format::R8G8B8A8_UNORM); + assert_eq!(RgbFormat::RGBA.vk_format(), vk::Format::R8G8B8A8_UNORM); assert_eq!( - InputFormat::ABGR2101010.vk_format(), + RgbFormat::ABGR2101010.vk_format(), vk::Format::A2B10G10R10_UNORM_PACK32 ); assert_eq!( - InputFormat::RGBA16F.vk_format(), + RgbFormat::RGBA16F.vk_format(), vk::Format::R16G16B16A16_SFLOAT ); } @@ -1118,7 +1118,7 @@ mod tests { let base = ColorConverterConfig::new( 64, 64, - InputFormat::BGRA, + RgbFormat::BGRA, OutputFormat::P010, ColorSpec::Bt709Linear, ColorSpec::Bt2020Pq, @@ -1143,7 +1143,7 @@ mod tests { let config = ColorConverterConfig::new( 64, 64, - InputFormat::BGRA, + RgbFormat::BGRA, OutputFormat::P010, ColorSpec::Bt2020Pq, ColorSpec::Bt2020Pq, @@ -1168,7 +1168,7 @@ mod tests { ColorConverterConfig::new( 64, 64, - InputFormat::BGRA, + RgbFormat::BGRA, OutputFormat::NV12, source, target, @@ -1345,7 +1345,7 @@ mod tests { let config = ColorConverterConfig::new( 1920, 1080, - InputFormat::BGRx, + RgbFormat::BGRx, OutputFormat::NV12, ColorSpec::Srgb, ColorSpec::Srgb, @@ -1355,7 +1355,7 @@ mod tests { let cloned = config.clone(); assert_eq!(cloned.width, 1920); assert_eq!(cloned.height, 1080); - assert_eq!(cloned.input_format, InputFormat::BGRx); + assert_eq!(cloned.input_format, RgbFormat::BGRx); assert_eq!(cloned.output_format, OutputFormat::NV12); assert_eq!(cloned.target, ColorSpec::Srgb); assert!(cloned.range.is_full()); @@ -1366,7 +1366,7 @@ mod tests { let config = ColorConverterConfig::new( 640, 480, - InputFormat::RGBA, + RgbFormat::RGBA, OutputFormat::I420, ColorSpec::Srgb, ColorSpec::Srgb, @@ -1408,7 +1408,7 @@ mod tests { let config = ColorConverterConfig::new( 64, 64, - InputFormat::BGRx, + RgbFormat::BGRx, OutputFormat::NV12, ColorSpec::Srgb, ColorSpec::Srgb, @@ -1431,10 +1431,10 @@ mod tests { }; let input_formats = [ - InputFormat::BGRx, - InputFormat::RGBx, - InputFormat::BGRA, - InputFormat::RGBA, + RgbFormat::BGRx, + RgbFormat::RGBx, + RgbFormat::BGRA, + RgbFormat::RGBA, ]; let output_formats = [OutputFormat::NV12, OutputFormat::I420, OutputFormat::YUV444]; diff --git a/src/decoder/common.rs b/src/decoder/common.rs index 4db02a5e..d618365c 100644 --- a/src/decoder/common.rs +++ b/src/decoder/common.rs @@ -24,7 +24,7 @@ use ash::vk; use crate::decoder::DecodedFrame; use crate::decoder::frames::SlotPins; use crate::decoder::pipeline::DecodePipeline; -use crate::encoder::{BitDepth, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat}; use crate::error::{PixelForgeError, Result}; use crate::video::{ VideoImageParams, allocate_session_memory, create_command_pool, create_dpb_images, @@ -80,7 +80,7 @@ pub(crate) struct SessionPlan { pub coded_height: u32, pub picture_format: vk::Format, pub bit_depth: BitDepth, - pub pixel_format: PixelFormat, + pub chroma_format: ChromaFormat, /// DPB slots: references, the picture being decoded, and `spare_slots`. pub slot_count: usize, /// How many of `slot_count` are spare capacity for pictures pinned past the @@ -166,7 +166,7 @@ pub(crate) struct DecodeSession { pub coded_height: u32, pub picture_format: vk::Format, pub bit_depth: BitDepth, - pub pixel_format: PixelFormat, + pub chroma_format: ChromaFormat, /// DPB images. One image per slot, or a single layered image. /// One entry per slot, or a single shared entry when the DPB is layered. @@ -487,7 +487,7 @@ impl DecoderCommon { coded_height: plan.coded_height, picture_format: plan.picture_format, bit_depth: plan.bit_depth, - pixel_format: plan.pixel_format, + chroma_format: plan.chroma_format, dpb, slot_pins: Arc::new(SlotPins::default()), dpb_slot_active: vec![false; plan.slot_count], diff --git a/src/decoder/frames.rs b/src/decoder/frames.rs index ca63ab4e..a2b32dd5 100644 --- a/src/decoder/frames.rs +++ b/src/decoder/frames.rs @@ -15,7 +15,7 @@ use ash::vk::{self, TaggedStructure as _}; use crate::decoder::DecodedFrame; use crate::decoder::common::{DecoderCommon, DpbImage}; -use crate::encoder::{BitDepth, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat}; use crate::error::{PixelForgeError, Result}; use crate::video::find_memory_type; use crate::vulkan::VideoContext; @@ -32,7 +32,7 @@ pub(crate) struct DecodedPicture { pub image_view: vk::ImageView, pub layout: vk::ImageLayout, pub array_layer: u32, - pub pixel_format: PixelFormat, + pub chroma_format: ChromaFormat, pub bit_depth: BitDepth, pub generation: u64, pub width: u32, @@ -329,7 +329,7 @@ struct ReorderEntry { height: u32, coded_width: u32, coded_height: u32, - pixel_format: PixelFormat, + chroma_format: ChromaFormat, bit_depth: BitDepth, } @@ -405,7 +405,7 @@ impl ReorderBuffer { height: picture.height, coded_width: picture.coded_width, coded_height: picture.coded_height, - pixel_format: picture.pixel_format, + chroma_format: picture.chroma_format, bit_depth: picture.bit_depth, }); @@ -505,7 +505,7 @@ impl ReorderBuffer { image_view, layout, array_layer, - pixel_format: entry.pixel_format, + chroma_format: entry.chroma_format, bit_depth: entry.bit_depth, width: entry.width, height: entry.height, diff --git a/src/decoder/h264/session.rs b/src/decoder/h264/session.rs index 858490e8..89a26919 100644 --- a/src/decoder/h264/session.rs +++ b/src/decoder/h264/session.rs @@ -16,7 +16,7 @@ use crate::decoder::h264::parser::{ use crate::decoder::h264::record::{DecodeRecord, LinkParams}; use crate::decoder::pipeline::DECODE_PIPELINE_DEPTH; use crate::decoder::{DecodeConfig, DecodeStatus, Framing}; -use crate::encoder::{BitDepth, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat}; use crate::error::{PixelForgeError, Result}; use crate::video::{align_up, get_video_format, query_supported_video_formats}; use crate::vulkan::VideoContext; @@ -162,12 +162,12 @@ impl H264Decoder { sps: &Sps, ) -> Result<( vk::VideoDecodeH264ProfileInfoKHR<'static>, - PixelFormat, + ChromaFormat, BitDepth, )> { - let pixel_format = match sps.chroma_format_idc { - 1 => PixelFormat::Yuv420, - 3 => PixelFormat::Yuv444, + let chroma_format = match sps.chroma_format_idc { + 1 => ChromaFormat::Yuv420, + 3 => ChromaFormat::Yuv444, other => { return Err(PixelForgeError::InvalidInput(format!( "H.264 decode: unsupported chroma_format_idc {}", @@ -211,7 +211,7 @@ impl H264Decoder { .std_profile_idc(std_profile) .picture_layout(vk::VideoDecodeH264PictureLayoutFlagsKHR::PROGRESSIVE); - Ok((profile, pixel_format, bit_depth)) + Ok((profile, chroma_format, bit_depth)) } /// Create (or recreate) the Vulkan session for the given parameter sets. @@ -237,13 +237,13 @@ impl H264Decoder { )); } - let (mut profile, pixel_format, bit_depth) = Self::profile_for(sps)?; - let chroma_subsampling = match pixel_format { - PixelFormat::Yuv420 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_420, - PixelFormat::Yuv444 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_444, + let (mut profile, chroma_format, bit_depth) = Self::profile_for(sps)?; + let chroma_subsampling = match chroma_format { + ChromaFormat::Yuv420 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_420, + ChromaFormat::Yuv444 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_444, other => { return Err(PixelForgeError::InvalidInput(format!( - "H.264 decode: unsupported pixel format {:?}", + "H.264 decode: unsupported chroma format {:?}", other ))); } @@ -395,7 +395,7 @@ impl H264Decoder { } else { (sampled_usage, true, sampled_formats) }; - let preferred = get_video_format(pixel_format, bit_depth); + let preferred = get_video_format(chroma_format, bit_depth); let chosen = supported .iter() .find(|f| f.format == preferred) @@ -443,7 +443,7 @@ impl H264Decoder { coded_height, picture_format, bit_depth, - pixel_format, + chroma_format, slot_count, spare_slots, max_active_references, @@ -826,7 +826,7 @@ impl H264Decoder { image_view, layout, array_layer, - pixel_format: active.pixel_format, + chroma_format: active.chroma_format, bit_depth: active.bit_depth, generation, width, @@ -855,9 +855,9 @@ impl H264Decoder { /// The copying is generic; only the profile (needed to size the buffer) is /// H.264's business. fn stage_slices(&mut self, picture: &Picture, sps: &Sps) -> Result<(u64, Vec)> { - let (mut profile, pixel_format, bit_depth) = Self::profile_for(sps)?; - let chroma_subsampling = match pixel_format { - PixelFormat::Yuv420 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_420, + let (mut profile, chroma_format, bit_depth) = Self::profile_for(sps)?; + let chroma_subsampling = match chroma_format { + ChromaFormat::Yuv420 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_420, _ => vk::VideoChromaSubsamplingFlagsKHR::TYPE_444, }; let depth_flag = match bit_depth { diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 965616eb..6c29707a 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -56,7 +56,7 @@ use std::task::{Context, Poll}; use ash::vk; use futures_core::Stream; -use crate::encoder::{BitDepth, Codec, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat, Codec}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -224,8 +224,8 @@ pub struct DecodedFrame { /// non-layered DPB or a distinct output image; the DPB slot's layer /// when the DPB is layered and the driver decoded straight into it. pub array_layer: u32, - /// The pixel format of this frame. - pub pixel_format: PixelFormat, + /// The chroma format of this frame. + pub chroma_format: ChromaFormat, /// Bit depth of the samples: eight for NV12, ten for P010. pub bit_depth: BitDepth, /// Which set of decoder images [`image`](Self::image) belongs to. diff --git a/src/decoder/transfer.rs b/src/decoder/transfer.rs index 2bb16df9..42d8d9ef 100644 --- a/src/decoder/transfer.rs +++ b/src/decoder/transfer.rs @@ -94,7 +94,7 @@ impl DecoderCommon { base_array_layer: 0, layer_count: 1, }; - let (cdiv_hor, cdiv_vert) = frame.pixel_format.chroma_div(); + let (cdiv_hor, cdiv_vert) = frame.chroma_format.chroma_div(); let regions = [ vk::ImageCopy::default() .src_subresource(vk::ImageSubresourceLayers { diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 019e2384..bdca213f 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -77,7 +77,7 @@ pub struct Encoder { /// The slot whose input image holds the last frame, for [`Self::repeat`]. last_frame_slot: Option, /// Counts the times the codec encoder was made again with a different - /// conversion. An [`ExternalImage`](super::external::ExternalImage) made + /// conversion. A [`DmaBufImage`](super::external::DmaBufImage) made /// for one cannot be read by another. generation: u64, #[cfg(feature = "dmabuf")] @@ -146,11 +146,11 @@ impl Encoder { /// an error: submit the RGB frames from a [`SourceImage`] instead. /// /// ```no_run - /// use pixelforge::{EncodeConfig, Encoder, Frame, InputImage, VideoContext, YuvLayout}; + /// use pixelforge::{EncodeConfig, Encoder, Frame, YuvUploader, VideoContext, YuvLayout}; /// /// # fn example(context: VideoContext) -> Result<(), Box> { /// let mut encoder = Encoder::new(context, EncodeConfig::h264(1920, 1080))?; - /// let mut upload = InputImage::for_encoder(&encoder)?; + /// let mut upload = YuvUploader::for_encoder(&encoder)?; /// # let yuv_data = vec![0u8; 1920 * 1080 * 3 / 2]; /// let slot = encoder.input_slot()?; /// upload.upload_to(&slot, YuvLayout::I420, &yuv_data)?; @@ -183,7 +183,7 @@ impl Encoder { /// itself, depending on the [`Conversion`] and the source. /// /// The call returns before the source is read. A [`SourceImage`] or an - /// `ExternalImage` keeps track of that itself, see + /// `DmaBufImage` keeps track of that itself, see /// [`SourceImage::released`]. pub fn submit(&mut self, frame: Frame<'_>) -> Result { if let FrameSource::Input(slot) = &frame.source @@ -448,7 +448,7 @@ impl Encoder { /// encoder is made again, because its video session fixes the input /// format, matrix and range. Packet timestamps carry on across that. /// - /// An `ExternalImage` made before a change of source or conversion has to + /// A `DmaBufImage` made before a change of source or conversion has to /// be made again. On error the encoder is left as it was. pub fn reconfigure(&mut self, source: Source, color: StreamColor) -> Result<()> { if (source, color) == (self.config.source, self.config.color) { @@ -516,7 +516,7 @@ impl Encoder { fn create_codec_encoder( context: &VideoContext, config: &EncodeConfig, - rgb_input: Option, + rgb_input: Option, ) -> Result> { let context = context.clone(); let config = config.clone(); diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index d8b60aa6..aa04ecec 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -1,6 +1,6 @@ use super::{Av1, MIN_BITSTREAM_BUFFER_SIZE, SUPERBLOCK_SIZE}; -use crate::converter::InputFormat; +use crate::converter::RgbFormat; use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, @@ -18,11 +18,11 @@ impl Av1 { pub(crate) fn create( context: VideoContext, config: EncodeConfig, - rgb_input: Option, + rgb_input: Option, ) -> Result> { info!( - "Creating AV1 encoder: {}x{}, pixel_format={:?}", - config.dimensions.width, config.dimensions.height, config.pixel_format + "Creating AV1 encoder: {}x{}, chroma_format={:?}", + config.dimensions.width, config.dimensions.height, config.chroma_format ); let profile = EncodeProfile::new(&config, rgb_input.is_some())?; diff --git a/src/encoder/av1/session_params.rs b/src/encoder/av1/session_params.rs index 682a3299..89d2d0bb 100644 --- a/src/encoder/av1/session_params.rs +++ b/src/encoder/av1/session_params.rs @@ -4,7 +4,7 @@ use super::Av1; use crate::encoder::codec::EncoderCommon; use crate::encoder::resources::get_encoded_session_params; -use crate::encoder::{BitDepth, ColorDescription, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat, ColorDescription}; use crate::error::{PixelForgeError, Result}; use ash::vk; use ash::vk::TaggedStructure; @@ -54,9 +54,9 @@ impl Av1 { BitDepth::Ten => 10, }; - let (subsampling_x, subsampling_y) = match config.pixel_format { - PixelFormat::Yuv420 => (1u8, 1u8), - PixelFormat::Yuv444 => (0u8, 0u8), + let (subsampling_x, subsampling_y) = match config.chroma_format { + ChromaFormat::Yuv420 => (1u8, 1u8), + ChromaFormat::Yuv444 => (0u8, 0u8), _ => (1u8, 1u8), }; @@ -78,8 +78,8 @@ impl Av1 { }, }; - let profile = match config.pixel_format { - PixelFormat::Yuv420 => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_MAIN, + let profile = match config.chroma_format { + ChromaFormat::Yuv420 => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_MAIN, _ => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_HIGH, }; diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 81832385..8f243ccc 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -15,7 +15,7 @@ use ash::vk; use ash::vk::TaggedStructure; -use crate::converter::InputFormat; +use crate::converter::RgbFormat; use crate::encoder::gop::{GopFrameType, GopPosition, GopStructure}; use crate::encoder::input::{FrameInput, FrameSync, SourceOwnership}; use crate::encoder::pipeline::{ @@ -104,7 +104,7 @@ pub(crate) struct EncoderCommon { #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] pub profile: EncodeProfile, /// The RGB format the encoder converts from, when it takes RGB input. - pub rgb_input: Option, + pub rgb_input: Option, } impl EncoderCommon { @@ -168,7 +168,7 @@ impl EncoderCommon { dst_image: slot_image, width: self.config.dimensions.width, height: self.config.dimensions.height, - pixel_format: self.config.pixel_format, + chroma_format: self.config.chroma_format, input_image_layout: slot_layout, rgb: self.rgb_input.is_some(), upload_queue: self.context.transfer_queue(), @@ -751,7 +751,7 @@ pub(crate) struct CommonInitRequest<'a> { /// later have to name it too. pub profile: EncodeProfile, /// The RGB format the encoder converts from, when it takes RGB input. - pub rgb_input: Option, + pub rgb_input: Option, /// Device capabilities for this profile, resolved by [`query_video_caps`]. pub caps: &'a DeviceVideoCaps, /// Codec block size for coded-extent alignment (macroblock / CTB / superblock). @@ -915,7 +915,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result }; // Pick input (SRC) and reference (DPB) formats. - let preferred_src_format = get_video_format(config.pixel_format, config.bit_depth); + let preferred_src_format = get_video_format(config.chroma_format, config.bit_depth); let supported_src_formats = query_supported_video_formats( context, profile_info, @@ -1067,7 +1067,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result aligned_height, picture_format, rgb_input: req.rgb_input.is_some(), - pixel_format: config.pixel_format, + chroma_format: config.chroma_format, bit_depth: config.bit_depth, bitstream_buffer_size: req.bitstream_buffer_size, profile_info, diff --git a/src/encoder/external.rs b/src/encoder/external.rs index c7fca2cf..bc41e982 100644 --- a/src/encoder/external.rs +++ b/src/encoder/external.rs @@ -2,11 +2,11 @@ //! //! An encoder can read a DMA-BUF straight into its encode, with no copy in //! pixelforge, when the device takes the buffer's DRM format modifier for that. -//! [`Encoder::input_formats`] says which modifiers those are, so a caller can +//! [`Encoder::dmabuf_format`] says which modifiers those are, so a caller can //! negotiate one with its producer. [`Encoder::allocate_input`] makes such a //! buffer for a producer that writes into buffers the consumer gives it, and //! [`Encoder::import`] takes one a producer made. Either gives an -//! [`ExternalImage`] to hand to [`Frame::external`](crate::Frame::external). +//! [`DmaBufImage`] to hand to [`Frame::external`](crate::Frame::external). //! //! A DMA-BUF is shared with another process, so each frame acquires it from //! the foreign queue family before reading it and releases it back after. @@ -21,7 +21,7 @@ use ash::vk::TaggedStructure; use super::api::Encoder; use super::input::{FrameInput, FrameSync, SourceOwnership, Uses}; use super::source::{Conversion, Source}; -use super::{EncodeFuture, PixelFormat}; +use super::{ChromaFormat, EncodeFuture}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::{create_video_image_view, get_video_format}; @@ -118,7 +118,7 @@ pub struct DmaBuf { /// What an encoder can do with buffers of one DRM format modifier. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct InputModifier { +pub struct DmaBufModifier { /// The DRM format modifier. pub modifier: u64, /// How many memory planes a buffer with it has. @@ -134,19 +134,19 @@ pub struct InputModifier { pub exportable: bool, } -/// The buffers an encoder takes, from [`Encoder::input_formats`]. +/// The buffers an encoder takes, from [`Encoder::dmabuf_format`]. #[derive(Debug, Clone)] -pub struct InputFormats { +pub struct DmaBufFormat { /// The DRM fourcc of the pixel format, see [`drm_format`]. pub format: u32, /// The same format, as Vulkan names it. pub vk_format: vk::Format, /// Every modifier the device knows for the format and the encoder can do /// something with. Offer the `zero_copy` ones to a producer first. - pub modifiers: Vec, + pub modifiers: Vec, } -/// How an [`ExternalImage`] is read. +/// How a [`DmaBufImage`] is read. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Access { /// By the encode, made with its video profile. @@ -177,7 +177,7 @@ struct ModifierSupport { /// /// Only the encoder that made it can read it. Dropping it waits until its last /// frame is done with it. -pub struct ExternalImage { +pub struct DmaBufImage { context: VideoContext, image: vk::Image, memory: vk::DeviceMemory, @@ -193,9 +193,9 @@ pub struct ExternalImage { uses: Uses, } -impl std::fmt::Debug for ExternalImage { +impl std::fmt::Debug for DmaBufImage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ExternalImage") + f.debug_struct("DmaBufImage") .field("extent", &self.extent) .field("format", &self.format) .field("modifier", &self.modifier) @@ -204,7 +204,7 @@ impl std::fmt::Debug for ExternalImage { } } -impl ExternalImage { +impl DmaBufImage { /// Whether the encoder reads this image directly, with no copy. pub fn is_zero_copy(&self) -> bool { self.access != Access::Copy @@ -290,7 +290,7 @@ impl ExternalImage { } } -impl Drop for ExternalImage { +impl Drop for DmaBufImage { fn drop(&mut self) { let _ = self.uses.wait(&self.context); let device = self.context.device(); @@ -364,13 +364,13 @@ impl Encoder { }; Ok(match self.conversion() { Conversion::None => { - if config.pixel_format == PixelFormat::Yuv422 { + if config.chroma_format == ChromaFormat::Yuv422 { return Err(PixelForgeError::InvalidInput( "DMA-BUF input: no 4:2:2 encode".to_string(), )); } InputPlan { - vk_format: get_video_format(config.pixel_format, config.bit_depth), + vk_format: get_video_format(config.chroma_format, config.bit_depth), zero_copy: Access::Encode, fallback: Some(Access::Copy), } @@ -516,7 +516,7 @@ impl Encoder { /// encoder's YUV for a YUV source, the RGB format for an RGB one. A /// producer that can choose should be offered the `zero_copy` modifiers /// first. - pub fn input_formats(&self) -> Result { + pub fn dmabuf_format(&self) -> Result { self.require_dmabuf()?; let plan = self.input_plan()?; let format = drm_format_of(plan.vk_format).ok_or_else(|| { @@ -539,7 +539,7 @@ impl Encoder { let zero_copy = direct_import || direct_export; let importable = direct_import || plan.fallback.is_some_and(imports); let exportable = direct_export || plan.fallback.is_some_and(exports); - (importable || exportable).then_some(InputModifier { + (importable || exportable).then_some(DmaBufModifier { modifier, plane_count, zero_copy, @@ -548,7 +548,7 @@ impl Encoder { }) }) .collect(); - Ok(InputFormats { + Ok(DmaBufFormat { format, vk_format: plan.vk_format, modifiers, @@ -556,17 +556,17 @@ impl Encoder { } /// Make an image with one of `modifiers`, which a producer can write into - /// once [exported](ExternalImage::export). + /// once [exported](DmaBufImage::export). /// /// For producers that write into buffers the consumer supplies, such as /// `ext-image-copy-capture`. The image has the encoder's display size, /// which is what such a protocol asks for. `modifiers` are the ones the /// producer takes. The driver picks among those the encoder reads directly - /// (`zero_copy` in [`Self::input_formats`]); only if there are none, among + /// (`zero_copy` in [`Self::dmabuf_format`]); only if there are none, among /// those it can copy from, and then each frame is copied - /// ([`ExternalImage::is_zero_copy`] says which). It fails if none of + /// ([`DmaBufImage::is_zero_copy`] says which). It fails if none of /// `modifiers` can be allocated either way. - pub fn allocate_input(&self, modifiers: &[u64]) -> Result { + pub fn allocate_input(&self, modifiers: &[u64]) -> Result { self.require_dmabuf()?; let plan = self.input_plan()?; let format = drm_format_of(plan.vk_format).ok_or_else(|| { @@ -611,12 +611,12 @@ impl Encoder { /// Import a producer's DMA-BUF for this encoder to read. /// /// The buffer is checked here, once: a modifier the encoder reads directly - /// gives an image it does ([`ExternalImage::is_zero_copy`]); one it can only + /// gives an image it does ([`DmaBufImage::is_zero_copy`]); one it can only /// copy from gives an image each frame copies; any other fails. The format - /// must be the one [`Self::input_formats`] names, or an RGB format that + /// must be the one [`Self::dmabuf_format`] names, or an RGB format that /// differs from it only in whether the fourth channel is alpha. The size /// must be at least the encoder's. - pub fn import(&self, buffer: &DmaBuf) -> Result { + pub fn import(&self, buffer: &DmaBuf) -> Result { self.require_dmabuf()?; let plan = self.input_plan()?; if vk_format_of(buffer.format) != Some(plan.vk_format) { @@ -675,7 +675,7 @@ impl Encoder { access: Access, modifier: ModifierChoice<'_>, import: Option<&DmaBuf>, - ) -> Result { + ) -> Result { let context = self.context(); let device = context.device(); let extent = match import { @@ -787,7 +787,7 @@ impl Encoder { extent.width, extent.height, ); - Ok(ExternalImage { + Ok(DmaBufImage { context: context.clone(), image, memory, @@ -892,15 +892,15 @@ impl Encoder { Ok(memory) } - /// Encode from an [`ExternalImage`] this encoder made. + /// Encode from a [`DmaBufImage`] this encoder made. pub(crate) fn submit_external( &mut self, - external: &ExternalImage, + external: &DmaBufImage, mut sync: FrameSync, ) -> Result { if external.owner != (self.id(), self.generation()) { return Err(PixelForgeError::InvalidInput( - "this ExternalImage was made by another encoder, or before a reconfigure \ + "this DmaBufImage was made by another encoder, or before a reconfigure \ changed the source or the conversion; import or allocate it again" .to_string(), )); diff --git a/src/encoder/frame.rs b/src/encoder/frame.rs index 8e79da7a..76812bf2 100644 --- a/src/encoder/frame.rs +++ b/src/encoder/frame.rs @@ -13,7 +13,7 @@ pub(crate) enum FrameSource<'a> { Image(&'a SourceImage), /// A DMA-BUF pixelforge imported or allocated. #[cfg(feature = "dmabuf")] - External(&'a crate::encoder::external::ExternalImage), + External(&'a crate::encoder::external::DmaBufImage), } /// The encoder's input image for the next frame, from @@ -91,7 +91,7 @@ impl<'a> Frame<'a> { /// Pixelforge acquires it from the foreign queue family before it reads, /// and releases it back after. #[cfg(feature = "dmabuf")] - pub fn external(image: &'a crate::encoder::external::ExternalImage) -> Self { + pub fn external(image: &'a crate::encoder::external::DmaBufImage) -> Self { Self::new(FrameSource::External(image)) } diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index 07249f94..c1ece12f 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -1,6 +1,6 @@ use super::{H264, MB_SIZE, ReferenceInfo}; -use crate::converter::InputFormat; +use crate::converter::RgbFormat; use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, @@ -20,15 +20,15 @@ impl H264 { pub(crate) fn create( context: VideoContext, config: EncodeConfig, - rgb_input: Option, + rgb_input: Option, ) -> Result> { info!( - "Creating H.264 encoder: {}x{}, pixel_format={:?}", - config.dimensions.width, config.dimensions.height, config.pixel_format + "Creating H.264 encoder: {}x{}, chroma_format={:?}", + config.dimensions.width, config.dimensions.height, config.chroma_format ); // Select the profile based on chroma subsampling. - let profile_idc = h264_profile_idc(config.pixel_format); + let profile_idc = h264_profile_idc(config.chroma_format); let profile = EncodeProfile::new(&config, rgb_input.is_some())?; let profile_info = profile.info(); diff --git a/src/encoder/h264/session_params.rs b/src/encoder/h264/session_params.rs index af56ee6f..f9f644ae 100644 --- a/src/encoder/h264/session_params.rs +++ b/src/encoder/h264/session_params.rs @@ -4,7 +4,7 @@ use super::H264; use crate::encoder::codec::EncoderCommon; use crate::encoder::resources::get_encoded_session_params; -use crate::encoder::{BitDepth, ColorDescription, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat, ColorDescription}; use crate::error::{PixelForgeError, Result}; use ash::vk; use ash::vk::TaggedStructure; @@ -28,13 +28,13 @@ impl H264 { let pic_height_in_map_units = common.aligned_height / 16; // Crop units depend on chroma subsampling (frame_mbs_only_flag=1). - let (crop_unit_x, crop_unit_y) = match config.pixel_format { - PixelFormat::Yuv420 => (2u32, 2u32), - PixelFormat::Yuv444 => (1u32, 1u32), + let (crop_unit_x, crop_unit_y) = match config.chroma_format { + ChromaFormat::Yuv420 => (2u32, 2u32), + ChromaFormat::Yuv444 => (1u32, 1u32), _ => { return Err(PixelForgeError::InvalidInput(format!( "Unsupported pixel format for H.264: {:?}", - config.pixel_format + config.chroma_format ))); } }; @@ -53,11 +53,11 @@ impl H264 { } sps_flags.set_vui_parameters_present_flag(1); - let chroma_format_idc = match config.pixel_format { - PixelFormat::Yuv420 => { + let chroma_format_idc = match config.chroma_format { + ChromaFormat::Yuv420 => { ash::vk::native::StdVideoH264ChromaFormatIdc_STD_VIDEO_H264_CHROMA_FORMAT_IDC_420 } - PixelFormat::Yuv444 => { + ChromaFormat::Yuv444 => { ash::vk::native::StdVideoH264ChromaFormatIdc_STD_VIDEO_H264_CHROMA_FORMAT_IDC_444 } _ => unreachable!("Pixel format validated above"), @@ -82,7 +82,7 @@ impl H264 { // rows (chroma_sample_loc_type 0), and the stream says so. vui_flags .set_chroma_loc_info_present_flag( - matches!(config.pixel_format, PixelFormat::Yuv420) as u32 + matches!(config.chroma_format, ChromaFormat::Yuv420) as u32 ); // Avoid HRD with zeroed bitrates (device loss on some drivers). vui_flags.set_nal_hrd_parameters_present_flag(0); @@ -145,8 +145,8 @@ impl H264 { pps_flags.set_deblocking_filter_control_present_flag(1); // vk_video_samples sets chroma QP offsets to 6 for 4:4:4 (driver compat). - let (chroma_qp_index_offset, second_chroma_qp_index_offset) = match config.pixel_format { - PixelFormat::Yuv444 => (6i8, 6i8), + let (chroma_qp_index_offset, second_chroma_qp_index_offset) = match config.chroma_format { + ChromaFormat::Yuv444 => (6i8, 6i8), _ => (0i8, 0i8), }; diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index a147b8e5..484418d6 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -1,6 +1,6 @@ use super::{CTB_SIZE, H265}; -use crate::converter::InputFormat; +use crate::converter::RgbFormat; use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, @@ -20,18 +20,18 @@ impl H265 { pub(crate) fn create( context: VideoContext, config: EncodeConfig, - rgb_input: Option, + rgb_input: Option, ) -> Result> { info!( - "Creating H.265 encoder: {}x{}, pixel_format={:?}", - config.dimensions.width, config.dimensions.height, config.pixel_format + "Creating H.265 encoder: {}x{}, chroma_format={:?}", + config.dimensions.width, config.dimensions.height, config.chroma_format ); let profile_idc = - h265_profile_idc(config.pixel_format, config.bit_depth).ok_or_else(|| { + h265_profile_idc(config.chroma_format, config.bit_depth).ok_or_else(|| { PixelForgeError::InvalidInput(format!( "Unsupported pixel format / bit depth for H.265: {:?} / {:?}", - config.pixel_format, config.bit_depth + config.chroma_format, config.bit_depth )) })?; diff --git a/src/encoder/h265/session_params.rs b/src/encoder/h265/session_params.rs index 9f708a2c..cd562f67 100644 --- a/src/encoder/h265/session_params.rs +++ b/src/encoder/h265/session_params.rs @@ -4,7 +4,7 @@ use super::H265; use crate::encoder::codec::EncoderCommon; use crate::encoder::resources::get_encoded_session_params; -use crate::encoder::{BitDepth, ColorDescription, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat, ColorDescription}; use crate::error::{PixelForgeError, Result}; use ash::vk; use ash::vk::TaggedStructure; @@ -27,9 +27,9 @@ impl H265 { let pic_width_in_luma_samples = common.aligned_width; let pic_height_in_luma_samples = common.aligned_height; - let (sub_width_c, sub_height_c) = match config.pixel_format { - PixelFormat::Yuv420 => (2u32, 2u32), - PixelFormat::Yuv444 => (1u32, 1u32), + let (sub_width_c, sub_height_c) = match config.chroma_format { + ChromaFormat::Yuv420 => (2u32, 2u32), + ChromaFormat::Yuv444 => (1u32, 1u32), _ => (2u32, 2u32), }; let conf_win_right_offset = (common.aligned_width - config.dimensions.width) / sub_width_c; @@ -130,7 +130,7 @@ impl H265 { 1, // chroma_loc_info_present_flag: 4:2:0 chroma sits on the left // luma sample, midway between the two rows (type 0). - matches!(config.pixel_format, PixelFormat::Yuv420) as u32, + matches!(config.chroma_format, ChromaFormat::Yuv420) as u32, 0, 0, 0, @@ -181,17 +181,17 @@ impl H265 { BitDepth::Ten => 2, }; - let chroma_format_idc = match config.pixel_format { - PixelFormat::Yuv420 => { + let chroma_format_idc = match config.chroma_format { + ChromaFormat::Yuv420 => { ash::vk::native::StdVideoH265ChromaFormatIdc_STD_VIDEO_H265_CHROMA_FORMAT_IDC_420 } - PixelFormat::Yuv444 => { + ChromaFormat::Yuv444 => { ash::vk::native::StdVideoH265ChromaFormatIdc_STD_VIDEO_H265_CHROMA_FORMAT_IDC_444 } _ => { return Err(PixelForgeError::InvalidInput(format!( - "Unsupported pixel format for H.265: {:?}", - config.pixel_format + "Unsupported chroma format for H.265: {:?}", + config.chroma_format ))); } }; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index a0aab588..b0ad6347 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -57,9 +57,9 @@ pub enum Codec { AV1, } -/// Pixel format / chroma subsampling. +/// Chroma subsampling: how many chroma samples there are per luma sample. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum PixelFormat { +pub enum ChromaFormat { /// YUV 4:2:0 (half horizontal and vertical chroma resolution). #[default] Yuv420, @@ -69,33 +69,33 @@ pub enum PixelFormat { Yuv444, } -impl From for vk::VideoChromaSubsamplingFlagsKHR { - fn from(format: PixelFormat) -> Self { +impl From for vk::VideoChromaSubsamplingFlagsKHR { + fn from(format: ChromaFormat) -> Self { match format { - PixelFormat::Yuv420 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_420, - PixelFormat::Yuv422 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_422, - PixelFormat::Yuv444 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_444, + ChromaFormat::Yuv420 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_420, + ChromaFormat::Yuv422 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_422, + ChromaFormat::Yuv444 => vk::VideoChromaSubsamplingFlagsKHR::TYPE_444, } } } -impl PixelFormat { +impl ChromaFormat { /// Calculate frame size in bytes for given dimensions. pub fn frame_size(&self, width: u32, height: u32) -> usize { let luma_size = (width * height) as usize; match self { - PixelFormat::Yuv420 => luma_size * 3 / 2, // Y + U/4 + V/4 - PixelFormat::Yuv422 => luma_size * 2, // Y + U/2 + V/2 - PixelFormat::Yuv444 => luma_size * 3, // Y + U + V + ChromaFormat::Yuv420 => luma_size * 3 / 2, // Y + U/4 + V/4 + ChromaFormat::Yuv422 => luma_size * 2, // Y + U/2 + V/2 + ChromaFormat::Yuv444 => luma_size * 3, // Y + U + V } } - /// Returns chroma divisor (horizontal, vertical) for the pixel format. + /// Returns chroma divisor (horizontal, vertical) for the chroma format. pub fn chroma_div(&self) -> (u32, u32) { match self { - PixelFormat::Yuv444 => (1, 1), - PixelFormat::Yuv422 => (2, 1), - PixelFormat::Yuv420 => (2, 2), + ChromaFormat::Yuv444 => (1, 1), + ChromaFormat::Yuv422 => (2, 1), + ChromaFormat::Yuv420 => (2, 2), } } } @@ -403,8 +403,8 @@ pub struct EncodeConfig { pub codec: Codec, /// Video dimensions. pub dimensions: Dimensions, - /// Pixel format (chroma subsampling). - pub pixel_format: PixelFormat, + /// Chroma format (chroma subsampling). + pub chroma_format: ChromaFormat, /// Bit depth per component. pub bit_depth: BitDepth, /// How the encoder spends bits. Checked by [`Encoder::new`]. @@ -458,7 +458,7 @@ impl EncodeConfig { Self { codec, dimensions: Dimensions { width, height }, - pixel_format: PixelFormat::Yuv420, + chroma_format: ChromaFormat::Yuv420, bit_depth: BitDepth::Eight, rate_control: RateControl::default_for(codec), frame_rate: FrameRate { @@ -499,9 +499,9 @@ impl EncodeConfig { self } - /// Set the pixel format (chroma subsampling). - pub fn with_pixel_format(mut self, format: PixelFormat) -> Self { - self.pixel_format = format; + /// Set the chroma format (chroma subsampling). + pub fn with_chroma_format(mut self, format: ChromaFormat) -> Self { + self.chroma_format = format; self } @@ -638,14 +638,14 @@ impl EncodedPacket { mod tests { use super::*; - // PixelFormat tests. - mod pixel_format_tests { + // ChromaFormat tests. + mod chroma_format_tests { use super::*; #[test] fn test_yuv420_frame_size() { // YUV420: Y = width * height, U = Y/4, V = Y/4 -> total = Y * 1.5 - let size = PixelFormat::Yuv420.frame_size(1920, 1080); + let size = ChromaFormat::Yuv420.frame_size(1920, 1080); let expected = (1920 * 1080) * 3 / 2; // 3110400 assert_eq!(size, expected); } @@ -653,7 +653,7 @@ mod tests { #[test] fn test_yuv422_frame_size() { // YUV422: Y = width * height, U = Y/2, V = Y/2 -> total = Y * 2 - let size = PixelFormat::Yuv422.frame_size(1920, 1080); + let size = ChromaFormat::Yuv422.frame_size(1920, 1080); let expected = (1920 * 1080) * 2; // 4147200 assert_eq!(size, expected); } @@ -661,7 +661,7 @@ mod tests { #[test] fn test_yuv444_frame_size() { // YUV444: Y = width * height, U = Y, V = Y -> total = Y * 3 - let size = PixelFormat::Yuv444.frame_size(1920, 1080); + let size = ChromaFormat::Yuv444.frame_size(1920, 1080); let expected = (1920 * 1080) * 3; // 6220800 assert_eq!(size, expected); } @@ -669,7 +669,7 @@ mod tests { #[test] fn test_small_resolution() { // Test with small resolution. - let size = PixelFormat::Yuv420.frame_size(320, 240); + let size = ChromaFormat::Yuv420.frame_size(320, 240); let expected = (320 * 240) * 3 / 2; // 115200 assert_eq!(size, expected); } @@ -677,25 +677,25 @@ mod tests { #[test] fn test_4k_resolution() { // Test with 4K resolution. - let size = PixelFormat::Yuv420.frame_size(3840, 2160); + let size = ChromaFormat::Yuv420.frame_size(3840, 2160); let expected = (3840 * 2160) * 3 / 2; // 12441600 assert_eq!(size, expected); } #[test] fn test_default() { - assert_eq!(PixelFormat::default(), PixelFormat::Yuv420); + assert_eq!(ChromaFormat::default(), ChromaFormat::Yuv420); } #[test] fn test_vk_chroma_subsampling_conversion() { - let vk_420: vk::VideoChromaSubsamplingFlagsKHR = PixelFormat::Yuv420.into(); + let vk_420: vk::VideoChromaSubsamplingFlagsKHR = ChromaFormat::Yuv420.into(); assert_eq!(vk_420, vk::VideoChromaSubsamplingFlagsKHR::TYPE_420); - let vk_422: vk::VideoChromaSubsamplingFlagsKHR = PixelFormat::Yuv422.into(); + let vk_422: vk::VideoChromaSubsamplingFlagsKHR = ChromaFormat::Yuv422.into(); assert_eq!(vk_422, vk::VideoChromaSubsamplingFlagsKHR::TYPE_422); - let vk_444: vk::VideoChromaSubsamplingFlagsKHR = PixelFormat::Yuv444.into(); + let vk_444: vk::VideoChromaSubsamplingFlagsKHR = ChromaFormat::Yuv444.into(); assert_eq!(vk_444, vk::VideoChromaSubsamplingFlagsKHR::TYPE_444); } } @@ -771,7 +771,7 @@ mod tests { assert_eq!(config.codec, Codec::H264); assert_eq!(config.dimensions.width, 1920); assert_eq!(config.dimensions.height, 1080); - assert_eq!(config.pixel_format, PixelFormat::Yuv420); + assert_eq!(config.chroma_format, ChromaFormat::Yuv420); assert_eq!(config.bit_depth, BitDepth::Eight); assert_eq!(config.rate_control, RateControl::Cqp { qp: 26 }); assert_eq!(config.gop_size, 30); @@ -799,10 +799,10 @@ mod tests { } #[test] - fn test_with_pixel_format() { - let config = EncodeConfig::h264(1920, 1080).with_pixel_format(PixelFormat::Yuv444); + fn test_with_chroma_format() { + let config = EncodeConfig::h264(1920, 1080).with_chroma_format(ChromaFormat::Yuv444); - assert_eq!(config.pixel_format, PixelFormat::Yuv444); + assert_eq!(config.chroma_format, ChromaFormat::Yuv444); } #[test] @@ -841,7 +841,7 @@ mod tests { assert_eq!(config.codec, Codec::AV1); assert_eq!(config.dimensions.width, 2560); assert_eq!(config.dimensions.height, 1440); - assert_eq!(config.pixel_format, PixelFormat::Yuv420); + assert_eq!(config.chroma_format, ChromaFormat::Yuv420); assert_eq!(config.bit_depth, BitDepth::Eight); // AV1 uses a 0-255 quantizer index. assert_eq!(config.rate_control, RateControl::Cqp { qp: 128 }); @@ -874,7 +874,7 @@ mod tests { }) .with_gop_size(120) .with_frame_rate(60, 1) - .with_pixel_format(PixelFormat::Yuv420) + .with_chroma_format(ChromaFormat::Yuv420) .with_bit_depth(BitDepth::Eight); assert_eq!( diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 126e7d95..c44b41e0 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -45,7 +45,7 @@ use crate::encoder::resources::{ create_image, map_bitstream_buffer, query_timestamp_diff, submit_encode_only, wait_and_read_bitstream, }; -use crate::encoder::{BitDepth, EncodedPacket, FrameType, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat, EncodedPacket, FrameType}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::{SlotSync, TimelineChain}; @@ -197,7 +197,7 @@ pub(crate) struct PipelineConfig<'a> { pub picture_format: vk::Format, /// Whether the input image is RGB, for the encoder to convert. pub rgb_input: bool, - pub pixel_format: PixelFormat, + pub chroma_format: ChromaFormat, pub bit_depth: BitDepth, pub bitstream_buffer_size: usize, /// Codec profile (with the codec-specific profile chained in) used for the @@ -287,7 +287,7 @@ impl EncodePipeline { image: input_image, width: config.aligned_width, height: config.aligned_height, - pixel_format: config.pixel_format, + chroma_format: config.chroma_format, bit_depth: config.bit_depth, rgb: config.rgb_input, }, diff --git a/src/encoder/profile.rs b/src/encoder/profile.rs index 9fe321a7..b15378cc 100644 --- a/src/encoder/profile.rs +++ b/src/encoder/profile.rs @@ -4,14 +4,14 @@ //! both choose their profile here, so the query answers for exactly the profile an //! encoder would request. -use super::{BitDepth, Codec, EncodeConfig, PixelFormat}; +use super::{BitDepth, ChromaFormat, Codec, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use ash::vk; /// H.264 profile for a pixel format: High 4:4:4 Predictive for 4:4:4, High otherwise. -pub(crate) fn h264_profile_idc(pixel_format: PixelFormat) -> u32 { - match pixel_format { - PixelFormat::Yuv444 => { +pub(crate) fn h264_profile_idc(chroma_format: ChromaFormat) -> u32 { + match chroma_format { + ChromaFormat::Yuv444 => { ash::vk::native::StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE } _ => ash::vk::native::StdVideoH264ProfileIdc_STD_VIDEO_H264_PROFILE_IDC_HIGH, @@ -20,15 +20,15 @@ pub(crate) fn h264_profile_idc(pixel_format: PixelFormat) -> u32 { /// H.265 profile for a pixel format and bit depth, or `None` for combinations the /// encoder does not support (4:2:2). -pub(crate) fn h265_profile_idc(pixel_format: PixelFormat, bit_depth: BitDepth) -> Option { - match (pixel_format, bit_depth) { - (PixelFormat::Yuv420, BitDepth::Eight) => { +pub(crate) fn h265_profile_idc(chroma_format: ChromaFormat, bit_depth: BitDepth) -> Option { + match (chroma_format, bit_depth) { + (ChromaFormat::Yuv420, BitDepth::Eight) => { Some(ash::vk::native::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN) } - (PixelFormat::Yuv420, BitDepth::Ten) => { + (ChromaFormat::Yuv420, BitDepth::Ten) => { Some(ash::vk::native::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_MAIN_10) } - (PixelFormat::Yuv444, _) => Some( + (ChromaFormat::Yuv444, _) => Some( ash::vk::native::StdVideoH265ProfileIdc_STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS, ), _ => None, @@ -36,9 +36,9 @@ pub(crate) fn h265_profile_idc(pixel_format: PixelFormat, bit_depth: BitDepth) - } /// AV1 profile for a pixel format: Main is 4:2:0, High adds 4:4:4. -pub(crate) fn av1_profile(pixel_format: PixelFormat) -> u32 { - match pixel_format { - PixelFormat::Yuv420 => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_MAIN, +pub(crate) fn av1_profile(chroma_format: ChromaFormat) -> u32 { + match chroma_format { + ChromaFormat::Yuv420 => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_MAIN, _ => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_HIGH, } } @@ -77,10 +77,10 @@ impl EncodeProfile { }; let h265_idc = match config.codec { Codec::H265 => { - h265_profile_idc(config.pixel_format, config.bit_depth).ok_or_else(|| { + h265_profile_idc(config.chroma_format, config.bit_depth).ok_or_else(|| { PixelForgeError::InvalidInput(format!( "Unsupported pixel format / bit depth for H.265: {:?} / {:?}", - config.pixel_format, config.bit_depth + config.chroma_format, config.bit_depth )) })? } @@ -90,14 +90,14 @@ impl EncodeProfile { let mut chain = Box::new(ProfileChain { info: vk::VideoProfileInfoKHR::default() .video_codec_operation(operation) - .chroma_subsampling(config.pixel_format.into()) + .chroma_subsampling(config.chroma_format.into()) .luma_bit_depth(bit_depth) .chroma_bit_depth(bit_depth), h264: vk::VideoEncodeH264ProfileInfoKHR::default() - .std_profile_idc(h264_profile_idc(config.pixel_format)), + .std_profile_idc(h264_profile_idc(config.chroma_format)), h265: vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(h265_idc), av1: vk::VideoEncodeAV1ProfileInfoKHR::default() - .std_profile(av1_profile(config.pixel_format)), + .std_profile(av1_profile(config.chroma_format)), usage: vk::VideoEncodeUsageInfoKHR::default() .video_usage_hints(config.encode_usage_hint.into()) .video_content_hints(config.encode_content_hint.into()) diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index ca373ee1..1a5d4eda 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -1,5 +1,5 @@ use crate::encoder::input::SourceOwnership; -use crate::encoder::{BitDepth, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk::TaggedStructure; @@ -193,17 +193,17 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam // meets VkBufferImageCopy::bufferOffset alignment requirements. // YUV420/422 dimensions are always even, so alignment is naturally satisfied. let plane0_raw = (params.width * params.height * bytes_per_component) as usize; - let plane0_size = match params.pixel_format { - PixelFormat::Yuv444 => crate::align4(plane0_raw) as u32, + let plane0_size = match params.chroma_format { + ChromaFormat::Yuv444 => crate::align4(plane0_raw) as u32, _ => plane0_raw as u32, }; - let plane1_size = match params.pixel_format { + let plane1_size = match params.chroma_format { // YUV 4:2:0 (e.g., NV12): UV plane is half width, half height, 2 components per pixel. - PixelFormat::Yuv420 => (params.width / 2) * (params.height / 2) * 2 * bytes_per_component, + ChromaFormat::Yuv420 => (params.width / 2) * (params.height / 2) * 2 * bytes_per_component, // YUV 4:2:2: UV plane is half width, full height, 2 components per pixel. - PixelFormat::Yuv422 => (params.width / 2) * params.height * 2 * bytes_per_component, + ChromaFormat::Yuv422 => (params.width / 2) * params.height * 2 * bytes_per_component, // YUV 4:4:4 (e.g., NV24): UV plane is full width, full height, 2 components per pixel. - PixelFormat::Yuv444 => params.width * params.height * 2 * bytes_per_component, + ChromaFormat::Yuv444 => params.width * params.height * 2 * bytes_per_component, }; // Every RGB input format is 4 bytes a pixel in one plane. let rgb_size = params.width * params.height * 4; @@ -292,9 +292,9 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam } // Copy from staging buffer to image planes. - let (uv_width, uv_height) = match params.pixel_format { - PixelFormat::Yuv420 => (params.width / 2, params.height / 2), - PixelFormat::Yuv444 => (params.width, params.height), + let (uv_width, uv_height) = match params.chroma_format { + ChromaFormat::Yuv420 => (params.width / 2, params.height / 2), + ChromaFormat::Yuv444 => (params.width, params.height), _ => (params.width / 2, params.height / 2), }; @@ -424,8 +424,8 @@ pub(crate) struct UploadParams<'a> { pub width: u32, /// The height of the image. pub height: u32, - /// The pixel format of the image. - pub pixel_format: PixelFormat, + /// The chroma format of the image. + pub chroma_format: ChromaFormat, /// The current layout of the input image. pub input_image_layout: vk::ImageLayout, /// Whether both images are single-plane RGB rather than YUV. @@ -521,9 +521,9 @@ pub(crate) fn upload_image_to_input( }; // Copy UV plane (plane 1). - let (uv_width, uv_height) = match params.pixel_format { - PixelFormat::Yuv420 => (params.width / 2, params.height / 2), - PixelFormat::Yuv444 => (params.width, params.height), + let (uv_width, uv_height) = match params.chroma_format { + ChromaFormat::Yuv420 => (params.width / 2, params.height / 2), + ChromaFormat::Yuv444 => (params.width, params.height), _ => (params.width / 2, params.height / 2), }; @@ -1147,7 +1147,7 @@ pub(crate) struct ClearImageParams { pub image: vk::Image, pub width: u32, pub height: u32, - pub pixel_format: PixelFormat, + pub chroma_format: ChromaFormat, pub bit_depth: BitDepth, /// Whether the image is a single-plane, 4-bytes-per-pixel RGB input /// image rather than a YUV one. diff --git a/src/encoder/rgb.rs b/src/encoder/rgb.rs index 9992d4a9..377a5c9a 100644 --- a/src/encoder/rgb.rs +++ b/src/encoder/rgb.rs @@ -8,8 +8,8 @@ //! matrix, range and chroma siting to convert with. use super::support::report; -use super::{BitDepth, Codec, ColorDescription, EncodeConfig, PixelFormat, StreamColor}; -use crate::converter::InputFormat; +use super::{BitDepth, ChromaFormat, Codec, ColorDescription, EncodeConfig, StreamColor}; +use crate::converter::RgbFormat; use crate::error::{PixelForgeError, Result}; use crate::video::query_supported_video_formats; use crate::vulkan::VideoContext; @@ -26,13 +26,13 @@ pub(crate) fn rgb_conversion_profile() -> vk::VideoEncodeProfileRgbConversionInf /// 8-bit RGB becomes 8-bit YUV and 10-bit RGB 10-bit YUV. Half floats carry /// values outside the 0 to 1 range that need tone mapping first, which only /// the colour converter does. -pub(crate) fn rgb_input_bit_depth(format: InputFormat) -> Option { +pub(crate) fn rgb_input_bit_depth(format: RgbFormat) -> Option { match format { - InputFormat::BGRx | InputFormat::BGRA | InputFormat::RGBx | InputFormat::RGBA => { + RgbFormat::BGRx | RgbFormat::BGRA | RgbFormat::RGBx | RgbFormat::RGBA => { Some(BitDepth::Eight) } - InputFormat::ABGR2101010 => Some(BitDepth::Ten), - InputFormat::RGBA16F => None, + RgbFormat::ABGR2101010 => Some(BitDepth::Ten), + RgbFormat::RGBA16F => None, } } @@ -45,7 +45,7 @@ pub(crate) fn rgb_input_bit_depth(format: InputFormat) -> Option { pub(crate) fn check_rgb_input( context: &VideoContext, config: &EncodeConfig, - format: InputFormat, + format: RgbFormat, ) -> Result<()> { if !context.has_video_encode_rgb_conversion() { return Err(PixelForgeError::NoSuitableDevice( @@ -60,11 +60,11 @@ pub(crate) fn check_rgb_input( converter does" ))); }; - if config.pixel_format != PixelFormat::Yuv420 || config.bit_depth != bit_depth { + if config.chroma_format != ChromaFormat::Yuv420 || config.bit_depth != bit_depth { return Err(PixelForgeError::InvalidInput(format!( "RGB input: {format:?} converts to {bit_depth:?} 4:2:0, but the encoder is set \ up for {:?} {:?}", - config.bit_depth, config.pixel_format + config.bit_depth, config.chroma_format ))); } Ok(()) @@ -159,7 +159,7 @@ impl VideoContext { /// profile exists, that its conversion does `color`'s matrix and range, /// and that it accepts `format` as an input image. The output is always /// 4:2:0, at 8 bits for 8-bit RGB and 10 bits for - /// [`InputFormat::ABGR2101010`]. Resolution and rate control are not + /// [`RgbFormat::ABGR2101010`]. Resolution and rate control are not /// considered. /// /// When this is true, an encoder for an RGB [`Source`](crate::Source) in @@ -168,7 +168,7 @@ impl VideoContext { pub fn supports_rgb_encode_input( &self, codec: Codec, - format: InputFormat, + format: RgbFormat, color: StreamColor, ) -> bool { if matches!(color, StreamColor::Other(_)) { @@ -185,7 +185,7 @@ impl VideoContext { let what = format!("{codec:?} encode from {format:?} RGB"); let formats = self.query_encode_profile( codec, - PixelFormat::Yuv420, + ChromaFormat::Yuv420, bit_depth, Some(&mut caps), |profile| { diff --git a/src/encoder/source.rs b/src/encoder/source.rs index a5019700..bd44e606 100644 --- a/src/encoder/source.rs +++ b/src/encoder/source.rs @@ -6,15 +6,15 @@ //! RGB conversion, and always submits frames the same way. use super::rgb::rgb_input_bit_depth; -use super::{BitDepth, EncodeConfig, PixelFormat, StreamColor}; -use crate::converter::{ColorConverterConfig, ColorRange, ColorSpec, InputFormat, OutputFormat}; +use super::{BitDepth, ChromaFormat, EncodeConfig, StreamColor}; +use crate::converter::{ColorConverterConfig, ColorRange, ColorSpec, OutputFormat, RgbFormat}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; /// What the frames handed to an [`Encoder`](crate::Encoder) are. #[derive(Debug, Clone, Copy, PartialEq, Default)] pub enum Source { - /// YUV frames, already in the encoder's pixel format and bit depth. + /// YUV frames, already in the encoder's chroma format and bit depth. #[default] Yuv, /// RGB frames, which pixelforge converts to YUV. @@ -25,7 +25,7 @@ pub enum Source { #[derive(Debug, Clone, Copy, PartialEq)] pub struct RgbSource { /// How the pixels are laid out. - pub format: InputFormat, + pub format: RgbFormat, /// What the pixel values mean. pub color: ColorSpec, /// How bright the source's white is, in nits, overriding @@ -36,7 +36,7 @@ pub struct RgbSource { impl RgbSource { /// RGB frames in `format`, holding colours in `color`. - pub fn new(format: InputFormat, color: ColorSpec) -> Self { + pub fn new(format: RgbFormat, color: ColorSpec) -> Self { Self { format, color, @@ -82,7 +82,7 @@ pub enum Conversion { pub(crate) struct Plan { pub conversion: Conversion, /// The RGB format the encoder converts from, for [`Conversion::Encoder`]. - pub rgb_input: Option, + pub rgb_input: Option, /// The colour converter's configuration, for [`Conversion::Shader`]. pub converter: Option, } @@ -98,12 +98,12 @@ pub(crate) fn target_space(color: &StreamColor) -> Option<(ColorSpec, ColorRange } /// The YUV the colour converter has to write for this encoder. -fn shader_output(pixel_format: PixelFormat, bit_depth: BitDepth) -> Option { - match (pixel_format, bit_depth) { - (PixelFormat::Yuv420, BitDepth::Eight) => Some(OutputFormat::NV12), - (PixelFormat::Yuv420, BitDepth::Ten) => Some(OutputFormat::P010), - (PixelFormat::Yuv444, BitDepth::Eight) => Some(OutputFormat::YUV444), - (PixelFormat::Yuv444, BitDepth::Ten) => Some(OutputFormat::YUV444P10), +fn shader_output(chroma_format: ChromaFormat, bit_depth: BitDepth) -> Option { + match (chroma_format, bit_depth) { + (ChromaFormat::Yuv420, BitDepth::Eight) => Some(OutputFormat::NV12), + (ChromaFormat::Yuv420, BitDepth::Ten) => Some(OutputFormat::P010), + (ChromaFormat::Yuv444, BitDepth::Eight) => Some(OutputFormat::YUV444), + (ChromaFormat::Yuv444, BitDepth::Ten) => Some(OutputFormat::YUV444P10), _ => None, } } @@ -193,11 +193,12 @@ fn encoder_conversion( )); } match rgb_input_bit_depth(rgb.format) { - Some(depth) if config.pixel_format == PixelFormat::Yuv420 && config.bit_depth == depth => {} + Some(depth) + if config.chroma_format == ChromaFormat::Yuv420 && config.bit_depth == depth => {} Some(depth) => { return no(format!( "{:?} converts to {depth:?} 4:2:0, the encoder is set up for {:?} {:?}", - rgb.format, config.bit_depth, config.pixel_format + rgb.format, config.bit_depth, config.chroma_format )); } None => return no(format!("{:?} needs tone mapping first", rgb.format)), @@ -225,10 +226,10 @@ fn shader_conversion( "RGB input: the colour converter needs VK_KHR_push_descriptor".to_string(), )); } - let output = shader_output(config.pixel_format, config.bit_depth).ok_or_else(|| { + let output = shader_output(config.chroma_format, config.bit_depth).ok_or_else(|| { PixelForgeError::InvalidInput(format!( "RGB input: the colour converter does not write {:?} {:?}", - config.pixel_format, config.bit_depth + config.chroma_format, config.bit_depth )) })?; let mut converter = ColorConverterConfig::new( @@ -279,9 +280,9 @@ mod tests { #[test] fn shader_output_matches_the_encoder() { assert_eq!( - shader_output(PixelFormat::Yuv420, BitDepth::Ten), + shader_output(ChromaFormat::Yuv420, BitDepth::Ten), Some(OutputFormat::P010) ); - assert_eq!(shader_output(PixelFormat::Yuv422, BitDepth::Eight), None); + assert_eq!(shader_output(ChromaFormat::Yuv422, BitDepth::Eight), None); } } diff --git a/src/encoder/support.rs b/src/encoder/support.rs index f4600a8c..daa103b9 100644 --- a/src/encoder/support.rs +++ b/src/encoder/support.rs @@ -3,7 +3,7 @@ use super::codec::query_video_capabilities; use super::profile::{av1_profile, h264_profile_idc, h265_profile_idc}; use super::rgb::rgb_conversion_profile; -use super::{BitDepth, Codec, PixelFormat}; +use super::{BitDepth, ChromaFormat, Codec}; use crate::vulkan::VideoContext; use ash::vk; use ash::vk::TaggedStructure; @@ -18,7 +18,7 @@ pub(crate) enum ProfileQueryError { } impl VideoContext { - /// Check whether the device can encode `codec` in the given pixel format and + /// Check whether the device can encode `codec` in the given chroma format and /// bit depth. /// /// [`supports_encode`](Self::supports_encode) only answers for 8-bit 4:2:0. @@ -29,21 +29,21 @@ impl VideoContext { pub fn supports_encode_format( &self, codec: Codec, - pixel_format: PixelFormat, + chroma_format: ChromaFormat, bit_depth: BitDepth, ) -> bool { if !self.supports_encode(codec) { return false; } - let result = self.query_encode_profile(codec, pixel_format, bit_depth, None, |_| ()); + let result = self.query_encode_profile(codec, chroma_format, bit_depth, None, |_| ()); report( - &format!("{codec:?} encode {pixel_format:?} {bit_depth:?}"), + &format!("{codec:?} encode {chroma_format:?} {bit_depth:?}"), result, ) .is_some() } - /// Build the profile an encoder would request for `codec`, `pixel_format` + /// Build the profile an encoder would request for `codec`, `chroma_format` /// and `bit_depth`, ask the driver for its capabilities, and on success /// hand the profile to `then` for any further query about it. /// @@ -57,12 +57,12 @@ impl VideoContext { pub(crate) fn query_encode_profile( &self, codec: Codec, - pixel_format: PixelFormat, + chroma_format: ChromaFormat, bit_depth: BitDepth, mut rgb_caps: Option<&mut vk::VideoEncodeRgbConversionCapabilitiesVALVE<'static>>, then: impl FnOnce(&vk::VideoProfileInfoKHR<'_>) -> R, ) -> Result { - let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = pixel_format.into(); + let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = chroma_format.into(); let component_bit_depth: vk::VideoComponentBitDepthFlagsKHR = bit_depth.into(); let mut encode_caps = vk::VideoEncodeCapabilitiesKHR::default(); let mut rgb_profile = rgb_conversion_profile(); @@ -99,11 +99,11 @@ impl VideoContext { Codec::H264 => query!( vk::VideoCodecOperationFlagsKHR::ENCODE_H264, vk::VideoEncodeH264ProfileInfoKHR::default() - .std_profile_idc(h264_profile_idc(pixel_format)), + .std_profile_idc(h264_profile_idc(chroma_format)), vk::VideoEncodeH264CapabilitiesKHR::default() ), Codec::H265 => { - let profile_idc = h265_profile_idc(pixel_format, bit_depth) + let profile_idc = h265_profile_idc(chroma_format, bit_depth) .ok_or(ProfileQueryError::NoProfile)?; query!( vk::VideoCodecOperationFlagsKHR::ENCODE_H265, @@ -113,7 +113,7 @@ impl VideoContext { } Codec::AV1 => query!( vk::VideoCodecOperationFlagsKHR::ENCODE_AV1, - vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(av1_profile(pixel_format)), + vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(av1_profile(chroma_format)), vk::VideoEncodeAV1CapabilitiesKHR::default() ), } diff --git a/src/image.rs b/src/image.rs index 79b1423c..1140d95d 100644 --- a/src/image.rs +++ b/src/image.rs @@ -1,20 +1,20 @@ //! Image utilities for creating and uploading video frames. //! -//! This module provides [`InputImage`], a helper for uploading YUV data from the CPU +//! This module provides [`YuvUploader`], a helper for uploading YUV data from the CPU //! to the GPU. The image it creates is a transfer-only staging image (not a Vulkan Video //! image) and must be copied into an encoder's input image before encoding. -//! Use [`InputImage::upload`] to upload to the internal image, then submit -//! [`InputImage::source`] with [`Frame::image`](crate::Frame::image). -//! Alternatively, use [`InputImage::upload_to`] to upload directly into an +//! Use [`YuvUploader::upload`] to upload to the internal image, then submit +//! [`YuvUploader::source`] with [`Frame::image`](crate::Frame::image). +//! Alternatively, use [`YuvUploader::upload_to`] to upload directly into an //! encoder's [`InputSlot`] (from //! [`Encoder::input_slot`](crate::encoder::Encoder::input_slot)). -use crate::encoder::{BitDepth, Encoder, Handoff, InputSlot, PixelFormat, Source, SourceImage}; +use crate::encoder::{BitDepth, ChromaFormat, Encoder, Handoff, InputSlot, Source, SourceImage}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; -/// How the YUV samples handed to [`InputImage::upload`] are laid out in +/// How the YUV samples handed to [`YuvUploader::upload`] are laid out in /// memory. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum YuvLayout { @@ -58,7 +58,7 @@ impl YuvLayout { /// input image, or /// - [`upload_to`](Self::upload_to) the encoder's input image directly, /// through an [`InputSlot`]. -pub struct InputImage { +pub struct YuvUploader { /// Keeps track of the frames that read `image`. source: SourceImage, context: VideoContext, @@ -77,11 +77,11 @@ pub struct InputImage { width: u32, height: u32, bit_depth: BitDepth, - pixel_format: PixelFormat, + chroma_format: ChromaFormat, } -impl InputImage { - /// A staging image for the frames of `encoder`: its size, pixel format +impl YuvUploader { + /// A staging image for the frames of `encoder`: its size, chroma format /// and bit depth. /// /// An error for an encoder with an RGB [`Source`]: this uploads YUV. @@ -89,7 +89,7 @@ impl InputImage { let config = encoder.config(); if config.source != Source::Yuv { return Err(PixelForgeError::InvalidInput( - "an InputImage uploads YUV, and the encoder's source is RGB".to_string(), + "a YuvUploader uploads YUV, and the encoder's source is RGB".to_string(), )); } Self::new( @@ -97,7 +97,7 @@ impl InputImage { config.dimensions.width, config.dimensions.height, config.bit_depth, - config.pixel_format, + config.chroma_format, ) } @@ -112,31 +112,31 @@ impl InputImage { /// * `width` - Image width in pixels /// * `height` - Image height in pixels /// * `bit_depth` - Bit depth for the image (8-bit or 10-bit) - /// * `pixel_format` - Pixel format (YUV420 or YUV444) + /// * `chroma_format` - Chroma format (YUV420 or YUV444) pub fn new( context: VideoContext, width: u32, height: u32, bit_depth: BitDepth, - pixel_format: PixelFormat, + chroma_format: ChromaFormat, ) -> Result { let device = context.device(); - // Select format based on pixel format and bit depth. + // Select format based on chroma format and bit depth. // Use 2-plane semi-planar formats for both YUV420 and YUV444. - let format = match (pixel_format, bit_depth) { - (PixelFormat::Yuv420, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_420_UNORM, - (PixelFormat::Yuv420, BitDepth::Ten) => { + let format = match (chroma_format, bit_depth) { + (ChromaFormat::Yuv420, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_420_UNORM, + (ChromaFormat::Yuv420, BitDepth::Ten) => { vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 } - (PixelFormat::Yuv444, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_444_UNORM, - (PixelFormat::Yuv444, BitDepth::Ten) => { + (ChromaFormat::Yuv444, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_444_UNORM, + (ChromaFormat::Yuv444, BitDepth::Ten) => { vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 } _ => { return Err(PixelForgeError::InvalidInput(format!( - "Unsupported pixel format / bit depth combination: {:?} / {:?}", - pixel_format, bit_depth + "Unsupported chroma format / bit depth combination: {:?} / {:?}", + chroma_format, bit_depth ))); } }; @@ -186,19 +186,19 @@ impl InputImage { .map_err(PixelForgeError::vk("vkBindImageMemory"))?; // Create staging buffer for uploads. - // Size depends on pixel format, bit depth, and alignment padding between planes. + // Size depends on chroma format, bit depth, and alignment padding between planes. let bytes_per_sample_init = match bit_depth { BitDepth::Eight => 1usize, BitDepth::Ten => 2, }; let luma_pixels = (width * height) as usize; - let staging_size = match pixel_format { - PixelFormat::Yuv444 => { + let staging_size = match chroma_format { + ChromaFormat::Yuv444 => { // Y plane (aligned to 4 bytes) + UV interleaved plane. let y_bytes = luma_pixels * bytes_per_sample_init; crate::align4(y_bytes) + luma_pixels * 2 * bytes_per_sample_init } - _ => pixel_format.frame_size(width, height) * bytes_per_sample_init, + _ => chroma_format.frame_size(width, height) * bytes_per_sample_init, }; let buffer_create_info = vk::BufferCreateInfo::default() .size(staging_size as vk::DeviceSize) @@ -276,7 +276,7 @@ impl InputImage { width, height, bit_depth, - pixel_format, + chroma_format, }; // Start from zeroed planes in GENERAL, so the image is a valid source // before the first upload. @@ -323,16 +323,16 @@ impl InputImage { /// Convert `data` into the staging buffer, in the image's format. fn write_staging(&mut self, layout: YuvLayout, data: &[u8]) -> Result<()> { let fits = matches!( - (layout, self.pixel_format, self.bit_depth), - (YuvLayout::I420, PixelFormat::Yuv420, _) - | (YuvLayout::Nv12, PixelFormat::Yuv420, BitDepth::Eight) - | (YuvLayout::I420P10, PixelFormat::Yuv420, BitDepth::Ten) - | (YuvLayout::I444, PixelFormat::Yuv444, _) + (layout, self.chroma_format, self.bit_depth), + (YuvLayout::I420, ChromaFormat::Yuv420, _) + | (YuvLayout::Nv12, ChromaFormat::Yuv420, BitDepth::Eight) + | (YuvLayout::I420P10, ChromaFormat::Yuv420, BitDepth::Ten) + | (YuvLayout::I444, ChromaFormat::Yuv444, _) ); if !fits { return Err(PixelForgeError::InvalidInput(format!( "{layout:?} data cannot be uploaded to a {:?} {:?}-bit image", - self.pixel_format, self.bit_depth + self.chroma_format, self.bit_depth ))); } let expected_size = layout.frame_size(self.width, self.height); @@ -344,16 +344,16 @@ impl InputImage { } let luma = (self.width * self.height) as usize; - let chroma = match self.pixel_format { - PixelFormat::Yuv444 => luma, + let chroma = match self.chroma_format { + ChromaFormat::Yuv444 => luma, _ => luma / 4, }; let ten_bit = self.bit_depth == BitDepth::Ten; let luma_bytes = if ten_bit { luma * 2 } else { luma }; // The UV plane of a 4:4:4 image starts 4-byte aligned, as // `copy_staging_to_image_internal` copies it. - let uv_offset = match self.pixel_format { - PixelFormat::Yuv444 => crate::align4(luma_bytes), + let uv_offset = match self.chroma_format { + ChromaFormat::Yuv444 => crate::align4(luma_bytes), _ => luma_bytes, }; @@ -449,8 +449,8 @@ impl InputImage { // YUV420 dimensions are always even (required for 4:2:0), so alignment is // naturally satisfied and must not pad to stay consistent with upload functions. let y_plane_bytes = (width * height) as usize * bytes_per_sample; - let y_plane_size_bytes = match self.pixel_format { - PixelFormat::Yuv444 => crate::align4(y_plane_bytes) as vk::DeviceSize, + let y_plane_size_bytes = match self.chroma_format { + ChromaFormat::Yuv444 => crate::align4(y_plane_bytes) as vk::DeviceSize, _ => y_plane_bytes as vk::DeviceSize, }; @@ -493,9 +493,9 @@ impl InputImage { ); } - // Build copy regions based on pixel format. - let copy_regions = match self.pixel_format { - PixelFormat::Yuv420 => { + // Build copy regions based on chroma format. + let copy_regions = match self.chroma_format { + ChromaFormat::Yuv420 => { // NV12/P010: Y plane (PLANE_0) + interleaved UV plane (PLANE_1) vec![ // Y plane. @@ -534,7 +534,7 @@ impl InputImage { }), ] } - PixelFormat::Yuv444 => { + ChromaFormat::Yuv444 => { // 2-plane semi-planar YUV444: Y (PLANE_0), UV interleaved (PLANE_1) at full resolution. vec![ // Y plane. @@ -575,8 +575,8 @@ impl InputImage { } _ => { return Err(PixelForgeError::InvalidInput(format!( - "Unsupported pixel format: {:?}", - self.pixel_format + "Unsupported chroma format: {:?}", + self.chroma_format ))); } }; @@ -657,7 +657,7 @@ impl InputImage { } } -impl Drop for InputImage { +impl Drop for YuvUploader { fn drop(&mut self) { let device = self.context.device(); unsafe { diff --git a/src/lib.rs b/src/lib.rs index 680d8393..6378cd6d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,7 @@ //! - **GPU-native API**: Encode directly from Vulkan images (`vk::Image`). //! - **Flexible configuration**: Rate control (CBR, VBR, CQP), quality levels, GOP settings. //! - **Multiple input formats**: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). -//! - **Utility helpers**: [`InputImage`] for easy YUV data upload to GPU. +//! - **Utility helpers**: [`YuvUploader`] for easy YUV data upload to GPU. //! - **Optional DMA-BUF input**: import a producer's buffers, or allocate buffers //! for it, read with no copy where the driver allows, with sync_file fences //! (Linux only). @@ -90,7 +90,7 @@ //! ## Encoding Video //! //! ```rust,no_run -//! use pixelforge::{Codec, EncodeConfig, Encoder, InputImage, RateControl, VideoContextBuilder}; +//! use pixelforge::{Codec, EncodeConfig, Encoder, YuvUploader, RateControl, VideoContextBuilder}; //! //! fn main() -> Result<(), Box> { //! let context = VideoContextBuilder::new() @@ -107,8 +107,8 @@ //! .with_gop_size(60); //! //! let mut encoder = Encoder::new(context, config)?; -//! // Create an InputImage helper for uploading YUV data to the GPU. -//! let mut input_image = InputImage::for_encoder(&encoder)?; +//! // Create a YuvUploader helper for uploading YUV data to the GPU. +//! let mut input_image = YuvUploader::for_encoder(&encoder)?; //! //! // For each frame: upload YUV data and encode. //! // let yuv_data: &[u8] = ...; // YUV420 frame data @@ -189,7 +189,7 @@ //! //! ```rust,no_run //! use pixelforge::{ -//! Codec, ColorRange, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, Source, +//! Codec, ColorRange, ColorSpec, EncodeConfig, Encoder, RgbFormat, RgbSource, Source, //! StreamColor, VideoContextBuilder, //! }; //! @@ -201,7 +201,7 @@ //! // SDR desktop content, encoded as HDR10 for an HDR streaming session. //! let config = EncodeConfig::h265(1920, 1080) //! .with_bit_depth(pixelforge::BitDepth::Ten) -//! .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) +//! .with_source(Source::Rgb(RgbSource::new(RgbFormat::BGRx, ColorSpec::Srgb))) //! .with_color(StreamColor::Hdr10 { //! range: ColorRange::Full, //! }); @@ -257,9 +257,9 @@ //! - a [`SourceImage`], an image of the caller's. Pixelforge copies or //! converts it into the input image after the call returns, and the //! `SourceImage` keeps track of that: write the image again only after -//! [`SourceImage::released`]. [`InputImage`], for frames uploaded from the +//! [`SourceImage::released`]. [`YuvUploader`], for frames uploaded from the //! CPU, waits for that itself. -//! - an `ExternalImage`, a DMA-BUF (see below). +//! - a `DmaBufImage`, a DMA-BUF (see below). //! //! ## Encoding on your own device //! @@ -303,12 +303,12 @@ //! ## DMA-BUF input //! //! With the `dmabuf` feature, an encoder reads DMA-BUFs from another process. -//! `Encoder::input_formats` gives the DRM format it wants and, per DRM +//! `Encoder::dmabuf_format` gives the DRM format it wants and, per DRM //! modifier, whether it reads such a buffer with no copy. Offer those //! modifiers to the producer first. `Encoder::import` takes a producer's //! buffer; `Encoder::allocate_input` makes one for a producer that writes //! into buffers the consumer supplies, such as `ext-image-copy-capture`. -//! Either is checked once and gives an `ExternalImage`, which +//! Either is checked once and gives a `DmaBufImage`, which //! `Frame::external` submits. A buffer the encoder cannot read directly is //! copied on the GPU instead. //! @@ -391,23 +391,23 @@ pub(crate) const fn align4(size: usize) -> usize { (size + 3) & !3 } -pub use converter::{ColorRange, ColorSpec, InputFormat}; +pub use converter::{ColorRange, ColorSpec, RgbFormat}; pub use decoder::{ DecodeConfig, DecodeSink, DecodeSource, DecodeStatus, DecodedFrame, Decoder, FramePoll, Framing, }; #[cfg(feature = "dmabuf")] -pub use encoder::external::{DmaBuf, DmaBufPlane, ExternalImage, InputFormats, InputModifier}; +pub use encoder::external::{DmaBuf, DmaBufFormat, DmaBufImage, DmaBufModifier, DmaBufPlane}; pub use encoder::{ - BitDepth, Codec, ColorDescription, Dimensions, EncodeConfig, EncodeContentHint, EncodeFuture, - EncodeUsageHint, EncodedPacket, Encoder, EncoderTuningMode, Frame, FrameRate, FrameType, - PixelFormat, RateControl, + BitDepth, ChromaFormat, Codec, ColorDescription, Dimensions, EncodeConfig, EncodeContentHint, + EncodeFuture, EncodeUsageHint, EncodedPacket, Encoder, EncoderTuningMode, Frame, FrameRate, + FrameType, RateControl, }; pub use encoder::{ Conversion, ConversionPreference, Handoff, InputSlot, RgbSource, Source, SourceImage, StreamColor, }; pub use error::PixelForgeError; -pub use image::{InputImage, YuvLayout}; +pub use image::{YuvLayout, YuvUploader}; pub use sync::TimelinePoint; pub use vulkan::VideoContextBuilder; diff --git a/src/video.rs b/src/video.rs index ad46ce92..be7ce1c8 100644 --- a/src/video.rs +++ b/src/video.rs @@ -6,7 +6,7 @@ //! resource management (encode slot pipelines, decode DPB pools) lives in //! `encoder::resources` and `decoder` respectively, built on these primitives. -use crate::encoder::{BitDepth, PixelFormat}; +use crate::encoder::{BitDepth, ChromaFormat}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; @@ -289,26 +289,26 @@ pub(crate) fn query_supported_video_formats( .collect()) } -/// Get the Vulkan format for a given pixel format and bit depth. +/// Get the Vulkan format for a given chroma format and bit depth. /// /// Supports YUV420 and YUV444 in 8-bit and 10-bit. /// For YUV444, uses 2-plane (semi-planar) formats from VK_EXT_ycbcr_2plane_444_formats /// which are supported by NVIDIA hardware for video encoding. -pub(crate) fn get_video_format(pixel_format: PixelFormat, bit_depth: BitDepth) -> vk::Format { - match (pixel_format, bit_depth) { - (PixelFormat::Yuv420, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_420_UNORM, - (PixelFormat::Yuv420, BitDepth::Ten) => { +pub(crate) fn get_video_format(chroma_format: ChromaFormat, bit_depth: BitDepth) -> vk::Format { + match (chroma_format, bit_depth) { + (ChromaFormat::Yuv420, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_420_UNORM, + (ChromaFormat::Yuv420, BitDepth::Ten) => { vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 } // Use 2-plane semi-planar formats for YUV444 (supported by NVIDIA for video encoding). - (PixelFormat::Yuv444, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_444_UNORM, - (PixelFormat::Yuv444, BitDepth::Ten) => { + (ChromaFormat::Yuv444, BitDepth::Eight) => vk::Format::G8_B8R8_2PLANE_444_UNORM, + (ChromaFormat::Yuv444, BitDepth::Ten) => { vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16 } // TODO: Add support for YUV422 formats. _ => unimplemented!( - "Unsupported pixel format / bit depth combination: {:?} / {:?}", - pixel_format, + "Unsupported chroma format / bit depth combination: {:?} / {:?}", + chroma_format, bit_depth ), } diff --git a/tests/adopted.rs b/tests/adopted.rs index 8b721fb8..6f64b1b2 100644 --- a/tests/adopted.rs +++ b/tests/adopted.rs @@ -19,8 +19,8 @@ use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::encoder::Codec; use pixelforge::vulkan::{DeviceQueue, DeviceRequirements, VideoContext, VideoContextBuilder}; use pixelforge::{ - ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, InputFormat, - RateControl, RgbSource, Source, SourceImage, TimelinePoint, + ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, RateControl, RgbFormat, + RgbSource, Source, SourceImage, TimelinePoint, }; /// An instance and device the test owns, destroyed on drop after everything @@ -251,7 +251,7 @@ fn encode_all(context: &VideoContext, wait: Wait) -> Result, Box], ) -> Result, Box> { let mut encoder = Encoder::new(context.clone(), config.clone())?; - let mut input = InputImage::for_encoder(&encoder)?; + let mut input = YuvUploader::for_encoder(&encoder)?; let mut stream = Vec::new(); for frame in frames { input.upload(YuvLayout::Nv12, frame)?; @@ -441,7 +441,7 @@ fn encode_reference( fn encode_external( encoder: &mut Encoder, producer: &mut Producer, - external: &ExternalImage, + external: &DmaBufImage, frames: &[Vec], ) -> Result, Box> { let mut stream = Vec::new(); @@ -455,7 +455,7 @@ fn encode_external( /// Every modifier the encoder can allocate. fn exportable_modifiers(encoder: &Encoder) -> Vec { - encoder.input_formats().map_or(Vec::new(), |formats| { + encoder.dmabuf_format().map_or(Vec::new(), |formats| { formats .modifiers .iter() @@ -518,7 +518,7 @@ fn copy_only_modifiers_are_copied() -> Result<(), Box> { let frames: Vec> = (0..FRAMES).map(|n| nv12_frame(w, h, n)).collect(); let reference = encode_reference(&context, config(Codec::H264, w, h), &frames)?; let mut encoder = Encoder::new(context.clone(), config(Codec::H264, w, h))?; - let formats = encoder.input_formats()?; + let formats = encoder.dmabuf_format()?; let Some(copy_only) = formats .modifiers .iter() @@ -543,7 +543,7 @@ fn copy_only_modifiers_are_copied() -> Result<(), Box> { fn rgb_buffers_are_converted_directly() -> Result<(), Box> { let context = context()?; let (w, h) = (320, 256); - let rgb = Source::Rgb(RgbSource::new(InputFormat::BGRA, ColorSpec::Srgb)); + let rgb = Source::Rgb(RgbSource::new(RgbFormat::BGRA, ColorSpec::Srgb)); let shader = |c: EncodeConfig| { c.with_source(rgb) .with_conversion(ConversionPreference::Shader) diff --git a/tests/encode_format_support.rs b/tests/encode_format_support.rs index 85d1aa8c..21d0a82b 100644 --- a/tests/encode_format_support.rs +++ b/tests/encode_format_support.rs @@ -4,15 +4,15 @@ #[allow(dead_code)] mod common; -use pixelforge::{BitDepth, Codec, EncodeConfig, Encoder, PixelFormat, VideoContextBuilder}; +use pixelforge::{BitDepth, ChromaFormat, Codec, EncodeConfig, Encoder, VideoContextBuilder}; -fn config(codec: Codec, pixel_format: PixelFormat, bit_depth: BitDepth) -> EncodeConfig { +fn config(codec: Codec, chroma_format: ChromaFormat, bit_depth: BitDepth) -> EncodeConfig { match codec { Codec::H264 => EncodeConfig::h264(1280, 720), Codec::H265 => EncodeConfig::h265(1280, 720), Codec::AV1 => EncodeConfig::av1(1280, 720), } - .with_pixel_format(pixel_format) + .with_chroma_format(chroma_format) .with_bit_depth(bit_depth) } @@ -26,17 +26,17 @@ fn query_matches_encoder_creation() { .expect("no Vulkan video device"); for codec in [Codec::H264, Codec::H265, Codec::AV1] { - for pixel_format in [PixelFormat::Yuv420, PixelFormat::Yuv444] { + for chroma_format in [ChromaFormat::Yuv420, ChromaFormat::Yuv444] { for bit_depth in [BitDepth::Eight, BitDepth::Ten] { - let supported = context.supports_encode_format(codec, pixel_format, bit_depth); + let supported = context.supports_encode_format(codec, chroma_format, bit_depth); let created = - Encoder::new(context.clone(), config(codec, pixel_format, bit_depth)).is_ok(); + Encoder::new(context.clone(), config(codec, chroma_format, bit_depth)).is_ok(); eprintln!( - "{codec:?} {pixel_format:?} {bit_depth:?}: query={supported} encoder={created}" + "{codec:?} {chroma_format:?} {bit_depth:?}: query={supported} encoder={created}" ); assert_eq!( supported, created, - "{codec:?} {pixel_format:?} {bit_depth:?}: query and encoder creation disagree" + "{codec:?} {chroma_format:?} {bit_depth:?}: query and encoder creation disagree" ); } } @@ -53,7 +53,7 @@ fn eight_bit_420_matches_supports_encode() { .expect("no Vulkan video device"); for codec in [Codec::H264, Codec::H265, Codec::AV1] { assert_eq!( - context.supports_encode_format(codec, PixelFormat::Yuv420, BitDepth::Eight), + context.supports_encode_format(codec, ChromaFormat::Yuv420, BitDepth::Eight), context.supports_encode(codec), "{codec:?}" ); @@ -69,10 +69,10 @@ fn unsupported_codec_or_format_is_false() { .build() .expect("no Vulkan video device"); // The encoder has no 4:2:2 H.265 profile. - assert!(!context.supports_encode_format(Codec::H265, PixelFormat::Yuv422, BitDepth::Eight)); + assert!(!context.supports_encode_format(Codec::H265, ChromaFormat::Yuv422, BitDepth::Eight)); for codec in [Codec::H264, Codec::H265, Codec::AV1] { if !context.supports_encode(codec) { - assert!(!context.supports_encode_format(codec, PixelFormat::Yuv420, BitDepth::Eight)); + assert!(!context.supports_encode_format(codec, ChromaFormat::Yuv420, BitDepth::Eight)); } } } diff --git a/tests/rfi.rs b/tests/rfi.rs index d9f59690..82027e7a 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -20,8 +20,8 @@ mod common; use pixelforge::{ - BitDepth, Codec, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, - VideoContextBuilder, YuvLayout, + BitDepth, ChromaFormat, Codec, EncodeConfig, Encoder, Frame, RateControl, VideoContextBuilder, + YuvLayout, YuvUploader, }; use std::collections::VecDeque; use std::fs::File; @@ -104,13 +104,13 @@ fn run_codec( Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), } .with_rate_control(RateControl::Cqp { qp: 10 }) - .with_pixel_format(PixelFormat::Yuv420) + .with_chroma_format(ChromaFormat::Yuv420) .with_bit_depth(BitDepth::Eight) // Infinite GOP so the recovery can only come from RFI, never a periodic IDR. .with_gop_size(0); let mut encoder = Encoder::new(context.clone(), config)?; - let mut input_image = InputImage::for_encoder(&encoder)?; + let mut input_image = YuvUploader::for_encoder(&encoder)?; let mut yuv_data = Vec::new(); File::open(input_path)?.read_to_end(&mut yuv_data)?; diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index 740ccc92..86cbcf11 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -17,7 +17,7 @@ use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ BitDepth, Codec, ColorRange, ColorSpec, Conversion, ConversionPreference, EncodeConfig, - Encoder, Frame, Handoff, InputFormat, RateControl, RgbSource, Source, SourceImage, StreamColor, + Encoder, Frame, Handoff, RateControl, RgbFormat, RgbSource, Source, SourceImage, StreamColor, VideoContext, VideoContextBuilder, }; @@ -65,7 +65,7 @@ fn encode_config(codec: Codec) -> EncodeConfig { .with_color(StreamColor::default()) } -fn rgb(format: InputFormat) -> Source { +fn rgb(format: RgbFormat) -> Source { Source::Rgb(RgbSource::new(format, ColorSpec::Srgb)) } @@ -87,7 +87,7 @@ fn source_image( fn encode_rgb( context: &VideoContext, codec: Codec, - format: InputFormat, + format: RgbFormat, preference: ConversionPreference, ) -> Result, Box> { let mut encoder = Encoder::new( @@ -137,7 +137,7 @@ fn psnr(a: &[u8], b: &[u8]) -> f64 { /// Whether the encoder can convert `format` for `codec` here; prints why a /// test is skipped when not. fn encoder_converts(context: &VideoContext, codec: Codec, test: &str) -> bool { - let ok = context.supports_rgb_encode_input(codec, InputFormat::BGRA, StreamColor::default()); + let ok = context.supports_rgb_encode_input(codec, RgbFormat::BGRA, StreamColor::default()); if !ok { eprintln!("skipping {test} for {codec:?}: the encoder cannot convert RGB here"); } @@ -151,11 +151,11 @@ fn auto_picks_the_encoder_where_it_can() -> Result<(), Box Result<(), Box> { &encode_rgb( &context, Codec::H264, - InputFormat::BGRA, + RgbFormat::BGRA, ConversionPreference::Shader, )?, )?; @@ -187,7 +187,7 @@ fn rgb_input_matches_the_converter() -> Result<(), Box> { &encode_rgb( &context, Codec::H264, - InputFormat::BGRA, + RgbFormat::BGRA, ConversionPreference::Encoder, )?, )?; @@ -224,7 +224,7 @@ fn rgb_input_encodes_every_codec() -> Result<(), Box> { let stream = encode_rgb( &context, codec, - InputFormat::BGRA, + RgbFormat::BGRA, ConversionPreference::Encoder, )?; assert!(!stream.is_empty(), "{codec:?}: empty stream"); @@ -267,12 +267,12 @@ fn rgb_query_matches_encoder_creation() -> Result<(), Box continue; } for format in [ - InputFormat::BGRA, - InputFormat::RGBx, - InputFormat::ABGR2101010, - InputFormat::RGBA16F, + RgbFormat::BGRA, + RgbFormat::RGBx, + RgbFormat::ABGR2101010, + RgbFormat::RGBA16F, ] { - let bit_depth = if format == InputFormat::ABGR2101010 { + let bit_depth = if format == RgbFormat::ABGR2101010 { BitDepth::Ten } else { BitDepth::Eight @@ -312,7 +312,7 @@ fn color_change_follows_into_the_conversion() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box= FRAMES / 2 { - encoder.reconfigure(rgb(InputFormat::RGBA), StreamColor::default())?; + encoder.reconfigure(rgb(RgbFormat::RGBA), StreamColor::default())?; for pixel in pixels.as_chunks_mut::<4>().0 { pixel.swap(0, 2); } diff --git a/tests/roundtrip.rs b/tests/roundtrip.rs index 6bedd536..ec51583a 100644 --- a/tests/roundtrip.rs +++ b/tests/roundtrip.rs @@ -22,7 +22,7 @@ use common::{Readback, decode_stream, write_nv12}; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ - Codec, EncodeConfig, Encoder, Frame, InputImage, RateControl, VideoContextBuilder, YuvLayout, + Codec, EncodeConfig, Encoder, Frame, RateControl, VideoContextBuilder, YuvLayout, YuvUploader, }; const WIDTH: u32 = 320; @@ -59,7 +59,7 @@ fn roundtrip_h264() -> Result<(), Box> { .with_gop_size(GOP); let mut encoder = Encoder::new(context.clone(), config)?; - let mut input_image = InputImage::for_encoder(&encoder)?; + let mut input_image = YuvUploader::for_encoder(&encoder)?; let mut bitstream = Vec::new(); let mut pending: VecDeque = VecDeque::new(); diff --git a/tests/submit.rs b/tests/submit.rs index ee5f8e9e..0612812c 100644 --- a/tests/submit.rs +++ b/tests/submit.rs @@ -8,9 +8,9 @@ mod common; use ash::vk; use pixelforge::{ - BitDepth, Codec, ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, - InputFormat, InputImage, PixelFormat, RgbSource, Source, SourceImage, VideoContext, - VideoContextBuilder, YuvLayout, + BitDepth, ChromaFormat, Codec, ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, + Handoff, RgbFormat, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, + YuvLayout, YuvUploader, }; const WIDTH: u32 = 320; @@ -29,7 +29,7 @@ fn context() -> Result> { fn an_input_slot_is_valid_until_the_next_submission() -> Result<(), Box> { let context = context()?; let mut encoder = Encoder::new(context.clone(), EncodeConfig::h264(WIDTH, HEIGHT))?; - let input = InputImage::for_encoder(&encoder)?; + let input = YuvUploader::for_encoder(&encoder)?; let stale = encoder.input_slot()?; let packet = encoder.submit(Frame::image(input.source()))?; @@ -52,12 +52,12 @@ fn an_input_slot_is_valid_until_the_next_submission() -> Result<(), Box Result<(), Box> { let context = context()?; - let input = InputImage::new( + let input = YuvUploader::new( context.clone(), WIDTH, HEIGHT, BitDepth::Eight, - PixelFormat::Yuv420, + ChromaFormat::Yuv420, )?; for layout in [vk::ImageLayout::UNDEFINED, vk::ImageLayout::PREINITIALIZED] { assert!( @@ -76,7 +76,7 @@ fn a_shader_conversion_has_no_input_slot() -> Result<(), Box Result<(), Box Result, Box> { let config = EncodeConfig::h265(WIDTH, HEIGHT).with_bit_depth(depth); let mut encoder = Encoder::new(context.clone(), config)?; - let mut input = InputImage::for_encoder(&encoder)?; + let mut input = YuvUploader::for_encoder(&encoder)?; let mut stream = Vec::new(); for i in 0..FRAMES { let data = frame(layout, i); @@ -120,7 +120,7 @@ fn ten_bit_and_eight_bit_upload_the_same_frame() -> Result<(), Box Result<(), Box> { let context = context()?; let encoder = Encoder::new(context.clone(), EncodeConfig::h265(WIDTH, HEIGHT))?; - let mut input = InputImage::for_encoder(&encoder)?; + let mut input = YuvUploader::for_encoder(&encoder)?; let size = |layout: YuvLayout| layout.frame_size(WIDTH, HEIGHT); for layout in [YuvLayout::I420P10, YuvLayout::I444] { assert!( diff --git a/tests/verify_all.rs b/tests/verify_all.rs index cc48a6e7..d686ce83 100644 --- a/tests/verify_all.rs +++ b/tests/verify_all.rs @@ -11,8 +11,8 @@ mod common; use pixelforge::{ - BitDepth, Codec, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, - VideoContextBuilder, YuvLayout, + BitDepth, ChromaFormat, Codec, EncodeConfig, Encoder, Frame, RateControl, VideoContextBuilder, + YuvLayout, YuvUploader, }; use std::fs::File; use std::io::{Read, Write}; @@ -49,18 +49,18 @@ fn verify_all() -> Result<(), Box> { let dir = common::scratch_dir("verify_all"); let combinations = [ - (Codec::H264, BitDepth::Eight, PixelFormat::Yuv420), - (Codec::H264, BitDepth::Eight, PixelFormat::Yuv444), - (Codec::H264, BitDepth::Ten, PixelFormat::Yuv420), - (Codec::H264, BitDepth::Ten, PixelFormat::Yuv444), - (Codec::H265, BitDepth::Eight, PixelFormat::Yuv420), - (Codec::H265, BitDepth::Eight, PixelFormat::Yuv444), - (Codec::H265, BitDepth::Ten, PixelFormat::Yuv420), - (Codec::H265, BitDepth::Ten, PixelFormat::Yuv444), - (Codec::AV1, BitDepth::Eight, PixelFormat::Yuv420), - (Codec::AV1, BitDepth::Eight, PixelFormat::Yuv444), - (Codec::AV1, BitDepth::Ten, PixelFormat::Yuv420), - (Codec::AV1, BitDepth::Ten, PixelFormat::Yuv444), + (Codec::H264, BitDepth::Eight, ChromaFormat::Yuv420), + (Codec::H264, BitDepth::Eight, ChromaFormat::Yuv444), + (Codec::H264, BitDepth::Ten, ChromaFormat::Yuv420), + (Codec::H264, BitDepth::Ten, ChromaFormat::Yuv444), + (Codec::H265, BitDepth::Eight, ChromaFormat::Yuv420), + (Codec::H265, BitDepth::Eight, ChromaFormat::Yuv444), + (Codec::H265, BitDepth::Ten, ChromaFormat::Yuv420), + (Codec::H265, BitDepth::Ten, ChromaFormat::Yuv444), + (Codec::AV1, BitDepth::Eight, ChromaFormat::Yuv420), + (Codec::AV1, BitDepth::Eight, ChromaFormat::Yuv444), + (Codec::AV1, BitDepth::Ten, ChromaFormat::Yuv420), + (Codec::AV1, BitDepth::Ten, ChromaFormat::Yuv444), ]; let context = VideoContextBuilder::new() @@ -106,7 +106,7 @@ fn run_test( dir: &std::path::Path, codec: Codec, depth: BitDepth, - format: PixelFormat, + format: ChromaFormat, ) -> Result> { // AV1 uses .obu extension for raw OBU streams (with temporal delimiters). // H.264/H.265 use .bin for raw Annex B bitstreams. @@ -130,7 +130,7 @@ fn run_test( Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), } .with_rate_control(RateControl::Cqp { qp: 10 }) - .with_pixel_format(format) + .with_chroma_format(format) .with_bit_depth(depth); let mut encoder = match Encoder::new(context.clone(), config) { @@ -138,11 +138,15 @@ fn run_test( Err(e) => return Err(format!("Failed to create encoder: {}", e).into()), }; - let mut input_image = InputImage::for_encoder(&encoder)?; + let mut input_image = YuvUploader::for_encoder(&encoder)?; let input_path = match format { - PixelFormat::Yuv420 => format!("testdata/test_frames_{}x{}_yuv420p.yuv", WIDTH, HEIGHT), - PixelFormat::Yuv444 => format!("testdata/test_frames_{}x{}_yuv444p.yuv", WIDTH, HEIGHT), + ChromaFormat::Yuv420 => { + format!("testdata/test_frames_{}x{}_yuv420p.yuv", WIDTH, HEIGHT) + } + ChromaFormat::Yuv444 => { + format!("testdata/test_frames_{}x{}_yuv444p.yuv", WIDTH, HEIGHT) + } _ => return Err("Unsupported format".into()), }; @@ -150,8 +154,8 @@ fn run_test( File::open(&input_path)?.read_to_end(&mut yuv_data)?; let frame_size = match format { - PixelFormat::Yuv420 => (WIDTH * HEIGHT * 3 / 2) as usize, - PixelFormat::Yuv444 => (WIDTH * HEIGHT * 3) as usize, + ChromaFormat::Yuv420 => (WIDTH * HEIGHT * 3 / 2) as usize, + ChromaFormat::Yuv444 => (WIDTH * HEIGHT * 3) as usize, _ => return Err("Unsupported format".into()), }; @@ -170,12 +174,12 @@ fn run_test( let frame = &yuv_data[start..end]; // Upload directly to encoder's input image to avoid cross-queue - // copy issues (InputImage uses the transfer queue, encoder uses the + // copy issues (YuvUploader uses the transfer queue, encoder uses the // video encode queue which doesn't support transfer ops). let slot = encoder.input_slot()?; let layout = match format { - PixelFormat::Yuv420 => YuvLayout::I420, - PixelFormat::Yuv444 => YuvLayout::I444, + ChromaFormat::Yuv420 => YuvLayout::I420, + ChromaFormat::Yuv444 => YuvLayout::I444, _ => return Err("Unsupported format".into()), }; input_image.upload_to(&slot, layout, frame)?; @@ -210,11 +214,11 @@ fn run_test( // Let's decode to the input format (8-bit). let (input_pix_fmt, input_path) = match format { - PixelFormat::Yuv420 => ( + ChromaFormat::Yuv420 => ( "yuv420p", format!("testdata/test_frames_{}x{}_yuv420p.yuv", WIDTH, HEIGHT), ), - PixelFormat::Yuv444 => ( + ChromaFormat::Yuv444 => ( "yuv444p", format!("testdata/test_frames_{}x{}_yuv444p.yuv", WIDTH, HEIGHT), ),