feat: add initial decoding for h265 and av1

AGENTS.md

@@ -46,6 +46,30 @@ cargo run --example decode -- tests/data/bframes.264 out.yuv \
     && cmp ref.yuv out.yuv && echo "decode matches ffmpeg"
 ```
 
+The `decode` example picks the codec from the extension (`.264`, `.265`,
+`.obu`, `.ivf`). For AV1 the reference is dav1d, and `-noautoscale` keeps a
+stream that changes size from being scaled to its first size (it applies to
+any codec; H.264 and H.265 streams in `tests/data` do not change size):
+
+```bash
+cargo run --example decode -- tests/data/av1/aom_altref.obu out.yuv \
+    && ffmpeg -hide_banner -loglevel error -y -c:v libdav1d -i tests/data/av1/aom_altref.obu \
+        -noautoscale -pix_fmt nv12 ref.yuv \
+    && cmp ref.yuv out.yuv && echo "decode matches dav1d"
+```
+
+`tests/decode_streams.rs` does this for every fixture in `tests/data` (use
+`p010le` for the 10-bit ones), and `tests/roundtrip.rs` for each codec's own
+encoder output. A run with `PIXELFORGE_NO_UNIFIED_LAYOUTS=1` set takes the
+copying path below.
+
+On NVIDIA (615.78.08) the AV1 decoder does not copy the `StdVideoAV1SequenceHeader`
+handed to `vkCreateVideoSessionParametersKHR`: it reads it again at every
+decode. The decoder keeps it alive for as long as the session parameters. Freed,
+nothing fails and only the chroma of inter frames comes out slightly wrong, so
+if AV1 inter frames ever differ from dav1d in chroma alone, look for a Std
+structure that does not live long enough.
+
 Decoding has two paths and both need checking. Where the device supports
 `VK_KHR_unified_image_layouts`, frames are the decoder's own images, handed over
 with no copy. Where it does not, they are copied into private images. Force the
@@ -57,7 +81,8 @@ PIXELFORGE_NO_UNIFIED_LAYOUTS=1 cargo run --example decode -- \
 ```
 
 Which path a run took is in the debug log (`RUST_LOG=debug`), on the
-`H.264 decode session:` line as `pinnable=`.
+`<codec> decode session:` line (for example `H.264 decode session:`) as
+`pinnable=`.
 
 The render path has its own example. `sample_frame` reads decoded frames in a
 compute shader through a sampler-YCbCr conversion, which is the path that never
@@ -199,7 +224,8 @@ view by image handle, and only RADV without validation showed it.
 - `benches/` - The encode timing/throughput benchmark (`cargo bench`)
 - `testdata/` - Test input files (note: `test_frames.yuv` is a git-LFS pointer;
   generate frames locally if LFS content is unavailable)
-- `tests/data/` - Small H.264 streams used by the decoder tests
+- `tests/data/` - Small H.264, H.265 and AV1 (`tests/data/av1/`) streams used
+  by the decoder tests
 
 ## Key Dependencies
 - Vulkan Video API

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add H.265 decoding: `DecodeConfig::new(Codec::H265)`. Main and Main 10, open
+  and closed GOPs, joining a stream at a CRA, scaling lists, temporal layers.
+- Add AV1 decoding: `DecodeConfig::new(Codec::AV1)`, for low-overhead OBU
+  streams. Hidden frames and `show_existing_frame`, tiles, superres, frame
+  size changes, 10-bit and film grain.
 - Add `EncodeConfig::new(codec, width, height)`, for a codec chosen at
   runtime.
 - Add RGB input: `EncodeConfig::with_source(Source::Rgb(..))`. The encoder
@@ -52,6 +57,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Breaking:** `ColorConverter`, `ColorConverterConfig` and `OutputFormat`
   are no longer public. Describe an RGB source with `Source::Rgb` instead, and
   the encoder runs the converter itself, with the right target and range.
+- **Breaking:** `DecodeConfig::new(codec)` replaces `DecodeConfig::h264()`,
+  in the same way as `EncodeConfig::new`.
 - **Breaking:** `RateControl` replaces `RateControlMode`, `quality_level`,
   `target_bitrate` and `max_bitrate`: `Cqp { qp }`, `Cbr { bitrate }` or
   `Vbr { target, max }`, so each mode carries exactly the values it uses.

README.md

@@ -5,12 +5,12 @@
 [![Documentation](https://docs.rs/pixelforge/badge.svg)](https://docs.rs/pixelforge)
 
 A Vulkan-based video encoding and decoding library for Rust, supporting H.264,
-H.265 and AV1 encode, and H.264 decode.
+H.265 and AV1, for both encode and decode.
 
 ## Features
 
 - **Hardware-accelerated** video encoding and decoding using Vulkan Video extensions.
-- **Multiple codec support**: H.264/AVC, H.265/HEVC, AV1 encode; H.264 decode.
+- **Multiple codec support**: H.264/AVC, H.265/HEVC and AV1, encode and decode.
 - **Asynchronous pipelines**: both directions submit without waiting.
   Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a
   [`DecodeSource`] as the GPU finishes with them.
@@ -37,11 +37,12 @@ H.265 and AV1 encode, and H.264 decode.
 | Codec | Encode | Decode |
 |-------|--------|--------|
 | H.264/AVC | ✓ | ✓ |
-| H.265/HEVC | ✓ | |
-| AV1 | ✓ | |
+| H.265/HEVC | ✓ | ✓ |
+| AV1 | ✓ | ✓ |
 
 H.264 decoding is verified byte-identical to `ffmpeg -pix_fmt nv12` on AMD
-(RADV), NVIDIA and Intel (ANV).
+(RADV), NVIDIA and Intel (ANV). H.265 decoding is verified byte-identical to
+ffmpeg, and AV1 decoding (film grain included) to dav1d, on NVIDIA so far.
 
 ## Requirements
 
@@ -152,7 +153,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // A file can cut anywhere, so let the decoder frame it. Input that
     // arrives already framed (RTP, a container) skips `with_byte_stream`.
-    let config = DecodeConfig::h264().with_byte_stream();
+    let config = DecodeConfig::new(Codec::H264).with_byte_stream();
     let mut decoder = Decoder::new(context, config)?;
     let stream: Vec<u8> = std::fs::read("input.264")?;
 

examples/common/mod.rs

@@ -17,29 +17,90 @@
 //!   from the same thread.
 
 use ash::vk;
-use pixelforge::decoder::{DecodedFrame, Decoder, FramePoll};
-use pixelforge::encoder::BitDepth;
+use pixelforge::decoder::{DecodeConfig, DecodedFrame, Decoder, FramePoll};
+use pixelforge::encoder::{BitDepth, Codec};
 use pixelforge::vulkan::VideoContext;
 use std::fs::File;
 use std::io::Write;
 
 /// Bytes handed to the decoder per call, standing in for a network read.
 pub const CHUNK_SIZE: usize = 64 * 1024;
 
-/// Drive `decoder` over `stream`, calling `on_frame` for every frame it emits.
+/// A coded stream read from a file.
+pub enum Input {
+    /// A raw stream that the decoder frames itself: Annex B for H.264 and
+    /// H.265, low-overhead OBUs for AV1.
+    ByteStream(Vec<u8>),
+    /// Whole coded frames, as a container gives them (IVF).
+    Frames(Vec<Vec<u8>>),
+}
+
+/// Read `path`, and pick the codec from its extension: `.264`/`.h264`,
+/// `.265`/`.h265`/`.hevc`, `.obu`/`.av1` or `.ivf` (AV1 in IVF).
+pub fn open_input(path: &str) -> Result<(Codec, Input), Box<dyn std::error::Error>> {
+    let extension = std::path::Path::new(path)
+        .extension()
+        .and_then(|e| e.to_str())
+        .unwrap_or("")
+        .to_ascii_lowercase();
+    let data = std::fs::read(path)?;
+    Ok(match extension.as_str() {
+        "264" | "h264" => (Codec::H264, Input::ByteStream(data)),
+        "265" | "h265" | "hevc" => (Codec::H265, Input::ByteStream(data)),
+        "obu" | "av1" => (Codec::AV1, Input::ByteStream(data)),
+        "ivf" => (Codec::AV1, Input::Frames(ivf_frames(&data)?)),
+        other => return Err(format!("unknown stream extension {other:?}").into()),
+    })
+}
+
+/// The frames of an IVF file: a 32-byte file header, then each frame behind a
+/// 12-byte header holding its size and timestamp.
+fn ivf_frames(data: &[u8]) -> Result<Vec<Vec<u8>>, Box<dyn std::error::Error>> {
+    if data.len() < 32 || &data[..4] != b"DKIF" {
+        return Err("not an IVF file".into());
+    }
+    let header_len = u16::from_le_bytes([data[6], data[7]]) as usize;
+    let mut frames = Vec::new();
+    let mut pos = header_len;
+    while pos + 12 <= data.len() {
+        let size = u32::from_le_bytes(data[pos..pos + 4].try_into()?) as usize;
+        let start = pos + 12;
+        let frame = data.get(start..start + size).ok_or("truncated IVF frame")?;
+        frames.push(frame.to_vec());
+        pos = start + size;
+    }
+    Ok(frames)
+}
+
+/// The decode configuration `input` needs: byte-stream framing for a raw
+/// stream, frame-aligned for frames from a container.
+pub fn decode_config(codec: Codec, input: &Input) -> DecodeConfig {
+    match input {
+        Input::ByteStream(_) => DecodeConfig::new(codec).with_byte_stream(),
+        Input::Frames(_) => DecodeConfig::new(codec),
+    }
+}
+
+/// Drive `decoder` over `input`, calling `on_frame` for every frame it emits.
 ///
-/// Frames are polled after each chunk and again after `finish`, so the
-/// in-stream and end-of-stream paths share one callback. `NeedsKeyframe` is not
+/// A raw stream is fed in chunks of [`CHUNK_SIZE`], standing in for network
+/// reads; frames from a container are fed one per call. Frames are polled
+/// after each call and again after `finish`, so the in-stream and
+/// end-of-stream paths share one callback. `NeedsKeyframe` is not
 /// special-cased: polling then is a no-op when nothing is ready.
-pub fn decode_stream<F>(
+pub fn decode_input<F>(
     decoder: &mut Decoder,
-    stream: &[u8],
+    input: &Input,
     mut on_frame: F,
 ) -> Result<(), Box<dyn std::error::Error>>
 where
     F: FnMut(DecodedFrame) -> Result<(), Box<dyn std::error::Error>>,
 {
-    for (i, chunk) in stream.chunks(CHUNK_SIZE).enumerate() {
+    let chunks: Box<dyn Iterator<Item = &[u8]>> = match input {
+        Input::ByteStream(stream) => Box::new(stream.chunks(CHUNK_SIZE)),
+        Input::Frames(frames) => Box::new(frames.iter().map(Vec::as_slice)),
+    };
+    for (i, chunk) in chunks.enumerate() {
         decoder.decode(chunk, i as u64)?;
         while let FramePoll::Frame(frame) = decoder.try_next_frame()? {
             on_frame(frame)?;
@@ -283,10 +344,13 @@ impl Readback {
 
         // Crop to the visible region on the way out: the image is coded-size,
         // which is padded up to the codec's macroblock granularity.
+        // An odd size rounds the chroma up: the last chroma sample covers
+        // the last luma column (and row) on its own.
         let visible_y_stride = frame.width as usize * bytes;
-        let visible_uv_stride = visible_y_stride;
+        let visible_uv_stride = frame.width.div_ceil(2) as usize * 2 * bytes;
+        let uv_rows = frame.height.div_ceil(2) as usize;
         let mut y = Vec::with_capacity(visible_y_stride * frame.height as usize);
-        let mut uv = Vec::with_capacity(visible_uv_stride * frame.height as usize / 2);
+        let mut uv = Vec::with_capacity(visible_uv_stride * uv_rows);
         unsafe {
             // Map the whole allocation: a sub-range map has to respect
             // nonCoherentAtomSize, and WHOLE_SIZE sidesteps that.
@@ -300,7 +364,7 @@ impl Readback {
                     visible_y_stride,
                 ));
             }
-            for row in 0..frame.height as usize / 2 {
+            for row in 0..uv_rows {
                 uv.extend_from_slice(std::slice::from_raw_parts(
                     ptr.add(y_size + row * uv_stride),
                     visible_uv_stride,

examples/decode.rs

@@ -1,4 +1,7 @@
-//! Decode an H.264 Annex B stream to raw YUV.
+//! Decode an H.264, H.265 or AV1 stream to raw YUV.
+//!
+//! The codec comes from the file extension: `.264`, `.265`/`.hevc`, `.obu`
+//! (AV1 OBUs) or `.ivf` (AV1 in IVF).
 //!
 //! Usage:
 //!   cargo run --example decode -- input.264 [output.yuv]
@@ -13,21 +16,21 @@ use std::fs::File;
 
 #[allow(dead_code)]
 mod common;
-use common::{Readback, decode_stream, write_nv12};
+use common::{Readback, decode_input, write_nv12};
 
-use pixelforge::decoder::{DecodeConfig, Decoder};
-use pixelforge::encoder::Codec;
+use pixelforge::decoder::Decoder;
 use pixelforge::vulkan::VideoContextBuilder;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     tracing_subscriber::fmt::init();
 
     let mut args = std::env::args().skip(1);
     let input_path = args.next().unwrap_or_else(|| {
-        eprintln!("usage: decode <input.264> [output.yuv]");
+        eprintln!("usage: decode <input.264|.265|.obu|.ivf> [output.yuv]");
         std::process::exit(1);
     });
     let output_path = args.next();
+    let (codec, input) = common::open_input(&input_path)?;
 
     // PIXELFORGE_VALIDATION=1 enables the Vulkan validation layers, which
     // report the exact offending call for any API misuse.
@@ -36,7 +39,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // hardware that would otherwise take the zero-copy path.
     let mut builder = VideoContextBuilder::new()
         .app_name("pixelforge-decode")
-        .require_decode(Codec::H264)
+        .require_decode(codec)
         .enable_validation(validation);
     if std::env::var("PIXELFORGE_NO_UNIFIED_LAYOUTS").is_ok() {
         builder = builder.without_unified_image_layouts();
@@ -59,7 +62,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // sized for one frame per call, so raise it: past the budget the decoder
     // copies pictures out instead of handing over its own images, which is
     // still correct but gives up the zero-copy path.
-    let config = DecodeConfig::h264().with_byte_stream().with_output_depth(8);
+    let config = common::decode_config(codec, &input).with_output_depth(8);
     let mut output = output_path.as_ref().map(File::create).transpose()?;
     // Readback lives in the consumer, not in pixelforge: the decoder hands over
     // a GPU image and a renderer would sample it in place. See examples/common.
@@ -69,7 +72,6 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .then(|| Readback::new(&context))
         .transpose()?;
     let mut decoder = Decoder::new(context, config)?;
-    let stream = std::fs::read(&input_path)?;
 
     let start = std::time::Instant::now();
     let mut frame_count = 0usize;
@@ -81,8 +83,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // flight, not that there are none, so they are collected on a later pass.
     // A renderer would instead `split()` the decoder and await `next_frame` on
     // its own thread, which is what the `Decoder::split` docs show.
-    // `common::decode_stream` owns that loop.
-    decode_stream(&mut decoder, &stream, |frame| {
+    // `common::decode_input` owns that loop.
+    decode_input(&mut decoder, &input, |frame| {
         write_nv12(&frame, &mut readback, &mut output)?;
         // A rebuilt session means a new set of images, so anything caching
         // per-image state has to notice. Reporting it here is what makes the

examples/sample_frame.rs

@@ -26,13 +26,12 @@ use std::fs::File;
 use std::io::Write;
 
 use ash::vk::{self, TaggedStructure as _};
-use pixelforge::decoder::{DecodeConfig, DecodedFrame, Decoder};
-use pixelforge::encoder::Codec;
+use pixelforge::decoder::{DecodedFrame, Decoder};
 use pixelforge::vulkan::{VideoContext, VideoContextBuilder};
 
 #[allow(dead_code)]
 mod common;
-use common::decode_stream;
+use common::decode_input;
 
 const SHADER: &[u8] = include_bytes!("shader/sample_frame.spv");
 
@@ -41,15 +40,16 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let mut args = std::env::args().skip(1);
     let input_path = args.next().unwrap_or_else(|| {
-        eprintln!("usage: sample_frame <input.264> [output.rgba]");
+        eprintln!("usage: sample_frame <input.264|.265|.obu|.ivf> [output.rgba]");
         std::process::exit(1);
     });
     let output_path = args.next();
+    let (codec, input) = common::open_input(&input_path)?;
 
     let validation = std::env::var("PIXELFORGE_VALIDATION").is_ok();
     let context = VideoContextBuilder::new()
         .app_name("pixelforge-sample")
-        .require_decode(Codec::H264)
+        .require_decode(codec)
         .enable_validation(validation)
         .build()?;
 
@@ -63,12 +63,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // picture's sharing set. Without this, sampling a decoded image from a
     // family the decoder did not share with is undefined.
     let consumer_family = context.compute_queue_family();
-    let config = DecodeConfig::h264()
-        .with_byte_stream()
+    let config = common::decode_config(codec, &input)
         .with_output_depth(8)
         .with_consumer_queue_family(consumer_family);
 
-    let stream = std::fs::read(&input_path)?;
     let mut output = output_path.as_ref().map(File::create).transpose()?;
     // Built lazily: the picture format is only known once the stream's first
     // parameter set has been parsed, and the ycbcr conversion depends on it.
@@ -78,7 +76,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut count = 0usize;
     let start = std::time::Instant::now();
 
-    decode_stream(&mut decoder, &stream, |frame| {
+    decode_input(&mut decoder, &input, |frame| {
         let sampler = match &mut sampler {
             Some(s) => s,
             none => none.insert(Sampler::new(&context, consumer_family, &frame)?),